{
  "schema_version": "ax.native_model.v1",
  "model_family": "qwen3_5",
  "tensor_format": "safetensors",
  "layer_count": 40,
  "hidden_size": 2048,
  "intermediate_size": 0,
  "attention_head_count": 16,
  "attention_head_dim": 256,
  "kv_head_count": 2,
  "vocab_size": 248320,
  "tie_word_embeddings": false,
  "rope_theta": 10000000,
  "no_rope_layer_interval": 0,
  "intermediate_size_mlp": 0,
  "attn_output_gate": true,
  "partial_rotary_factor": 0.25,
  "rms_norm_eps": 0.000001,
  "moe_norm_topk_prob": true,
  "hidden_size_per_layer_input": 0,
  "linear_attention": {
    "full_attention_interval": 4,
    "num_value_heads": 32,
    "num_key_heads": 16,
    "key_head_dim": 128,
    "value_head_dim": 128,
    "conv_kernel_dim": 4
  },
  "moe": {
    "expert_count": 256,
    "experts_per_token": 8,
    "expert_intermediate_size": 512,
    "sigmoid_routing": false
  },
  "think_start_token_id": 248068,
  "think_end_token_id": 248069,
  "tensors": [
    {
      "name": "language_model.model.norm.weight",
      "role": "final_norm",
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 837260189,
      "length_bytes": 4096
    },
    {
      "name": "language_model.lm_head.weight",
      "role": "lm_head",
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        248320,
        512
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 17009565,
      "length_bytes": 508559360
    },
    {
      "name": "language_model.model.embed_tokens.weight",
      "role": "token_embedding",
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        248320,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4292335354,
      "length_bytes": 508559360
    },
    {
      "name": "language_model.model.layers.0.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4842022650,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.0.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4816787194,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3434849978,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2250852410,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.0.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3998201594,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3269382906,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3291162298,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2545604858,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3281411706,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3729766138,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.0.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4902915898,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.0.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2839706234,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.0.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3271875834,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2234320954,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5317275962,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2839771770,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2831252090,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.0.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5316685882,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.0.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2554063482,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.1.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1445666810,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.1.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2991422074,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4902915962,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2856548986,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.1.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2990799482,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5305835514,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5326125370,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2545606906,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2232746042,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2562816634,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.1.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3268594298,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.1.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3271548154,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.1.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3268594362,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2991356538,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 835933946,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4275551994,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1457125370,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.1.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3281345914,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.1.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2991753850,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.2.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4292329210,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.2.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2072729594,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1509851130,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3000158842,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.2.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2072205306,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1465810938,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1466859514,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1457090554,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1134190586,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1795332090,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.2.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5305831354,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.2.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2528200826,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.2.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2098198522,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2232418362,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4825179898,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1467908090,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4886103866,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.2.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2554063226,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.2.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2089809914,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.3.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1447063546,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.3.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 827217146,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.3.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 692913082,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.3.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5317605690,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.3.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2545076474,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.3.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2072737786,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.3.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3270990586,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.3.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1446014970,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 865755130,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1168809978,
      "length_bytes": 268435456
    },
    {
      "name": "language_model.model.layers.3.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4868243194,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5307146234,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 855269370,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 844650426,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5344999738,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5171613626,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 692384442,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.4.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2072733690,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 549237434,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2098198586,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 691860154,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 530326202,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 531407546,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4292333306,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 854187962,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4132945658,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 692917178,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.4.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3281346170,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.4.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 835999482,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 827151610,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 827217658,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4843077370,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1484685306,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.4.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 692912826,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.4.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1437278202,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.5.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 494731834,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.5.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 540844730,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3292243642,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 359987770,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 494207546,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 856317946,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 333773370,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 494205498,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3271875898,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 692933626,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2528200762,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.5.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 530195002,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.5.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4877156090,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 512561722,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5316686138,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 513417786,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3272924474,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.5.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 827151354,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.5.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 2545641722,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.6.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 314792122,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.6.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 306399418,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 29558970,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 172165306,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 494998074,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 20121786,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3703547578,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 20054202,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 844652474,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3998727930,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 530260602,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.6.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 333576762,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.6.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 315320762,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 333511226,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 323746298,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 5327698234,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 324073978,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.6.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 315320506,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.6.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 495784506,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.7.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1179834,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.7.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 588986,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.7.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 1786937338,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.7.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 845766586,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.7.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 584890,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.7.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 3276986,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.7.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 589498,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.7.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 4842028794,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 928194602,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5026801450,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.7.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00005.safetensors",
      "offset_bytes": 60602,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4544115306,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4400126506,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1062412330,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4221441514,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4401240618,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3517338794,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.8.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4535460394,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4882750634,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4553618538,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1384026730,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4068085162,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4203351530,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4687950954,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2916768682,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1240304170,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5358913322,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.8.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4265843242,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.8.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2764966762,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3882317226,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1403216618,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5170489130,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4840787050,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.8.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2441357930,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.8.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3535910122,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.9.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3881194858,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.9.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3230428202,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3720987882,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5196347178,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3872019818,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3534861546,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3881201066,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1384551018,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2753936234,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3578117354,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3881198954,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.9.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 928096298,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.9.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4874345578,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3882249642,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4401175082,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4050189738,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4025023914,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.9.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1430317866,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.9.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4249065514,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.10.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3061537706,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.10.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4066966954,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3882384810,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2134837482,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2439784810,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2917817258,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 760563562,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2270110058,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1213786154,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3230465066,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3230428074,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.10.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3517916330,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.10.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3230428138,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1939720426,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3374138474,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5342134058,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1412492074,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.10.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4222522922,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.10.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3213650858,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.11.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2610246506,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.11.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5016968362,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.11.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2764962666,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.11.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4832398442,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.11.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2916600746,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.11.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3561340138,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.11.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4545163882,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.11.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5341085482,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2458200938,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3078908842,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.11.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3508421802,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5338988330,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5169407786,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4204662250,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2440309354,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2611295082,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 473315178,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.12.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3517334698,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4265908778,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1974388970,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2108623082,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1939785962,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 770000746,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1938554794,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1938589866,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2918865834,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2270103786,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.12.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1213720618,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.12.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2270103850,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4687885418,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1590981738,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4857564266,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2278762858,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.12.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4249065258,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.12.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2126448874,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.13.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1592687786,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.13.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1787391786,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4069133738,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1456764010,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 925175722,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1438805098,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4066971050,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1402935978,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1420880682,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1619062570,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1601408426,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.13.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1804173098,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.13.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 473313066,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1609801194,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1601342890,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1770614570,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1761668906,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.13.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1601342634,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.13.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1601408490,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.14.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5196277546,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.14.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1240037930,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1062709290,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2765294506,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1592112234,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 927047722,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3373087786,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1403314986,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2269055210,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3374204074,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4203351466,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.14.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1938489258,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.14.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1383696938,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1383697002,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2916703146,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1385077354,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3508946090,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.14.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1619062314,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.14.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1375308330,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.15.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 924127146,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.15.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1240037418,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.15.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 3871982826,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.15.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 914491242,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.15.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4696341610,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.15.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4033412522,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.15.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4265842730,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.15.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 922879850,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 492128106,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 779437930,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.15.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 913655658,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2754984810,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2592418666,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5016968874,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 491079530,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 626345834,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1231628330,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.16.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 927010794,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 178841322,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4697918570,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 472491818,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 169371370,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4696345706,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1609866730,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1401887402,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 329836266,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1939687594,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.16.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 491013994,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.16.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1592687722,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1430350890,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 473319274,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 474171242,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2756574058,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.16.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1938556842,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.16.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 5161019178,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.17.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 151633514,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.17.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 151629418,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2288724330,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 16871018,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 151105130,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4462029511,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1591063658,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3982316039,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3209042439,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1804238698,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1618259434,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.17.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 169240234,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.17.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 160550506,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1456665706,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 923996010,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 1787395882,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 4874345642,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.17.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 2440309098,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.17.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00002-of-00005.safetensors",
      "offset_bytes": 152161898,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.18.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4632738631,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.18.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 5247532807,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2598619463,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1630139079,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3419763783,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3001272711,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4127732295,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3616730247,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2597304583,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4490115911,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4775353671,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.18.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4290984519,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.18.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3001272647,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4442831431,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3002845639,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4811140871,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 771212807,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.18.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2400923271,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.18.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3382850503,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.19.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3227458375,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.19.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 797007751,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.19.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4125514311,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.19.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4128813639,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.19.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3447227463,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.19.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3948202503,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.19.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4775349063,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.19.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4272517703,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4970708743,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2732837191,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.19.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1474906759,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3670047239,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4479892167,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3973368327,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3668965895,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 806773127,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3616732295,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.20.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4462025415,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3805313543,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3482233927,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1077830023,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4809828103,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3447528519,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4811138823,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1075208583,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3982318151,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3617015239,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.20.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3660511751,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.20.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2598615303,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3208976903,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4127666759,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3365759879,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4273599047,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.20.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4809303559,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.20.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3210091015,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.21.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4451994247,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.21.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3208710663,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3231015815,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4137251399,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3199666695,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3392287687,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3181824519,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3473843271,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3002944007,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2077235975,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3227458311,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.21.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3419698247,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.21.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3230178119,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3660446215,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3208645127,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4463082183,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3410146247,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.21.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3227458055,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.21.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3219036679,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.22.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4463078087,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.22.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3634158087,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4836425479,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2444912647,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2579216391,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4460976839,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2076168967,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2419349447,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1076257159,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4632742727,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1474904647,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.22.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4451926663,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.22.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4451998343,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3429233735,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4970643207,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3393369031,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4766960455,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.22.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2597304327,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.22.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2384146055,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.23.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4125520455,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.23.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3617277447,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.23.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2401451911,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.23.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1924264647,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.23.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2401447815,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.23.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2402539399,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.23.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3617277959,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.23.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2401456007,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2249927815,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3671095815,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.23.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2400923527,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2230197383,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2231315591,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3448577095,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2229083271,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2867054919,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1475463815,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.24.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2066870983,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1495528135,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1789784775,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3230178183,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1484939911,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3138210311,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1474904711,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1475467911,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 5113315079,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 5247536967,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.24.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3200190983,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.24.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2067137223,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3447161927,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2067190535,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2212306055,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2067256071,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.24.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3617014983,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.24.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2428135431,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.25.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2579212295,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.25.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4775349575,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1096214983,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4291574343,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4489591559,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1078356423,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1485992647,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1078354375,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4271469127,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1247211975,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3982318087,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.25.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4827918087,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.25.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1415508423,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1432287751,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1440743943,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1458110983,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4425792071,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.25.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 628606215,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.25.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1407119815,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.26.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1485988551,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.26.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 779634183,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 628606471,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1932653255,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3420320839,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 805691783,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3228506951,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 618839303,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 618841351,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 940990855,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 779900679,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.26.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2231248007,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.26.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 796677959,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 796680071,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3418534855,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3429332039,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1398731207,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.26.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 779900423,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.26.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1440809479,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.27.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 617266375,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.27.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 608221895,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.27.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 608217799,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.27.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 608746695,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.27.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3446109255,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.27.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1079404999,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.27.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2384145543,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.27.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3446113351,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3003992583,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 473443015,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.27.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 607693511,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 3391239111,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4126618183,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 312910535,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 312978119,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 322448071,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 294226439,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.28.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 294222343,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4482030380,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 159447559,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 293698055,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4872040556,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4463102764,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4277103020,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3226642412,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 16841223,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 5247536903,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.28.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1629745863,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.28.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 302881287,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 4775386567,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 302883399,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 1415508487,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 303473223,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.28.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 2211978119,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.28.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00003-of-00005.safetensors",
      "offset_bytes": 294492679,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.29.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4465199916,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.29.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2860276588,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4310958892,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4005288364,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4652854636,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2685534956,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2380264108,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4139506092,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4140655020,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4142752172,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4277105068,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.29.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2380198572,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.29.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4653905260,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4276969900,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 950085932,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3836451180,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4863651948,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.29.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4277105132,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.29.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3548727532,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.30.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4635794284,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.30.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3700361580,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4881529004,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3853736364,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2667606700,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4844478508,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4141703596,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3385606380,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3384522988,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 5015814316,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4873123948,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.30.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1717122092,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.30.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 932642924,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3853605292,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3047433196,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3988511148,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4635798380,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.30.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4793365932,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.30.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2677113516,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.31.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2378553772,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.31.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3056149484,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.31.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 789964908,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.31.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 5168187628,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.31.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3047394284,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.31.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3209340908,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.31.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3190367724,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.31.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 637233260,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1135966572,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2533388972,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.31.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4616313708,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2707643180,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4464151340,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4844378028,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2706594604,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2895971180,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2668425900,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.32.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4855226476,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3414477036,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3236376620,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2868706156,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2704970476,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3405039852,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 941103276,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4616837996,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2398089900,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2685502124,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.32.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3853261164,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.32.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3853605228,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3047015340,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4810426028,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3388229868,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2708691756,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.32.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4827600556,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.32.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3227955244,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.33.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2868669292,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.33.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2379635180,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2191207788,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3056149996,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4653380972,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2325425516,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4139606444,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2360881068,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2173349228,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4659148204,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3047132076,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.33.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 798359660,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.33.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2380163564,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4844412908,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2377941164,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2182819180,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2878145388,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.33.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2361161644,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.33.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2869756780,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.34.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2173279596,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.34.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2019296300,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1733985324,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3566111084,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3387705580,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4461988652,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2162433324,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1708106796,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2532340396,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1876624428,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2164530476,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.34.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 5015748780,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.34.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2379635116,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3853670828,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2172919148,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3191711724,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        4096,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4285496108,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.34.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2162431020,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.34.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2153780268,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.35.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2163481900,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.35.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4644186988,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.35.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1689651628,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.35.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1699617708,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.35.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1681226156,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.35.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1725596716,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.35.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1690704300,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.35.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3385608428,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 993360236,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1404402092,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.35.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2695533292,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3374823532,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3700889964,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 5015746732,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4617886572,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1270184300,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 968092012,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.36.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 932642988,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 655747180,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2717113132,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3700365676,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3190368236,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2894922604,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 635871340,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 636184684,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 798425196,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 941105324,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.36.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 941037740,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.36.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 941629676,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 950020396,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 958540076,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2668429996,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4653905324,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.36.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2378549420,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.36.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 932647084,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.37.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4793366188,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.37.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2378549676,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1547008428,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2019300396,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2706021100,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 582696748,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3386657004,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 635869292,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1689655724,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3702233452,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2380198444,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.37.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 635803756,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.37.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3375872172,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 635213932,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4139508140,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3038626732,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3370627116,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.37.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 626562924,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.37.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1538619820,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.38.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 568665516,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.38.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 568661420,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 140301740,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 433903020,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 568137132,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4854177836,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 139253164,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 138071468,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 138139052,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 282908076,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1404402028,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.38.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 3566045548,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.38.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 592199468,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 577582764,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        32,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4616248172,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4618967916,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 577912492,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.38.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 577582508,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.38.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 569193900,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.39.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 135974252,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.39.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 134826860,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.39.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 134822764,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.39.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        1024
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2361163948,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.39.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 1725576236,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.39.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        8192,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 4827600812,
      "length_bytes": 16777216
    },
    {
      "name": "language_model.model.layers.39.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 2343284076,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.39.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 134892908,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 837297117,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 64364,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.39.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00005.safetensors",
      "offset_bytes": 134298476,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        2048,
        128
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 525601693,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 535071645,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 544508829,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        512,
        512
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 15928221,
      "length_bytes": 1048576
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512,
        512
      ],
      "file": "model-00005-of-00005.safetensors",
      "offset_bytes": 544510877,
      "length_bytes": 268435456
    }
  ]
}