recipe.yaml

default_stage:
  default_modifiers:
    AWQModifier:
      targets: [Linear]
      ignore: [lm_head, 're:.*gate.weight', model.layers.0.mlp.*]
      scheme: W4A16
      bypass_divisibility_checks: false
      mappings:
      - smooth_layer: re:.*v_proj$
        balance_layers: ['re:.*o_proj$']
        activation_hook_target: null
      - smooth_layer: model.layers.1.post_attention_layernorm
        balance_layers: ['re:model\.layers\.1\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.1\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.1.mlp.shared_experts.gate_proj, model.layers.1.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.2.post_attention_layernorm
        balance_layers: ['re:model\.layers\.2\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.2\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.2.mlp.shared_experts.gate_proj, model.layers.2.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.3.post_attention_layernorm
        balance_layers: ['re:model\.layers\.3\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.3\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.3.mlp.shared_experts.gate_proj, model.layers.3.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.4.post_attention_layernorm
        balance_layers: ['re:model\.layers\.4\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.4\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.4.mlp.shared_experts.gate_proj, model.layers.4.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.5.post_attention_layernorm
        balance_layers: ['re:model\.layers\.5\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.5\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.5.mlp.shared_experts.gate_proj, model.layers.5.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.6.post_attention_layernorm
        balance_layers: ['re:model\.layers\.6\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.6\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.6.mlp.shared_experts.gate_proj, model.layers.6.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.7.post_attention_layernorm
        balance_layers: ['re:model\.layers\.7\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.7\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.7.mlp.shared_experts.gate_proj, model.layers.7.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.8.post_attention_layernorm
        balance_layers: ['re:model\.layers\.8\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.8\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.8.mlp.shared_experts.gate_proj, model.layers.8.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.9.post_attention_layernorm
        balance_layers: ['re:model\.layers\.9\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.9\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.9.mlp.shared_experts.gate_proj, model.layers.9.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.10.post_attention_layernorm
        balance_layers: ['re:model\.layers\.10\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.10\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.10.mlp.shared_experts.gate_proj, model.layers.10.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.11.post_attention_layernorm
        balance_layers: ['re:model\.layers\.11\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.11\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.11.mlp.shared_experts.gate_proj, model.layers.11.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.12.post_attention_layernorm
        balance_layers: ['re:model\.layers\.12\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.12\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.12.mlp.shared_experts.gate_proj, model.layers.12.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.13.post_attention_layernorm
        balance_layers: ['re:model\.layers\.13\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.13\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.13.mlp.shared_experts.gate_proj, model.layers.13.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.14.post_attention_layernorm
        balance_layers: ['re:model\.layers\.14\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.14\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.14.mlp.shared_experts.gate_proj, model.layers.14.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.15.post_attention_layernorm
        balance_layers: ['re:model\.layers\.15\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.15\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.15.mlp.shared_experts.gate_proj, model.layers.15.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.16.post_attention_layernorm
        balance_layers: ['re:model\.layers\.16\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.16\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.16.mlp.shared_experts.gate_proj, model.layers.16.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.17.post_attention_layernorm
        balance_layers: ['re:model\.layers\.17\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.17\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.17.mlp.shared_experts.gate_proj, model.layers.17.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.18.post_attention_layernorm
        balance_layers: ['re:model\.layers\.18\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.18\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.18.mlp.shared_experts.gate_proj, model.layers.18.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.19.post_attention_layernorm
        balance_layers: ['re:model\.layers\.19\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.19\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.19.mlp.shared_experts.gate_proj, model.layers.19.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.20.post_attention_layernorm
        balance_layers: ['re:model\.layers\.20\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.20\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.20.mlp.shared_experts.gate_proj, model.layers.20.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.21.post_attention_layernorm
        balance_layers: ['re:model\.layers\.21\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.21\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.21.mlp.shared_experts.gate_proj, model.layers.21.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.22.post_attention_layernorm
        balance_layers: ['re:model\.layers\.22\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.22\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.22.mlp.shared_experts.gate_proj, model.layers.22.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.23.post_attention_layernorm
        balance_layers: ['re:model\.layers\.23\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.23\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.23.mlp.shared_experts.gate_proj, model.layers.23.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.24.post_attention_layernorm
        balance_layers: ['re:model\.layers\.24\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.24\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.24.mlp.shared_experts.gate_proj, model.layers.24.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.25.post_attention_layernorm
        balance_layers: ['re:model\.layers\.25\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.25\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.25.mlp.shared_experts.gate_proj, model.layers.25.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.26.post_attention_layernorm
        balance_layers: ['re:model\.layers\.26\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.26\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.26.mlp.shared_experts.gate_proj, model.layers.26.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.27.post_attention_layernorm
        balance_layers: ['re:model\.layers\.27\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.27\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.27.mlp.shared_experts.gate_proj, model.layers.27.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.28.post_attention_layernorm
        balance_layers: ['re:model\.layers\.28\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.28\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.28.mlp.shared_experts.gate_proj, model.layers.28.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.29.post_attention_layernorm
        balance_layers: ['re:model\.layers\.29\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.29\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.29.mlp.shared_experts.gate_proj, model.layers.29.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.30.post_attention_layernorm
        balance_layers: ['re:model\.layers\.30\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.30\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.30.mlp.shared_experts.gate_proj, model.layers.30.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.31.post_attention_layernorm
        balance_layers: ['re:model\.layers\.31\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.31\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.31.mlp.shared_experts.gate_proj, model.layers.31.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.32.post_attention_layernorm
        balance_layers: ['re:model\.layers\.32\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.32\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.32.mlp.shared_experts.gate_proj, model.layers.32.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.33.post_attention_layernorm
        balance_layers: ['re:model\.layers\.33\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.33\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.33.mlp.shared_experts.gate_proj, model.layers.33.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.34.post_attention_layernorm
        balance_layers: ['re:model\.layers\.34\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.34\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.34.mlp.shared_experts.gate_proj, model.layers.34.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.35.post_attention_layernorm
        balance_layers: ['re:model\.layers\.35\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.35\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.35.mlp.shared_experts.gate_proj, model.layers.35.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.36.post_attention_layernorm
        balance_layers: ['re:model\.layers\.36\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.36\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.36.mlp.shared_experts.gate_proj, model.layers.36.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.37.post_attention_layernorm
        balance_layers: ['re:model\.layers\.37\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.37\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.37.mlp.shared_experts.gate_proj, model.layers.37.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.38.post_attention_layernorm
        balance_layers: ['re:model\.layers\.38\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.38\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.38.mlp.shared_experts.gate_proj, model.layers.38.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.39.post_attention_layernorm
        balance_layers: ['re:model\.layers\.39\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.39\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.39.mlp.shared_experts.gate_proj, model.layers.39.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.40.post_attention_layernorm
        balance_layers: ['re:model\.layers\.40\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.40\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.40.mlp.shared_experts.gate_proj, model.layers.40.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.41.post_attention_layernorm
        balance_layers: ['re:model\.layers\.41\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.41\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.41.mlp.shared_experts.gate_proj, model.layers.41.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.42.post_attention_layernorm
        balance_layers: ['re:model\.layers\.42\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.42\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.42.mlp.shared_experts.gate_proj, model.layers.42.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.43.post_attention_layernorm
        balance_layers: ['re:model\.layers\.43\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.43\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.43.mlp.shared_experts.gate_proj, model.layers.43.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.44.post_attention_layernorm
        balance_layers: ['re:model\.layers\.44\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.44\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.44.mlp.shared_experts.gate_proj, model.layers.44.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.45.post_attention_layernorm
        balance_layers: ['re:model\.layers\.45\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.45\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.45.mlp.shared_experts.gate_proj, model.layers.45.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.46.post_attention_layernorm
        balance_layers: ['re:model\.layers\.46\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.46\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.46.mlp.shared_experts.gate_proj, model.layers.46.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: model.layers.47.post_attention_layernorm
        balance_layers: ['re:model\.layers\.47\.mlp\.experts\.[0-9]+\.gate_proj$', 're:model\.layers\.47\.mlp\.experts\.[0-9]+\.up_proj$',
          model.layers.47.mlp.shared_experts.gate_proj, model.layers.47.mlp.shared_experts.up_proj]
        activation_hook_target: null
      - smooth_layer: re:.*up_proj$
        balance_layers: ['re:.*down_proj$']
        activation_hook_target: null
      offload_device: !!python/object/apply:torch.device [cpu]
      duo_scaling: true
      n_grid: 20
      cache_chunk_size_batches: 32