granite-switch-4.0-350m 3-adapter aLoRA demo: cti-technique-mapping + text-to-json + genai-attack-vector

BUILD.md

@@ -16,17 +16,17 @@ Total adapters: **3**
 
 | # | Name | Technology | Control Token | Token ID | Rank | Alpha | Target Modules | Source |
 |---|------|------------|---------------|----------|------|-------|----------------|--------|
-| 1 | cti-technique-mapping | lora | `<\|cti-technique-mapping\|>` | 100352 | 16 | 32 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | cti-technique-mapping |
-| 2 | text-to-json | lora | `<\|text-to-json\|>` | 100353 | 16 | 32 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | text-to-json |
-| 3 | genai-attack-vector | lora | `<\|genai-attack-vector\|>` | 100354 | 16 | 32 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | genai-attack-vector |
+| 1 | cti-technique-mapping | alora | `<\|cti-technique-mapping\|>` | 100352 | 32 | 64 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | cti-technique-mapping |
+| 2 | text-to-json | alora | `<\|text-to-json\|>` | 100353 | 32 | 64 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | text-to-json |
+| 3 | genai-attack-vector | alora | `<\|genai-attack-vector\|>` | 100354 | 32 | 64 | input_linear, k_proj, o_proj, output_linear, q_proj, v_proj | genai-attack-vector |
 
 ## Composition Details
 
 - base_param_count: 352,379,904
-- composed_param_count: 373,714,944
-- Param delta: +6.05%
+- composed_param_count: 370,274,304
+- Param delta: +5.08%
 - compose_settings:
-  - technology: "lora"
+  - technology: "alora"
   - adapter_substitute_token_ids:
     - 100264
     - 100264

adapter_index.json

@@ -7,7 +7,7 @@
     {
       "adapter_index": 1,
       "adapter_name": "cti-technique-mapping",
-      "technology": "lora",
+      "technology": "alora",
       "control_token": {
         "token": "<|cti-technique-mapping|>",
         "id": 100352
@@ -17,7 +17,7 @@
     {
       "adapter_index": 2,
       "adapter_name": "text-to-json",
-      "technology": "lora",
+      "technology": "alora",
       "control_token": {
         "token": "<|text-to-json|>",
         "id": 100353
@@ -27,7 +27,7 @@
     {
       "adapter_index": 3,
       "adapter_name": "genai-attack-vector",
-      "technology": "lora",
+      "technology": "alora",
       "control_token": {
         "token": "<|genai-attack-vector|>",
         "id": 100354

chat_template.jinja

@@ -1,7 +1,7 @@
 {%- set adapter_map = {
-    'cti-technique-mapping': {'token': '<|cti-technique-mapping|>', 'type': 'lora'},
-    'text-to-json': {'token': '<|text-to-json|>', 'type': 'lora'},
-    'genai-attack-vector': {'token': '<|genai-attack-vector|>', 'type': 'lora'}
+    'cti-technique-mapping': {'token': '<|cti-technique-mapping|>', 'type': 'alora', 'invocation_text': '<|start_of_role|>assistant<|end_of_role|>'},
+    'text-to-json': {'token': '<|text-to-json|>', 'type': 'alora', 'invocation_text': '<|start_of_role|>assistant<|end_of_role|>'},
+    'genai-attack-vector': {'token': '<|genai-attack-vector|>', 'type': 'alora', 'invocation_text': '<|start_of_role|>assistant<|end_of_role|>'}
 } %}
 {#- Look up adapter token, type, and invocation text from adapter_name -#}
 {%- set adapter_token = '' %}

compose_report.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "timestamp": "2026-06-15T11:17:40.360098"
+    "timestamp": "2026-06-16T06:00:21.167677"
   },
   "base_model_mapping": [
     {
@@ -28,14 +28,14 @@
       "source": [
         "model.layers.0.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.0.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.0.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.0.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.0.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.0.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -63,14 +63,14 @@
       "source": [
         "model.layers.1.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.1.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.1.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.1.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.1.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.1.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -98,14 +98,14 @@
       "source": [
         "model.layers.2.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.2.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.2.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.2.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.2.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.2.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -133,14 +133,14 @@
       "source": [
         "model.layers.3.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.3.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.3.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.3.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.3.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.3.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -168,14 +168,14 @@
       "source": [
         "model.layers.4.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.4.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.4.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.4.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.4.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.4.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -203,14 +203,14 @@
       "source": [
         "model.layers.5.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.5.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.5.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.5.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.5.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.5.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -238,14 +238,14 @@
       "source": [
         "model.layers.6.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.6.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.6.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.6.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.6.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.6.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -273,14 +273,14 @@
       "source": [
         "model.layers.7.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.7.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.7.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.7.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.7.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.7.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -308,14 +308,14 @@
       "source": [
         "model.layers.8.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.8.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.8.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.8.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.8.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.8.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -343,14 +343,14 @@
       "source": [
         "model.layers.9.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.9.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.9.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.9.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.9.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.9.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -378,14 +378,14 @@
       "source": [
         "model.layers.10.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.10.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.10.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.10.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.10.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.10.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -413,14 +413,14 @@
       "source": [
         "model.layers.11.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.11.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.11.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.11.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.11.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.11.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -448,14 +448,14 @@
       "source": [
         "model.layers.12.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.12.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.12.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.12.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.12.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.12.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -483,14 +483,14 @@
       "source": [
         "model.layers.13.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.13.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.13.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.13.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.13.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.13.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -518,14 +518,14 @@
       "source": [
         "model.layers.14.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.14.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.14.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.14.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.14.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.14.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -553,14 +553,14 @@
       "source": [
         "model.layers.15.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.15.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.15.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.15.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.15.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.15.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -588,14 +588,14 @@
       "source": [
         "model.layers.16.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.16.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.16.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.16.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.16.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.16.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -623,14 +623,14 @@
       "source": [
         "model.layers.17.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.17.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.17.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.17.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.17.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.17.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -658,14 +658,14 @@
       "source": [
         "model.layers.18.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.18.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.18.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.18.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.18.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.18.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -693,14 +693,14 @@
       "source": [
         "model.layers.19.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.19.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.19.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.19.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.19.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.19.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -728,14 +728,14 @@
       "source": [
         "model.layers.20.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.20.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.20.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.20.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.20.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.20.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -763,14 +763,14 @@
       "source": [
         "model.layers.21.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.21.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.21.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.21.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.21.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.21.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -798,14 +798,14 @@
       "source": [
         "model.layers.22.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.22.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.22.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.22.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.22.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.22.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -833,14 +833,14 @@
       "source": [
         "model.layers.23.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.23.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.23.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.23.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.23.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.23.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -868,14 +868,14 @@
       "source": [
         "model.layers.24.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.24.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.24.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.24.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.24.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.24.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -903,14 +903,14 @@
       "source": [
         "model.layers.25.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.25.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.25.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.25.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.25.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.25.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -938,14 +938,14 @@
       "source": [
         "model.layers.26.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.26.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.26.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.26.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.26.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.26.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -973,14 +973,14 @@
       "source": [
         "model.layers.27.shared_mlp.input_linear.weight"
       ],
-      "target": "model.layers.27.shared_mlp.input_linear.base_layer.weight",
+      "target": "model.layers.27.shared_mlp.input_linear.weight",
       "type": "direct"
     },
     {
       "source": [
         "model.layers.27.shared_mlp.output_linear.weight"
       ],
-      "target": "model.layers.27.shared_mlp.output_linear.base_layer.weight",
+      "target": "model.layers.27.shared_mlp.output_linear.weight",
       "type": "direct"
     },
     {
@@ -1330,48 +1330,6 @@
       "target": "model.layers.0.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.0.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.0.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.1.self_attn.o_proj.lora_A.weight",
@@ -1444,48 +1402,6 @@
       "target": "model.layers.1.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.1.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.1.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.10.self_attn.o_proj.lora_A.weight",
@@ -1558,48 +1474,6 @@
       "target": "model.layers.10.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.10.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.10.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.11.self_attn.o_proj.lora_A.weight",
@@ -1672,48 +1546,6 @@
       "target": "model.layers.11.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.11.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.11.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.12.self_attn.o_proj.lora_A.weight",
@@ -1786,48 +1618,6 @@
       "target": "model.layers.12.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.12.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.12.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.13.self_attn.o_proj.lora_A.weight",
@@ -1900,48 +1690,6 @@
       "target": "model.layers.13.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.13.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.13.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.14.self_attn.o_proj.lora_A.weight",
@@ -2014,48 +1762,6 @@
       "target": "model.layers.14.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.14.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.14.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.15.self_attn.o_proj.lora_A.weight",
@@ -2128,48 +1834,6 @@
       "target": "model.layers.15.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.15.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.15.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.16.self_attn.o_proj.lora_A.weight",
@@ -2244,80 +1908,38 @@
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.input_linear.lora_A.weight"
+        "adapter_0::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight",
+        "adapter_1::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight",
+        "adapter_2::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight"
       ],
-      "target": "model.layers.16.shared_mlp.input_linear.lora_A_slices.0",
+      "target": "model.layers.17.self_attn.o_proj.lora_A",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.input_linear.lora_A.weight"
+        "adapter_0::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight",
+        "adapter_1::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight",
+        "adapter_2::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight"
       ],
-      "target": "model.layers.16.shared_mlp.input_linear.lora_A_slices.1",
+      "target": "model.layers.17.self_attn.o_proj.lora_B",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.input_linear.lora_B.weight"
+        "adapter_0::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight",
+        "adapter_1::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight",
+        "adapter_2::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight"
       ],
-      "target": "model.layers.16.shared_mlp.input_linear.lora_B_slices.0",
+      "target": "model.layers.17.self_attn.qkv_proj.lora_A_slices.0",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.input_linear.lora_B.weight"
+        "adapter_0::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight",
+        "adapter_1::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight",
+        "adapter_2::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight"
       ],
-      "target": "model.layers.16.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.16.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.16.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.16.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight",
-        "adapter_1::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight",
-        "adapter_2::base_model.model.model.layers.17.self_attn.o_proj.lora_A.weight"
-      ],
-      "target": "model.layers.17.self_attn.o_proj.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight",
-        "adapter_1::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight",
-        "adapter_2::base_model.model.model.layers.17.self_attn.o_proj.lora_B.weight"
-      ],
-      "target": "model.layers.17.self_attn.o_proj.lora_B",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight",
-        "adapter_1::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight",
-        "adapter_2::base_model.model.model.layers.17.self_attn.q_proj.lora_A.weight"
-      ],
-      "target": "model.layers.17.self_attn.qkv_proj.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight",
-        "adapter_1::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight",
-        "adapter_2::base_model.model.model.layers.17.self_attn.k_proj.lora_A.weight"
-      ],
-      "target": "model.layers.17.self_attn.qkv_proj.lora_A_slices.1",
+      "target": "model.layers.17.self_attn.qkv_proj.lora_A_slices.1",
       "type": "adapter_stacked"
     },
     {
@@ -2356,48 +1978,6 @@
       "target": "model.layers.17.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.17.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.17.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.18.self_attn.o_proj.lora_A.weight",
@@ -2470,48 +2050,6 @@
       "target": "model.layers.18.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.18.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.18.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.19.self_attn.o_proj.lora_A.weight",
@@ -2584,48 +2122,6 @@
       "target": "model.layers.19.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.19.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.19.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.2.self_attn.o_proj.lora_A.weight",
@@ -2698,48 +2194,6 @@
       "target": "model.layers.2.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.2.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.2.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.20.self_attn.o_proj.lora_A.weight",
@@ -2812,48 +2266,6 @@
       "target": "model.layers.20.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.20.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.20.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.21.self_attn.o_proj.lora_A.weight",
@@ -2926,48 +2338,6 @@
       "target": "model.layers.21.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.21.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.21.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.22.self_attn.o_proj.lora_A.weight",
@@ -3040,48 +2410,6 @@
       "target": "model.layers.22.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.22.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.22.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.23.self_attn.o_proj.lora_A.weight",
@@ -3154,48 +2482,6 @@
       "target": "model.layers.23.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.23.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.23.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.24.self_attn.o_proj.lora_A.weight",
@@ -3268,48 +2554,6 @@
       "target": "model.layers.24.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.24.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.24.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.25.self_attn.o_proj.lora_A.weight",
@@ -3382,48 +2626,6 @@
       "target": "model.layers.25.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.25.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.25.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.26.self_attn.o_proj.lora_A.weight",
@@ -3462,80 +2664,38 @@
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight",
-        "adapter_1::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight",
-        "adapter_2::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight"
-      ],
-      "target": "model.layers.26.self_attn.qkv_proj.lora_A_slices.2",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight",
-        "adapter_1::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight",
-        "adapter_2::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight"
-      ],
-      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight",
-        "adapter_1::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight",
-        "adapter_2::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight"
-      ],
-      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight",
-        "adapter_1::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight",
-        "adapter_2::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight"
-      ],
-      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.2",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.26.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.26.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.input_linear.lora_B.weight"
+        "adapter_0::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight",
+        "adapter_1::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight",
+        "adapter_2::base_model.model.model.layers.26.self_attn.v_proj.lora_A.weight"
       ],
-      "target": "model.layers.26.shared_mlp.input_linear.lora_B_slices.0",
+      "target": "model.layers.26.self_attn.qkv_proj.lora_A_slices.2",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.input_linear.lora_B.weight"
+        "adapter_0::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight",
+        "adapter_1::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight",
+        "adapter_2::base_model.model.model.layers.26.self_attn.q_proj.lora_B.weight"
       ],
-      "target": "model.layers.26.shared_mlp.input_linear.lora_B_slices.1",
+      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.0",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.output_linear.lora_A.weight"
+        "adapter_0::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight",
+        "adapter_1::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight",
+        "adapter_2::base_model.model.model.layers.26.self_attn.k_proj.lora_B.weight"
       ],
-      "target": "model.layers.26.shared_mlp.output_linear.lora_A",
+      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.1",
       "type": "adapter_stacked"
     },
     {
       "source": [
-        "adapter_0::base_model.model.model.layers.26.shared_mlp.output_linear.lora_B.weight"
+        "adapter_0::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight",
+        "adapter_1::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight",
+        "adapter_2::base_model.model.model.layers.26.self_attn.v_proj.lora_B.weight"
       ],
-      "target": "model.layers.26.shared_mlp.output_linear.lora_B",
+      "target": "model.layers.26.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
     {
@@ -3610,48 +2770,6 @@
       "target": "model.layers.27.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.27.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.27.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.3.self_attn.o_proj.lora_A.weight",
@@ -3724,48 +2842,6 @@
       "target": "model.layers.3.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.3.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.3.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.4.self_attn.o_proj.lora_A.weight",
@@ -3838,48 +2914,6 @@
       "target": "model.layers.4.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.4.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.4.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.5.self_attn.o_proj.lora_A.weight",
@@ -3952,48 +2986,6 @@
       "target": "model.layers.5.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.5.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.5.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.6.self_attn.o_proj.lora_A.weight",
@@ -4066,48 +3058,6 @@
       "target": "model.layers.6.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.6.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.6.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.7.self_attn.o_proj.lora_A.weight",
@@ -4180,48 +3130,6 @@
       "target": "model.layers.7.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.7.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.7.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.8.self_attn.o_proj.lora_A.weight",
@@ -4294,48 +3202,6 @@
       "target": "model.layers.8.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
     },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.8.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.8.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
-    },
     {
       "source": [
         "adapter_0::base_model.model.model.layers.9.self_attn.o_proj.lora_A.weight",
@@ -4407,48 +3273,6 @@
       ],
       "target": "model.layers.9.self_attn.qkv_proj.lora_B_slices.2",
       "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.input_linear.lora_A_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.input_linear.lora_A.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.input_linear.lora_A_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.input_linear.lora_B_slices.0",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.input_linear.lora_B.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.input_linear.lora_B_slices.1",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.output_linear.lora_A.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.output_linear.lora_A",
-      "type": "adapter_stacked"
-    },
-    {
-      "source": [
-        "adapter_0::base_model.model.model.layers.9.shared_mlp.output_linear.lora_B.weight"
-      ],
-      "target": "model.layers.9.shared_mlp.output_linear.lora_B",
-      "type": "adapter_stacked"
     }
   ]
 }

config.json

@@ -5,9 +5,9 @@
     "genai-attack-vector"
   ],
   "adapter_ranks": [
-    16,
-    16,
-    16
+    32,
+    32,
+    32
   ],
   "adapter_substitute_token_ids": [
     100264,
@@ -69,9 +69,7 @@
   "logits_scaling": 4,
   "lora_target_modules": [
     "o_proj",
-    "qkv_proj",
-    "shared_input_linear",
-    "shared_output_linear"
+    "qkv_proj"
   ],
   "mamba_chunk_size": 256,
   "mamba_conv_bias": true,
@@ -82,7 +80,7 @@
   "mamba_n_groups": 1,
   "mamba_n_heads": 128,
   "mamba_proj_bias": false,
-  "max_lora_rank": 16,
+  "max_lora_rank": 32,
   "max_position_embeddings": 32768,
   "model_type": "granite_switch",
   "num_adapters": 3,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3018762a225cdb75cbc184ef19b7dc143e2dcd7d4c7a1b704188fb4b3d1a0c75
-size 748304360
+oid sha256:4592ded273debc664515c23822627b3b7357299410ed4bda6cffb76724ea13ef
+size 741400464