Upload model file

configuration.py

@@ -0,0 +1,44 @@
+from typing import Optional
+import logging
+from transformers import LlavaNextConfig
+
+logger = logging.getLogger(__name__)
+
+
+class Granite4VisionConfig(LlavaNextConfig):
+    model_type = "granite4_vision"
+    def __init__(
+        self,
+        downsample_rate=None,
+        use_image_newline_parameter=True,
+        deepstack_layer_map: Optional[list] = None,
+        use_spatial_sampling: bool = False,
+        spatial_stride: int = 2,
+        spatial_vision_layer: int = -1,
+        spatial_target_layers: Optional[list] = None,
+        projector_dropout=0.1,
+        **kwargs
+    ):
+        self.downsample_rate = downsample_rate
+        self.use_image_newline_parameter = use_image_newline_parameter
+        self.projector_dropout = projector_dropout
+
+        # Deepstack layer map: list of (vision_layer_idx, llm_layer_idx) tuples.
+        # Features from each vision layer are extracted, downsampled, and injected
+        # at the corresponding LLM layer during forward pass.
+        # e.g., [(-25, 12), (-17, 8), (-9, 4), (-1, 0)]
+        if deepstack_layer_map is not None:
+            self.deepstack_layer_map = [(int(v), int(l)) for v, l in deepstack_layer_map]
+            assert len(self.deepstack_layer_map) == len(set(self.deepstack_layer_map)), "expecting no duplicates"
+        else:
+            self.deepstack_layer_map = None
+
+        # Spatial sampling: extracts 4 groups from a single vision layer using
+        # spatial offset sampling (top-left, top-right, bottom-left, bottom-right
+        # of each 2x2 block), each injected at a different LLM layer.
+        self.use_spatial_sampling = use_spatial_sampling
+        self.spatial_stride = spatial_stride
+        self.spatial_vision_layer = spatial_vision_layer
+        self.spatial_target_layers = spatial_target_layers or [0, 10, 20, 30]
+
+        super().__init__(**kwargs)