feat: swap backbone from CLIP to SigLIP 2 (google/siglip2-base-patch16-224)

app.py

@@ -125,7 +125,7 @@ async def score(
         pixel_values = inputs["pixel_values"]  # (1, 3, 224, 224), CPU
 
         with torch.no_grad():
-            clip_out = model.clip(pixel_values=pixel_values)
+            clip_out = model.backbone(pixel_values=pixel_values)
             embedding = clip_out.pooler_output  # (1, 768)
             outputs = model(embedding=embedding)
 

clip_head.py

@@ -1,21 +1,21 @@
 import torch
 import torch.nn as nn
-from transformers import CLIPVisionModel
+from transformers import SiglipVisionModel
 
 
 class CreativeScorer(nn.Module):
     def __init__(self):
         super().__init__()
-        # Frozen CLIP backbone — NEVER set requires_grad=True on these params
-        self.clip = CLIPVisionModel.from_pretrained(
-            "openai/clip-vit-base-patch32",
+        # Frozen SigLIP 2 backbone — NEVER set requires_grad=True on these params
+        self.backbone = SiglipVisionModel.from_pretrained(
+            "google/siglip2-base-patch16-224",
             use_safetensors=True,
         )
-        for param in self.clip.parameters():
+        for param in self.backbone.parameters():
             param.requires_grad = False
 
         # Fail fast if backbone accidentally gets unfrozen anywhere downstream
-        assert not any(p.requires_grad for p in self.clip.parameters())
+        assert not any(p.requires_grad for p in self.backbone.parameters())
 
         # Trainable head only
         self.projection = nn.Sequential(

gradcam.py

@@ -5,14 +5,14 @@ import cv2
 import numpy as np
 import torch
 from PIL import Image
-from transformers import CLIPProcessor
+from transformers import AutoProcessor
 
 from clip_head import CreativeScorer
 
 
 def _compute_cam(
     model: CreativeScorer,
-    processor: CLIPProcessor,
+    processor: AutoProcessor,
     image: Image.Image,
     device: str,
 ) -> tuple[np.ndarray, np.ndarray]:
@@ -72,7 +72,7 @@ def _compute_cam(
 
 def generate_heatmap(
     model: CreativeScorer,
-    processor: CLIPProcessor,
+    processor: AutoProcessor,
     image: Image.Image,
     device: str = "cpu",
 ) -> np.ndarray:
@@ -83,7 +83,7 @@ def generate_heatmap(
 
 def generate_heatmap_with_cam(
     model: CreativeScorer,
-    processor: CLIPProcessor,
+    processor: AutoProcessor,
     image: Image.Image,
     device: str = "cpu",
 ) -> tuple[np.ndarray, np.ndarray]:
@@ -93,7 +93,7 @@ def generate_heatmap_with_cam(
 
 def save_heatmaps(
     model: CreativeScorer,
-    processor: CLIPProcessor,
+    processor: AutoProcessor,
     image_paths: List[str],
     output_dir: str,
     device: str = "cpu",

model_loader.py

@@ -2,15 +2,15 @@ import os
 
 import torch
 from huggingface_hub import hf_hub_download
-from transformers import CLIPProcessor
+from transformers import AutoProcessor
 
 from clip_head import CreativeScorer
 
 _model: CreativeScorer | None = None
-_processor: CLIPProcessor | None = None
+_processor: AutoProcessor | None = None
 
 
-def get_model() -> tuple[CreativeScorer, CLIPProcessor]:
+def get_model() -> tuple[CreativeScorer, AutoProcessor]:
     global _model, _processor
     if _model is None:
         try:
@@ -19,7 +19,7 @@ def get_model() -> tuple[CreativeScorer, CLIPProcessor]:
 
             print(f"[model_loader] Loading from repo: {hf_repo}", flush=True)
 
-            _processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+            _processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224")
             print("[model_loader] Processor loaded", flush=True)
 
             _model = CreativeScorer()