Keep classifier fp16 on CPU by default

app.py

@@ -806,15 +806,19 @@ def _load_tag_classifier_bundle() -> Optional[Dict[str, Any]]:
         labels_raw = json.loads(labels_path.read_text(encoding="utf-8"))
         labels = [_norm_tag_for_lookup(str(x)) for x in labels_raw]
         tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)
-        model = AutoModelForSequenceClassification.from_pretrained(model_dir, local_files_only=True)
         device_raw = (os.environ.get("PSQ_TAG_CLASSIFIER_DEVICE", "auto") or "auto").strip().lower()
         if device_raw == "auto":
             device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         else:
             device = torch.device(device_raw)
-        if device.type == "cpu":
-            # The deployed classifier may be stored as fp16 to fit Space storage.
-            # Run CPU inference in fp32 for broader operator support.
+        load_kwargs: Dict[str, Any] = {"local_files_only": True}
+        cpu_dtype = (os.environ.get("PSQ_TAG_CLASSIFIER_CPU_DTYPE", "float16") or "").strip().lower()
+        if device.type == "cpu" and cpu_dtype in {"float16", "fp16", "half"}:
+            # Keep the deployed classifier resident in fp16 on CPU by default.
+            # This reduces RAM pressure on the Space; set CPU_DTYPE=float32 if needed.
+            load_kwargs["torch_dtype"] = torch.float16
+        model = AutoModelForSequenceClassification.from_pretrained(model_dir, **load_kwargs)
+        if device.type == "cpu" and cpu_dtype in {"float32", "fp32", "full"}:
             model.float()
         model.to(device)
         model.eval()
@@ -886,16 +890,21 @@ def _run_tag_classifier(
     if not text:
         return empty
 
-    with torch.no_grad():
-        enc = tokenizer(
-            [text],
-            padding=True,
-            truncation=True,
-            max_length=max_len,
-            return_tensors="pt",
-        )
-        enc = {k: v.to(device) for k, v in enc.items()}
-        probs = torch.sigmoid(model(**enc).logits)[0].detach().cpu().tolist()
+    try:
+        with torch.no_grad():
+            enc = tokenizer(
+                [text],
+                padding=True,
+                truncation=True,
+                max_length=max_len,
+                return_tensors="pt",
+            )
+            enc = {k: v.to(device) for k, v in enc.items()}
+            probs = torch.sigmoid(model(**enc).logits)[0].detach().cpu().tolist()
+    except Exception as e:
+        if log:
+            log(f"Classifier: failed during inference; skipping ({type(e).__name__}: {_redact_console_error_text(e)})")
+        return empty
 
     ranked = sorted(
         ((labels[i], float(score)) for i, score in enumerate(probs) if i < len(labels)),