feat(caption): Qwen2.5-VL-3B captioner option (Anima-style NL captions); per-call kind dispatch in Captioner facade, kind in tagger cache key, transformers>=4.53

app.py

@@ -507,7 +507,7 @@ def _build_tagger_html(result: dict, lc: str) -> str:
     return "\n".join(lines)
 
 
-def on_tag_image(image, gen_threshold, char_threshold, fmt, lang, progress=gr.Progress(), skip_pose=False, with_caption=False):
+def on_tag_image(image, gen_threshold, char_threshold, fmt, lang, progress=gr.Progress(), skip_pose=False, with_caption=False, caption_kind="florence"):
     lc = "ru" if lang == "RU" else "en"
     # fmt carries "prompt|<mode>" from the tagger-mode dropdown: "prompt|wd:eva02", "prompt|ensemble".
     if "|" in fmt:
@@ -542,6 +542,7 @@ def on_tag_image(image, gen_threshold, char_threshold, fmt, lang, progress=gr.Pr
         result = get_ensemble_tagger().tag_image(
             image, gen_threshold, char_threshold,
             mode=mode, skip_pose=skip_pose, with_caption=with_caption,
+            caption_kind=caption_kind,
         )
         progress(0.9, desc=t("tagger_results", lc))
         if not result["general"] and not result["characters"] and not result.get("nl_caption"):
@@ -1643,6 +1644,7 @@ with gr.Blocks(**_blocks_kw) as demo:
                                     (t("tagger_mode_eva_large", "en"), "wd:eva02"),
                                     (t("tagger_mode_dd", "en"), "deepdanbooru"),
                                     (t("tagger_mode_florence", "en"), "nl"),
+                                    (t("tagger_mode_qwen", "en"), "qwen"),
                                 ],
                                 value="ensemble",
                                 label="",
@@ -1866,9 +1868,12 @@ with gr.Blocks(**_blocks_kw) as demo:
         effective_fmt = f"{fmt}|{mode_key}" if mode_key != "ensemble" else fmt
         skip_pose = not bool(pose_toggle)
         # The caption checkbox is the single source of truth for the NL
-        # caption; the "nl" radio item only checks the box (see on_tag_rerun).
+        # caption; the "nl"/"qwen" radio items only check the box (see
+        # on_tag_rerun). "qwen" selects the Qwen2.5-VL-3B captioner
+        # (Anima-style descriptions), everything else uses Florence-2-base.
         with_caption = bool(caption_toggle)
-        return on_tag_image(image, gen_threshold, char_threshold, effective_fmt, lang, progress=progress, skip_pose=skip_pose, with_caption=with_caption)
+        caption_kind = "qwen" if mode == "qwen" else "florence"
+        return on_tag_image(image, gen_threshold, char_threshold, effective_fmt, lang, progress=progress, skip_pose=skip_pose, with_caption=with_caption, caption_kind=caption_kind)
 
     tagger_outputs = [
         tagger_output, tagger_apply_btn, tagger_chips, tagger_chars,
@@ -1892,15 +1897,15 @@ with gr.Blocks(**_blocks_kw) as demo:
 
     # Auto re-run when the model or the pose toggle changes (never for the
     # caption checkbox — captioning is opt-in and slow). No-op without an
-    # image. Selecting the "nl" radio item checks the caption box and runs
-    # the caption in the same event, so the checkbox stays the visible source
-    # of truth. The component outputs are deliberately NOT read back as
+    # image. Selecting the "nl"/"qwen" radio item checks the caption box and
+    # runs the caption in the same event, so the checkbox stays the visible
+    # source of truth. The component outputs are deliberately NOT read back as
     # inputs here: in Gradio 6, CheckboxGroup.preprocess validates the value
     # against the current choices and raises on stale selections, which
     # crashed re-runs ("Value: ... is not in the list of choices").
     def on_tag_rerun(image, gen_threshold, char_threshold, fmt, mode, pose_toggle, caption_toggle, lang, progress=gr.Progress()):
-        with_caption = bool(caption_toggle) or mode == "nl"
-        caption_update = gr.update(value=True) if mode == "nl" else gr.skip()
+        with_caption = bool(caption_toggle) or mode in ("nl", "qwen")
+        caption_update = gr.update(value=True) if mode in ("nl", "qwen") else gr.skip()
         if image is None:
             return (gr.skip(),) * 8 + (caption_update,)
         return tuple(on_tag_image_with_mode(image, gen_threshold, char_threshold, fmt, mode, pose_toggle, with_caption, lang, progress=progress)) + (caption_update,)

requirements.txt

@@ -6,7 +6,7 @@ numpy>=1.26.4
 torch>=2.3,<3
 torchvision>=0.18,<1
 timm>=1.0.12,<2
-transformers>=4.45,<5
+transformers>=4.53,<5
 onnxruntime>=1.16
 einops>=0.8.0
 

src/ensemble_tagger.py

@@ -296,15 +296,17 @@ class EnsembleTagger:
         # | "deepdanbooru" (legacy) | legacy singles "wd:vit"/"wd:swinv2"/"joytag"
         skip_pose: bool = False,
         with_caption: bool = False,
+        caption_kind: str = "florence",
     ) -> dict:
         """Run the ensemble tagger on one image.
 
         `skip_pose=True` suppresses the YOLO pose pass (faster on restricted VMs).
         `with_caption=True` additionally runs the *optional* lightweight VLM
-        captioner (Florence-2-base by default) and stores the result under
-        `nl_caption`. It is honored only in "ensemble" mode — single-backbone
-        and legacy modes never attach a caption. It is never enabled by
-        default because CPU VLM inference is slow on the HF Spaces free tier.
+        captioner (Florence-2-base by default, Qwen2.5-VL-3B with
+        `caption_kind="qwen"`) and stores the result under `nl_caption`. It is
+        honored only in "ensemble" mode — single-backbone and legacy modes
+        never attach a caption. It is never enabled by default because CPU VLM
+        inference is slow on the HF Spaces free tier.
         """
         if not _TAGGER_DEPS_OK and not _ORT_AVAILABLE:
             raise RuntimeError("Tagger dependencies are not available.")
@@ -317,7 +319,7 @@ class EnsembleTagger:
             raise ValueError(f"Invalid image input for tagging: {exc}") from exc
 
         # Cache lookups are per-mode so changing the model doesn't reuse stale results.
-        key = f"{mode}:{skip_pose}:{with_caption}:{self._image_key(pil_img)}"
+        key = f"{mode}:{skip_pose}:{with_caption}:{caption_kind}:{self._image_key(pil_img)}"
         cached = self._cache.get(key)
         if cached is not None:
             return cached
@@ -429,7 +431,7 @@ class EnsembleTagger:
         if with_caption:
             try:
                 from src.vlm_caption import get_captioner
-                result["nl_caption"] = get_captioner().caption(pil_img) or ""
+                result["nl_caption"] = get_captioner().caption(pil_img, kind=caption_kind) or ""
             except Exception:  # pragma: no cover
                 result["nl_caption"] = ""
 

src/i18n.py

@@ -329,8 +329,9 @@ L10N = {
         "tagger_mode_eva_large": "WD14 EVA Large Mod",
         "tagger_mode_dd": "DeepDanbooru (legacy)",
         "tagger_mode_florence": "Florence NL Caption (Natural Language Mod)",
+        "tagger_mode_qwen": "Qwen NL Caption (Qwen2.5-VL-3B)",
         "tagger_caption_toggle_label": "Add NL caption",
-        "tagger_caption_toggle_info": "Adds a natural-language description (Florence-2-base). Only applies with the Smart tagger; slow on CPU — off by default.",
+        "tagger_caption_toggle_info": "Adds a natural-language description (Florence-2-base by default, Qwen2.5-VL-3B for Anima-style captions). Only applies with the Smart tagger; slow on CPU — off by default.",
         "tagger_pose_label": "Pose",
         "tagger_pose_single": "{n} person detected",
         "tagger_pose_many": "{n} people detected",
@@ -699,8 +700,9 @@ L10N = {
         "tagger_mode_eva_large": "WD14 EVA Large Mod",
         "tagger_mode_dd": "DeepDanbooru (legacy)",
         "tagger_mode_florence": "Florence NL-описание (языковая модель)",
+        "tagger_mode_qwen": "Qwen NL-описание (Qwen2.5-VL-3B)",
         "tagger_caption_toggle_label": "Добавить NL-описание",
-        "tagger_caption_toggle_info": "Добавляет описание на естественном языке (Florence-2-base). Работает только с умным теггером; на CPU медленно — выключено по умолчанию.",
+        "tagger_caption_toggle_info": "Добавляет описание на естественном языке (Florence-2-base по умолчанию, Qwen2.5-VL-3B для капшнов в стиле Anima). Работает только с умным теггером; на CPU медленно — выключено по умолчанию.",
         "tagger_nl_caption": "🗣️ Описание сцены",
         "tagger_pose_label": "Поза",
         "tagger_pose_single": "Обнаружен {n} человек",

src/vlm_caption.py

@@ -6,10 +6,13 @@ opt-in only and uses a much smaller model by default:
 
 - ``WHYX_CAPTION_MODEL=florence`` (default)  -> microsoft/Florence-2-base
 - ``WHYX_CAPTION_MODEL=smolvlm``             -> HuggingFaceTB/SmolVLM-256M-Instruct
+- ``WHYX_CAPTION_MODEL=qwen``                -> Qwen/Qwen2.5-VL-3B-Instruct (slow on CPU)
 - ``WHYX_CAPTION_MODEL=moondream``           -> vikhyatk/moondream2 (legacy)
 
 The captioner loads lazily on first use and never runs inside the default
-ensemble path. ``WHYX_DISABLE_CAPTION=1`` turns it off entirely.
+ensemble path. ``WHYX_DISABLE_CAPTION=1`` turns it off entirely. The active
+impl can be chosen per call via ``Captioner.caption(pil, kind=...)``; only the
+most recent impl stays resident so two VLMs don't double the memory footprint.
 """
 
 from __future__ import annotations
@@ -29,6 +32,20 @@ _CAPTION_DEFAULT = os.environ.get("WHYX_CAPTION_MODEL", "florence").strip().lowe
 # "new version of the code file" download warning.
 _FLORENCE_REVISION = "5ca5edf5bd017b9919c05d08aebef5e4c7ac3bac"
 
+# Qwen2.5-VL-3B captioner. Qwen-style dense descriptions match video models
+# whose text encoder was trained on Qwen embeddings (e.g. Anima). Needs
+# transformers>=4.53 (AutoModelForImageTextToText mapping); bf16 on CPU to fit
+# the 16 GB free tier alongside the WD14 ensemble.
+_QWEN_MODEL_ID = os.environ.get("WHYX_QWEN_MODEL", "Qwen/Qwen2.5-VL-3B-Instruct")
+_QWEN_PROMPT = os.environ.get(
+    "WHYX_QWEN_PROMPT",
+    "Describe this anime illustration in one dense paragraph as a caption for a "
+    "video model: character appearance, pose, action, clothing, lighting, "
+    "background and mood.",
+).strip()
+_QWEN_MAX_TOKENS = int(os.environ.get("WHYX_QWEN_MAX_TOKENS", "256"))
+_QWEN_MAX_PIXELS = int(os.environ.get("WHYX_QWEN_MAX_PIXELS", "4000000"))
+
 
 def _caption_enabled() -> bool:
     return os.environ.get("WHYX_DISABLE_CAPTION", "0").strip().lower() not in ("1", "true", "yes", "on")
@@ -148,32 +165,109 @@ class _MoondreamCaptioner:
         return self._impl.caption(pil, length="short")
 
 
+class _QwenCaptioner:
+    """Qwen2.5-VL-3B captioning via the chat template.
+
+    Produces dense Qwen-style descriptions aimed at video models with a
+    Qwen-based text encoder (e.g. Anima). Heavy (~6.1 GB bf16) and slow on
+    CPU — opt-in only.
+    """
+
+    def __init__(self, model_id: str = _QWEN_MODEL_ID):
+        self._model_id = model_id
+        self._processor = None
+        self._model = None
+
+    def _load(self) -> bool:
+        try:
+            from transformers import AutoProcessor
+        except Exception:
+            return False
+        try:
+            from transformers import AutoModelForImageTextToText as _M
+        except Exception:
+            try:
+                from transformers import AutoModelForVision2Seq as _M
+            except Exception:
+                return False
+        try:
+            import torch
+            self._processor = AutoProcessor.from_pretrained(self._model_id)
+            self._model = _M.from_pretrained(
+                self._model_id, torch_dtype=torch.bfloat16
+            ).to("cuda" if torch.cuda.is_available() else "cpu")
+        except Exception as exc:  # pragma: no cover
+            logger.warning("Qwen captioner failed to load: %s", exc)
+            return False
+        return True
+
+    def caption(self, pil) -> str:
+        if self._processor is None or self._model is None:
+            if not self._load():
+                return ""
+        import torch
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image", "image": pil},
+                    {"type": "text", "text": _QWEN_PROMPT},
+                ],
+            }
+        ]
+        text = self._processor.apply_chat_template(messages, add_generation_prompt=True)
+        inputs = self._processor(
+            text=[text], images=[pil], return_tensors="pt", max_pixels=_QWEN_MAX_PIXELS
+        ).to(self._model.device)
+        with torch.inference_mode():
+            gen = self._model.generate(
+                **inputs, max_new_tokens=_QWEN_MAX_TOKENS, do_sample=False
+            )
+        return self._processor.batch_decode(
+            gen[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True
+        )[0].strip()
+
+
 class Captioner:
     """Facade over the selected lightweight caption model."""
 
     def __init__(self, kind: str = _CAPTION_DEFAULT):
         self._kind = kind
-        self._impl: Optional[object] = None
+        self._impls: dict[str, object] = {}
+        self._active: Optional[str] = None
         self._lock = threading.Lock()
 
-    def caption(self, pil) -> str:
+    def caption(self, pil, kind: Optional[str] = None) -> str:
         if not _caption_enabled():
             return ""
+        kind = (kind or self._kind or "").strip().lower()
         with self._lock:
-            if self._impl is None:
-                kind = self._kind
-                if kind == "smolvlm":
-                    self._impl = _SmolVLMCaptioner()
-                elif kind == "moondream":
-                    self._impl = _MoondreamCaptioner()
-                else:
-                    self._impl = _FlorenceCaptioner()
+            if kind != self._active:
+                impl = self._impls.get(kind)
+                if impl is None:
+                    impl = self._make_impl(kind)
+                    # Keep only the most recent captioner resident: Qwen
+                    # (~6.1 GB) plus Florence alongside the WD14 ensemble
+                    # would strain the 16 GB free tier.
+                    self._impls.clear()
+                    self._impls[kind] = impl
+                self._active = kind
             try:
-                return self._impl.caption(pil) or ""
+                return self._impls[kind].caption(pil) or ""
             except Exception as exc:  # pragma: no cover
                 logger.warning("Captioner failed: %s", exc)
                 return ""
 
+    @staticmethod
+    def _make_impl(kind: str):
+        if kind == "smolvlm":
+            return _SmolVLMCaptioner()
+        if kind == "moondream":
+            return _MoondreamCaptioner()
+        if kind == "qwen":
+            return _QwenCaptioner()
+        return _FlorenceCaptioner()
+
 
 _captioner_instance: Captioner | None = None
 

tests/test_captioner.py

@@ -0,0 +1,79 @@
+"""Facade dispatch for the tagger captioner (no model downloads)."""
+import pytest
+
+from src.vlm_caption import (
+    _FlorenceCaptioner,
+    _MoondreamCaptioner,
+    _QwenCaptioner,
+    _SmolVLMCaptioner,
+    Captioner,
+)
+
+
+class _Fake:
+    def __init__(self, kind):
+        self.kind = kind
+        self.calls = 0
+
+    def caption(self, pil):
+        self.calls += 1
+        return f"caption from {self.kind}"
+
+
+@pytest.fixture
+def fake_impls(monkeypatch):
+    made = {}
+
+    def make(kind):
+        impl = _Fake(kind)
+        made[kind] = impl
+        return impl
+
+    monkeypatch.setattr(Captioner, "_make_impl", staticmethod(make))
+    return made
+
+
+def test_default_kind_is_florence(fake_impls):
+    c = Captioner(kind="florence")
+    assert c.caption(None) == "caption from florence"
+    assert "florence" in fake_impls
+
+
+def test_impl_reused_across_calls(fake_impls):
+    c = Captioner(kind="florence")
+    assert c.caption(None) == "caption from florence"
+    assert c.caption(None) == "caption from florence"
+    assert fake_impls["florence"].calls == 2
+    assert len(fake_impls) == 1
+
+
+def test_kind_param_overrides_default(fake_impls):
+    c = Captioner(kind="florence")
+    assert c.caption(None, kind="qwen") == "caption from qwen"
+    assert "qwen" in fake_impls
+
+
+def test_switching_kind_keeps_single_resident_impl(fake_impls):
+    c = Captioner(kind="florence")
+    assert c.caption(None) == "caption from florence"
+    assert c.caption(None, kind="qwen") == "caption from qwen"
+    assert c.caption(None) == "caption from florence"
+    assert len(fake_impls) == 2
+    assert list(c._impls) == ["florence"]
+    assert c._active == "florence"
+    assert fake_impls["qwen"].calls == 1
+
+
+def test_make_impl_kinds():
+    assert isinstance(Captioner._make_impl("florence"), _FlorenceCaptioner)
+    assert isinstance(Captioner._make_impl("smolvlm"), _SmolVLMCaptioner)
+    assert isinstance(Captioner._make_impl("moondream"), _MoondreamCaptioner)
+    assert isinstance(Captioner._make_impl("qwen"), _QwenCaptioner)
+    assert isinstance(Captioner._make_impl("unknown"), _FlorenceCaptioner)
+
+
+def test_disabled_returns_empty(fake_impls, monkeypatch):
+    monkeypatch.setenv("WHYX_DISABLE_CAPTION", "1")
+    c = Captioner(kind="florence")
+    assert c.caption(None) == ""
+    assert fake_impls == {}