Merge remote-tracking branch 'origin/main'

app.py

@@ -147,8 +147,8 @@ _CORPORATE_HARDBLOCK_PATTERNS = [
     # Common kink/fetish markers.
     re.compile(r"(^|_)(fetish|bdsm|bondage|dominatrix|submission|vore|inflation|watersports)(_|$)", re.IGNORECASE),
 ]
-
-
+
+
 def _split_prompt_commas(s: str) -> List[str]:
     return [p.strip() for p in (s or "").split(",") if p.strip()]
 
@@ -213,23 +213,23 @@ def _rewrite_prompt(prompt_in: str, log) -> str:
         log("Rewrite source=t5 returned empty; fallback to llm")
         return llm_rewrite_prompt(prompt_in, log)
     return ""
-
-def compose_final_prompt(rewritten_prompt: str, selected_tags: List[str]) -> str:
-    parts = _split_prompt_commas(rewritten_prompt)
-    parts.extend(selected_tags)
-
-    seen = set()
-    out = []
-    for p in parts:
-        key = _norm_for_dedupe(p)
-        if key in seen:
-            continue
-        seen.add(key)
-        out.append(p)
-
-    return ", ".join(out)
-
-
+
+def compose_final_prompt(rewritten_prompt: str, selected_tags: List[str]) -> str:
+    parts = _split_prompt_commas(rewritten_prompt)
+    parts.extend(selected_tags)
+
+    seen = set()
+    out = []
+    for p in parts:
+        key = _norm_for_dedupe(p)
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(p)
+
+    return ", ".join(out)
+
+
 def _display_tag_text(tag: str) -> str:
     return tag.replace("_", " ")
 
@@ -569,13 +569,13 @@ def _order_selected_tags_for_row(
 
 
 def _escape_prompt_tag(tag: str) -> str:
-    return (
-        tag.replace("_", " ")
-        .replace("(", "\\(")
-        .replace(")", "\\)")
-    )
-
-
+    return (
+        tag.replace("_", " ")
+        .replace("(", "\\(")
+        .replace(")", "\\)")
+    )
+
+
 def _ordered_selected_for_prompt(selected: Set[str], row_defs: List[Dict[str, Any]]) -> List[str]:
     out: List[str] = []
     seen: Set[str] = set()
@@ -585,22 +585,22 @@ def _ordered_selected_for_prompt(selected: Set[str], row_defs: List[Dict[str, An
                 out.append(tag)
                 seen.add(tag)
     return out
-
-
-def _compose_toggle_prompt_text(selected_tags: List[str], row_defs: List[Dict[str, Any]]) -> str:
-    selected = {t for t in (selected_tags or []) if t}
-    ordered = _ordered_selected_for_prompt(selected, row_defs or [])
-    return ", ".join(_escape_prompt_tag(t) for t in ordered)
-
-
-def _is_artist_tag(tag: str) -> bool:
-    t = _norm_tag_for_lookup(str(tag))
-    if not t:
-        return False
-    # Keep a resilient fallback for malformed/missing tag typing metadata.
-    return get_tag_type_name(t) == "artist" or t.startswith("by_")
-
-
+
+
+def _compose_toggle_prompt_text(selected_tags: List[str], row_defs: List[Dict[str, Any]]) -> str:
+    selected = {t for t in (selected_tags or []) if t}
+    ordered = _ordered_selected_for_prompt(selected, row_defs or [])
+    return ", ".join(_escape_prompt_tag(t) for t in ordered)
+
+
+def _is_artist_tag(tag: str) -> bool:
+    t = _norm_tag_for_lookup(str(tag))
+    if not t:
+        return False
+    # Keep a resilient fallback for malformed/missing tag typing metadata.
+    return get_tag_type_name(t) == "artist" or t.startswith("by_")
+
+
 @lru_cache(maxsize=1)
 def _load_excluded_recommendation_tags() -> Set[str]:
     out: Set[str] = set()
@@ -686,27 +686,27 @@ def _filter_min_count_tags(tags: List[str], min_count: int) -> Tuple[List[str],
         seen.add(t)
         keep.append(t)
     return keep, sorted(set(removed))
-
-
+
+
 def _filter_excluded_recommendation_tags(tags: List[str]) -> Tuple[List[str], List[str]]:
-    excluded = _load_excluded_recommendation_tags()
-    if not excluded:
-        return list(dict.fromkeys(_norm_tag_for_lookup(t) for t in (tags or []) if t)), []
-
-    keep: List[str] = []
-    removed: List[str] = []
-    seen: Set[str] = set()
-    for raw in (tags or []):
-        t = _norm_tag_for_lookup(str(raw))
-        if not t:
-            continue
-        if t in excluded:
-            removed.append(t)
-            continue
-        if t in seen:
-            continue
-        seen.add(t)
-        keep.append(t)
+    excluded = _load_excluded_recommendation_tags()
+    if not excluded:
+        return list(dict.fromkeys(_norm_tag_for_lookup(t) for t in (tags or []) if t)), []
+
+    keep: List[str] = []
+    removed: List[str] = []
+    seen: Set[str] = set()
+    for raw in (tags or []):
+        t = _norm_tag_for_lookup(str(raw))
+        if not t:
+            continue
+        if t in excluded:
+            removed.append(t)
+            continue
+        if t in seen:
+            continue
+        seen.add(t)
+        keep.append(t)
     return keep, sorted(set(removed))
 
 
@@ -1044,12 +1044,12 @@ def _build_toggle_rows(
     top_tags_per_group: int,
     group_rank_top_k: int,
 ) -> List[Dict[str, Any]]:
-    ranked_rows = rank_groups_from_tfidf(
-        seed_terms=seed_terms,
-        top_groups=max(1, int(top_groups)),
-        top_tags_per_group=max(1, int(top_tags_per_group)),
-        group_rank_top_k=max(1, int(group_rank_top_k)),
-    )
+    ranked_rows = rank_groups_from_tfidf(
+        seed_terms=seed_terms,
+        top_groups=max(1, int(top_groups)),
+        top_tags_per_group=max(1, int(top_tags_per_group)),
+        group_rank_top_k=max(1, int(group_rank_top_k)),
+    )
     groups_map = _load_enabled_groups()
     selected_active = list(
         dict.fromkeys(
@@ -1085,7 +1085,7 @@ def _build_toggle_rows(
     tags_in_any_displayed_group: Set[str] = set()
     for tag_set in displayed_group_tag_sets.values():
         tags_in_any_displayed_group.update(tag_set)
-
+
     retrieved_uncategorized_ranked = list(
         dict.fromkeys(
             _norm_tag_for_lookup(t)
@@ -1198,36 +1198,36 @@ def _build_toggle_rows(
         row_defs.append(retrieved_other_row)
 
     return row_defs
-
-
+
+
 def _build_display_audit_line(
-    row_defs: List[Dict[str, Any]],
-    *,
-    active_selected_tags: List[str],
-    direct_selected_tags: List[str],
-    implied_selected_tags: List[str],
-) -> str:
-    active_set = {
-        _norm_tag_for_lookup(t)
-        for t in (active_selected_tags or [])
-        if t and not _is_artist_tag(t)
-    }
-    direct_set = {
-        _norm_tag_for_lookup(t)
-        for t in (direct_selected_tags or [])
-        if t and not _is_artist_tag(t)
-    }
-    implied_set = {
-        _norm_tag_for_lookup(t)
-        for t in (implied_selected_tags or [])
-        if t and not _is_artist_tag(t)
-    }
-    info_by_tag: Dict[str, Dict[str, Any]] = {}
-
-    for row in row_defs or []:
-        row_name = row.get("name", "")
-        row_label = row.get("label", row_name)
-        for tag in row.get("tags", []):
+    row_defs: List[Dict[str, Any]],
+    *,
+    active_selected_tags: List[str],
+    direct_selected_tags: List[str],
+    implied_selected_tags: List[str],
+) -> str:
+    active_set = {
+        _norm_tag_for_lookup(t)
+        for t in (active_selected_tags or [])
+        if t and not _is_artist_tag(t)
+    }
+    direct_set = {
+        _norm_tag_for_lookup(t)
+        for t in (direct_selected_tags or [])
+        if t and not _is_artist_tag(t)
+    }
+    implied_set = {
+        _norm_tag_for_lookup(t)
+        for t in (implied_selected_tags or [])
+        if t and not _is_artist_tag(t)
+    }
+    info_by_tag: Dict[str, Dict[str, Any]] = {}
+
+    for row in row_defs or []:
+        row_name = row.get("name", "")
+        row_label = row.get("label", row_name)
+        for tag in row.get("tags", []):
             rec = info_by_tag.setdefault(tag, {"rows": [], "sources": set()})
             rec["rows"].append(row_label)
             if row_name == "selected_other":
@@ -1236,23 +1236,23 @@ def _build_display_audit_line(
                 rec["sources"].add("other_retrieved_row")
             else:
                 rec["sources"].add("ranked_group_row")
-            if tag in active_set:
-                rec["sources"].add("selected_active")
-            if tag in direct_set:
-                rec["sources"].add("selected_direct")
-            if tag in implied_set:
-                rec["sources"].add("selected_implied")
-
-    payload = {
-        "n_tags": len(info_by_tag),
-        "tags": [
-            {
-                "tag": tag,
-                "rows": rec["rows"],
-                "sources": sorted(rec["sources"]),
-            }
-            for tag, rec in sorted(info_by_tag.items())
-        ],
+            if tag in active_set:
+                rec["sources"].add("selected_active")
+            if tag in direct_set:
+                rec["sources"].add("selected_direct")
+            if tag in implied_set:
+                rec["sources"].add("selected_implied")
+
+    payload = {
+        "n_tags": len(info_by_tag),
+        "tags": [
+            {
+                "tag": tag,
+                "rows": rec["rows"],
+                "sources": sorted(rec["sources"]),
+            }
+            for tag, rec in sorted(info_by_tag.items())
+        ],
     }
     return "Display Tag Audit: " + json.dumps(payload, ensure_ascii=True)
 
@@ -1313,16 +1313,16 @@ def _build_row_component_updates(
                     choices=choices,
                     value=values,
                     visible=visible,
-                )
-            )
+                )
+            )
         else:
             header_updates.append(gr.update(value="", visible=False))
             checkbox_updates.append(gr.update(choices=[], value=[], visible=False))
 
     prompt_text = _compose_toggle_prompt_text(list(selected), row_defs_ui)
     return prompt_text, row_values_state, header_updates, checkbox_updates
-
-
+
+
 def _on_toggle_row(
     row_idx: int,
     changed_values: List[str],
@@ -1408,8 +1408,8 @@ def _on_toggle_row(
         prompt_text,
         *checkbox_updates,
     ]
-
-
+
+
 def _build_ui_payload(
     *,
     console_text: str,
@@ -1774,114 +1774,114 @@ def _build_selection_query(
     probe_tags: List[str],
     classifier_auto_tags: Optional[List[str]] = None,
 ) -> str:
-    lines = [f"IMAGE DESCRIPTION: {prompt_in.strip()}"]
-    if rewritten and rewritten.strip():
-        lines.append(f"REWRITE PHRASES: {rewritten.strip()}")
-    hint_tags = []
-    if structural_tags:
-        hint_tags.extend(structural_tags)
+    lines = [f"IMAGE DESCRIPTION: {prompt_in.strip()}"]
+    if rewritten and rewritten.strip():
+        lines.append(f"REWRITE PHRASES: {rewritten.strip()}")
+    hint_tags = []
+    if structural_tags:
+        hint_tags.extend(structural_tags)
     if probe_tags:
         hint_tags.extend(probe_tags)
     if classifier_auto_tags:
         hint_tags.extend(classifier_auto_tags)
-    if hint_tags:
-        # Keep hints as context only; selection still must choose by candidate indices.
-        lines.append(
-            "INFERRED TAG HINTS (context only): " + ", ".join(sorted(set(hint_tags)))
-        )
-    return "\n".join(lines)
-
-
-# Set up logging
-# Minimal prod logging: warnings+ to stderr, no file by default
-import os, logging
-
-LOG_LEVEL = os.environ.get("PSQ_LOG_LEVEL", "WARNING").upper()
-logging.basicConfig(
-    level=getattr(logging, LOG_LEVEL, logging.WARNING),
-    format="%(asctime)s %(levelname)s:%(message)s",
-    handlers=[logging.StreamHandler()]  # no file -> avoids huge logs on Spaces
-)
-
-# Quiet down common noisy libs (optional)
-for _name in ("gensim", "gradio", "hnswlib", "httpx", "uvicorn"):
-    logging.getLogger(_name).setLevel(logging.ERROR)
-
-# Turn off Gradio analytics phone-home to avoid those background thread errors (optional)
-os.environ["GRADIO_ANALYTICS_ENABLED"] = "0"
-
-
-MASCOT_DIR = Path(__file__).parent / "mascotimages"
-MASCOT_FILE = MASCOT_DIR / "transparentsquirrel.png"
-
-
-def _load_mascot_image():
-    """Load mascot image if available; return None when missing/unreadable."""
-    if not MASCOT_FILE.exists():
-        logging.warning("Mascot image missing: %s", MASCOT_FILE)
-        return None
-    try:
-        return Image.open(MASCOT_FILE).convert("RGBA")
-    except Exception as e:
-        logging.warning("Failed to load mascot image (%s): %s", MASCOT_FILE, e)
-        return None
-
-try:
-    from gradio_client import utils as _gc_utils
-
-    _orig_get_type = _gc_utils.get_type
-    _orig_j2p = _gc_utils._json_schema_to_python_type
-    _orig_pub = _gc_utils.json_schema_to_python_type
-
-    def _get_type_safe(schema):
-        # Sometimes schema is a bare True/False (JSON Schema boolean form)
-        if not isinstance(schema, dict):
-            return "any"
-        return _orig_get_type(schema)
-
-    def _j2p_safe(schema, defs=None):
-        # Accept non-dict schemas (True/False/None) and treat as "any"
-        if not isinstance(schema, dict):
-            return "any"
-        return _orig_j2p(schema, defs or schema.get("$defs"))
-
-    def _pub_safe(schema):
-        # Public wrapper used by Gradio; keep it resilient too
-        if not isinstance(schema, dict):
-            return "any"
-        return _j2p_safe(schema, schema.get("$defs"))
-
-    _gc_utils.get_type = _get_type_safe
-    _gc_utils._json_schema_to_python_type = _j2p_safe
-    _gc_utils.json_schema_to_python_type = _pub_safe
-
-except Exception as e:
-    print("gradio_client hotfix not applied:", e)
-# -------------------------------------------------------------------------------
-
-
-allow_nsfw_tags = False
-def _is_production_runtime() -> bool:
-    """Best-effort detection for deployed runtime (HF Spaces or explicit env)."""
-    if os.environ.get("PSQ_PRODUCTION", "").strip().lower() in {"1", "true", "yes"}:
-        return True
-    if os.environ.get("SPACE_ID"):
-        return True
-    if os.environ.get("HF_SPACE_ID"):
-        return True
-    if os.environ.get("SYSTEM") == "spaces":
-        return True
-    return False
-
-
-verbose_retrieval_default = "0" if _is_production_runtime() else "1"
+    if hint_tags:
+        # Keep hints as context only; selection still must choose by candidate indices.
+        lines.append(
+            "INFERRED TAG HINTS (context only): " + ", ".join(sorted(set(hint_tags)))
+        )
+    return "\n".join(lines)
+
+
+# Set up logging
+# Minimal prod logging: warnings+ to stderr, no file by default
+import os, logging
+
+LOG_LEVEL = os.environ.get("PSQ_LOG_LEVEL", "WARNING").upper()
+logging.basicConfig(
+    level=getattr(logging, LOG_LEVEL, logging.WARNING),
+    format="%(asctime)s %(levelname)s:%(message)s",
+    handlers=[logging.StreamHandler()]  # no file -> avoids huge logs on Spaces
+)
+
+# Quiet down common noisy libs (optional)
+for _name in ("gensim", "gradio", "hnswlib", "httpx", "uvicorn"):
+    logging.getLogger(_name).setLevel(logging.ERROR)
+
+# Turn off Gradio analytics phone-home to avoid those background thread errors (optional)
+os.environ["GRADIO_ANALYTICS_ENABLED"] = "0"
+
+
+MASCOT_DIR = Path(__file__).parent / "mascotimages"
+MASCOT_FILE = MASCOT_DIR / "transparentsquirrel.png"
+
+
+def _load_mascot_image():
+    """Load mascot image if available; return None when missing/unreadable."""
+    if not MASCOT_FILE.exists():
+        logging.warning("Mascot image missing: %s", MASCOT_FILE)
+        return None
+    try:
+        return Image.open(MASCOT_FILE).convert("RGBA")
+    except Exception as e:
+        logging.warning("Failed to load mascot image (%s): %s", MASCOT_FILE, e)
+        return None
+
+try:
+    from gradio_client import utils as _gc_utils
+
+    _orig_get_type = _gc_utils.get_type
+    _orig_j2p = _gc_utils._json_schema_to_python_type
+    _orig_pub = _gc_utils.json_schema_to_python_type
+
+    def _get_type_safe(schema):
+        # Sometimes schema is a bare True/False (JSON Schema boolean form)
+        if not isinstance(schema, dict):
+            return "any"
+        return _orig_get_type(schema)
+
+    def _j2p_safe(schema, defs=None):
+        # Accept non-dict schemas (True/False/None) and treat as "any"
+        if not isinstance(schema, dict):
+            return "any"
+        return _orig_j2p(schema, defs or schema.get("$defs"))
+
+    def _pub_safe(schema):
+        # Public wrapper used by Gradio; keep it resilient too
+        if not isinstance(schema, dict):
+            return "any"
+        return _j2p_safe(schema, schema.get("$defs"))
+
+    _gc_utils.get_type = _get_type_safe
+    _gc_utils._json_schema_to_python_type = _j2p_safe
+    _gc_utils.json_schema_to_python_type = _pub_safe
+
+except Exception as e:
+    print("gradio_client hotfix not applied:", e)
+# -------------------------------------------------------------------------------
+
+
+allow_nsfw_tags = False
+def _is_production_runtime() -> bool:
+    """Best-effort detection for deployed runtime (HF Spaces or explicit env)."""
+    if os.environ.get("PSQ_PRODUCTION", "").strip().lower() in {"1", "true", "yes"}:
+        return True
+    if os.environ.get("SPACE_ID"):
+        return True
+    if os.environ.get("HF_SPACE_ID"):
+        return True
+    if os.environ.get("SYSTEM") == "spaces":
+        return True
+    return False
+
+
+verbose_retrieval_default = "0" if _is_production_runtime() else "1"
 verbose_retrieval = os.environ.get("PSQ_VERBOSE_RETRIEVAL", verbose_retrieval_default).strip().lower() in {"1", "true", "yes"}
 verbose_retrieval_all = False
 verbose_retrieval_limit = 20
 display_top_groups_default = int(os.environ.get("PSQ_DISPLAY_TOP_GROUPS", "10"))
 display_top_tags_per_group_default = int(os.environ.get("PSQ_DISPLAY_TOP_TAGS_PER_GROUP", "7"))
 display_rank_top_k_default = int(os.environ.get("PSQ_DISPLAY_GROUP_RANK_TOP_K", "7"))
-display_max_rows_default = int(os.environ.get("PSQ_DISPLAY_MAX_ROWS", "14"))
+display_max_rows_default = int(os.environ.get("PSQ_DISPLAY_MAX_ROWS", "14"))
 retrieval_global_k = int(os.environ.get("PSQ_RETRIEVAL_GLOBAL_K", "300"))
 retrieval_per_phrase_k = int(os.environ.get("PSQ_RETRIEVAL_PER_PHRASE_K", "10"))
 retrieval_per_phrase_final_k = int(os.environ.get("PSQ_RETRIEVAL_PER_PHRASE_FINAL_K", "1"))
@@ -1919,28 +1919,28 @@ _startup_profile_mark(
     "startup_preflight.done",
     error_count=len(STARTUP_PREFLIGHT_ERRORS),
 )
-
+
 css = """
-.scrollable-content{
-  max-height: 420px;
-  overflow-y: scroll;          /* always show scrollbar */
-  overflow-x: hidden;
-  padding-right: 8px;
-  padding-bottom: 14px;   /* <— add this */
-  scrollbar-gutter: stable;    /* prevent layout shift as it fills */
-
-  /* Firefox */
-  scrollbar-width: auto;                          
-  scrollbar-color: rgba(180,180,180,.9) rgba(0,0,0,.15);
-}
-
-/* WebKit/Chromium (Chrome/Edge/Safari) */
-.scrollable-content::-webkit-scrollbar{ width: 10px; }
-.scrollable-content::-webkit-scrollbar-thumb{ background: rgba(180,180,180,.9); border-radius: 8px; }
-.scrollable-content::-webkit-scrollbar-track{ background: rgba(0,0,0,.15); }
-
-/* (Optional) make both scroll panes taller so they fill more of the column */
-.pane-left  .scrollable-content,
+.scrollable-content{
+  max-height: 420px;
+  overflow-y: scroll;          /* always show scrollbar */
+  overflow-x: hidden;
+  padding-right: 8px;
+  padding-bottom: 14px;   /* <— add this */
+  scrollbar-gutter: stable;    /* prevent layout shift as it fills */
+
+  /* Firefox */
+  scrollbar-width: auto;                          
+  scrollbar-color: rgba(180,180,180,.9) rgba(0,0,0,.15);
+}
+
+/* WebKit/Chromium (Chrome/Edge/Safari) */
+.scrollable-content::-webkit-scrollbar{ width: 10px; }
+.scrollable-content::-webkit-scrollbar-thumb{ background: rgba(180,180,180,.9); border-radius: 8px; }
+.scrollable-content::-webkit-scrollbar-track{ background: rgba(0,0,0,.15); }
+
+/* (Optional) make both scroll panes taller so they fill more of the column */
+.pane-left  .scrollable-content,
 .pane-right .scrollable-content {
   max-height: 610px;                /* was 420px; tweak to taste */
 }
@@ -2765,28 +2765,28 @@ def rag_pipeline_ui(
 
         def _record_timing(stage: str, dt_s: float):
             stage_timings[stage] = float(dt_s)
-
-        def _emit_timing_summary(total_s: float):
-            summary_order = [
+
+        def _emit_timing_summary(total_s: float):
+            summary_order = [
                 "preprocess",
                 "rewrite",
                 "structural",
                 "classifier",
                 "retrieval",
-                "selection",
-                "implication_expansion",
-                "prompt_composition",
-                "group_display",
-            ]
-            lines = []
-            for k in summary_order:
-                if k in stage_timings:
-                    lines.append(f"{k}={stage_timings[k]:.2f}s")
-            slowest = max(stage_timings.items(), key=lambda kv: kv[1])[0] if stage_timings else "n/a"
-            log("Timing Summary: " + ", ".join(lines))
-            log(f"Timing Slowest Stage: {slowest}")
-            log(f"Timing Total: {total_s:.2f}s")
-
+                "selection",
+                "implication_expansion",
+                "prompt_composition",
+                "group_display",
+            ]
+            lines = []
+            for k in summary_order:
+                if k in stage_timings:
+                    lines.append(f"{k}={stage_timings[k]:.2f}s")
+            slowest = max(stage_timings.items(), key=lambda kv: kv[1])[0] if stage_timings else "n/a"
+            log("Timing Summary: " + ", ".join(lines))
+            log(f"Timing Slowest Stage: {slowest}")
+            log(f"Timing Total: {total_s:.2f}s")
+
         def _append_timing_jsonl(total_s: float):
             try:
                 timing_log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -2805,12 +2805,12 @@ def rag_pipeline_ui(
                     "timeout_select_s": stage3_select_timeout_s,
                     },
                 }
-                with timing_log_path.open("a", encoding="utf-8") as f:
-                    f.write(json.dumps(rec, ensure_ascii=True) + "\n")
-                log(f"Timing Log: wrote {timing_log_path}")
+                with timing_log_path.open("a", encoding="utf-8") as f:
+                    f.write(json.dumps(rec, ensure_ascii=True) + "\n")
+                log(f"Timing Log: wrote {timing_log_path}")
             except Exception as e:
                 log(f"Timing Log: failed ({type(e).__name__}: {_redact_console_error_text(e)})")
-
+
         def _future_with_timeout(
             fut,
             timeout_s: float,
@@ -2856,7 +2856,7 @@ def rag_pipeline_ui(
                     raise RuntimeError(msg)
                 log(f"{msg}; using fallback")
                 return fallback
-
+
         t_total0 = time.perf_counter()
         log("Start: received prompt")
         if STARTUP_PREFLIGHT_ERRORS:
@@ -2883,7 +2883,7 @@ def rag_pipeline_ui(
                 suggested_prompt_text='Enter a prompt and click "Run".',
             )
             return
-            
+            
         log("Input:")
         log(prompt_in)
         log("")
@@ -2997,7 +2997,7 @@ def rag_pipeline_ui(
                 f"{', '.join(removed_exact_excluded)}"
             )
         log("")
-
+
         rewrite_prefilled = (rewrite_override or "").strip()
         if rewrite_prefilled:
             log("Step 1: structural + classifier inference (rewrite already prepared)")
@@ -3117,9 +3117,9 @@ def rag_pipeline_ui(
         if retrieval_query_hints:
             # keep them separate in logs, but allow them to help retrieval
             rewrite_for_retrieval = (rewrite_for_retrieval + ", " + ", ".join(retrieval_query_hints)).strip(", ").strip()
-
-
-        log("Step 2: Prompt Squirrel retrieval (hidden)")
+
+
+        log("Step 2: Prompt Squirrel retrieval (hidden)")
         try:
             t0 = time.perf_counter()
             retrieval_context_tags = list(dict.fromkeys((structural_tags or []) + (probe_tags or [])))
@@ -3181,42 +3181,42 @@ def rag_pipeline_ui(
             _record_timing("retrieval", dt)
             log(f"Retrieval: {dt:.2f}s")
             log(f"Retrieved {len(candidates)} candidate tags")
-            if verbose_retrieval:
-                log(f"Total unique candidates: {len(candidates)}")
-                limit = None if verbose_retrieval_all else max(1, int(verbose_retrieval_limit))
-                for report in phrase_reports:
-                    phrase = report.get("normalized") or report.get("phrase") or ""
-                    lookup = report.get("lookup") or ""
-                    tfidf_vocab = report.get("tfidf_vocab")
-                    log(f"Phrase: {phrase} (lookup={lookup}) tfidf_vocab={tfidf_vocab}")
-                    rows = report.get("candidates", [])
-                    shown = rows if limit is None else rows[:limit]
-                    for row in shown:
-                          tag = row.get("tag")
-                          alias_token = row.get("alias_token")
-                          score_fasttext = row.get("score_fasttext")
-                          score_context = row.get("score_context")
-                          score_combined = row.get("score_combined")
-                          count = row.get("count")
-                          alias_part = ""
-                          if alias_token and alias_token != tag:
-                              alias_part = f" [alias_token={alias_token}]"
-                          fasttext_str = (
-                              f"{score_fasttext:.3f}" if isinstance(score_fasttext, (int, float)) else score_fasttext
-                          )
-                          if score_context is None:
-                              context_str = "None"
-                          else:
-                              context_str = (
-                                  f"{score_context:.3f}" if isinstance(score_context, (int, float)) else score_context
-                              )
-                          combined_str = (
-                              f"{score_combined:.3f}" if isinstance(score_combined, (int, float)) else score_combined
-                          )
-                          log(
-                              f"  {tag}{alias_part} | fasttext={fasttext_str} context={context_str} "
-                              f"combined={combined_str} count={count}"
-                          )
+            if verbose_retrieval:
+                log(f"Total unique candidates: {len(candidates)}")
+                limit = None if verbose_retrieval_all else max(1, int(verbose_retrieval_limit))
+                for report in phrase_reports:
+                    phrase = report.get("normalized") or report.get("phrase") or ""
+                    lookup = report.get("lookup") or ""
+                    tfidf_vocab = report.get("tfidf_vocab")
+                    log(f"Phrase: {phrase} (lookup={lookup}) tfidf_vocab={tfidf_vocab}")
+                    rows = report.get("candidates", [])
+                    shown = rows if limit is None else rows[:limit]
+                    for row in shown:
+                          tag = row.get("tag")
+                          alias_token = row.get("alias_token")
+                          score_fasttext = row.get("score_fasttext")
+                          score_context = row.get("score_context")
+                          score_combined = row.get("score_combined")
+                          count = row.get("count")
+                          alias_part = ""
+                          if alias_token and alias_token != tag:
+                              alias_part = f" [alias_token={alias_token}]"
+                          fasttext_str = (
+                              f"{score_fasttext:.3f}" if isinstance(score_fasttext, (int, float)) else score_fasttext
+                          )
+                          if score_context is None:
+                              context_str = "None"
+                          else:
+                              context_str = (
+                                  f"{score_context:.3f}" if isinstance(score_context, (int, float)) else score_context
+                              )
+                          combined_str = (
+                              f"{score_combined:.3f}" if isinstance(score_combined, (int, float)) else score_combined
+                          )
+                          log(
+                              f"  {tag}{alias_part} | fasttext={fasttext_str} context={context_str} "
+                              f"combined={combined_str} count={count}"
+                          )
                     if limit is not None and len(rows) > limit:
                         log(f"  ... ({len(rows) - limit} more)")
         except Exception as e:
@@ -3301,16 +3301,16 @@ def rag_pipeline_ui(
                 f"(<{min_tag_count}): {', '.join(removed_stage3_low)}"
             )
         selected_tags = list(selection_selected_tags)
-
-        if structural_tags:
-            # Add structural tags that aren't already selected
-            existing = {t for t in selected_tags}
-            new_structural = [t for t in structural_tags if t not in existing]
-            selected_tags.extend(new_structural)
-            log(f"  Added {len(new_structural)} structural tags: {', '.join(new_structural)}")
-        else:
-            log("  No structural tags inferred")
-
+
+        if structural_tags:
+            # Add structural tags that aren't already selected
+            existing = {t for t in selected_tags}
+            new_structural = [t for t in structural_tags if t not in existing]
+            selected_tags.extend(new_structural)
+            log(f"  Added {len(new_structural)} structural tags: {', '.join(new_structural)}")
+        else:
+            log("  No structural tags inferred")
+
         if probe_tags:
             existing = {t for t in selected_tags}
             new_probe = [t for t in probe_tags if t not in existing]
@@ -3335,9 +3335,9 @@ def rag_pipeline_ui(
             log("  No high-confidence classifier tags")
 
         selected_tags, removed_excluded_direct = _filter_excluded_recommendation_tags(selected_tags)
-        if removed_excluded_direct:
-            log(f"  Removed {len(removed_excluded_direct)} excluded tags: {', '.join(removed_excluded_direct)}")
-
+        if removed_excluded_direct:
+            log(f"  Removed {len(removed_excluded_direct)} excluded tags: {', '.join(removed_excluded_direct)}")
+
         direct_selected_tags = list(dict.fromkeys(selected_tags))
 
         status_states["reranker"] = {
@@ -3349,12 +3349,12 @@ def rag_pipeline_ui(
         yield _progress_payload()
 
         log("Step 3c: Expand via tag implications")
-        t0 = time.perf_counter()
-        tag_set = set(selected_tags)
-        expanded, implied_only = expand_tags_via_implications(tag_set)
-        dt = time.perf_counter()-t0
-        _record_timing("implication_expansion", dt)
-        log(f"Implication expansion: {dt:.2f}s")
+        t0 = time.perf_counter()
+        tag_set = set(selected_tags)
+        expanded, implied_only = expand_tags_via_implications(tag_set)
+        dt = time.perf_counter()-t0
+        _record_timing("implication_expansion", dt)
+        log(f"Implication expansion: {dt:.2f}s")
         implied_selected_tags = sorted(implied_only) if implied_only else []
         if implied_only:
             implied_added = sorted(implied_only)
@@ -3370,24 +3370,24 @@ def rag_pipeline_ui(
                 )
         else:
             log("  No additional implied tags")
-
-        selected_tags, removed_excluded_implied = _filter_excluded_recommendation_tags(selected_tags)
-        implied_selected_tags = [
-            t for t in implied_selected_tags if not _is_excluded_recommendation_tag(t)
-        ]
-        if removed_excluded_implied:
-            log(
-                f"  Removed {len(removed_excluded_implied)} excluded tags after implications: "
-                f"{', '.join(removed_excluded_implied)}"
-            )
-
-        log("Step 4: Compose final prompt")
-        t0 = time.perf_counter()
-        final_prompt = compose_final_prompt(rewritten, selected_tags)
-        dt = time.perf_counter()-t0
-        _record_timing("prompt_composition", dt)
-        log(f"Prompt composition: {dt:.2f}s")
-
+
+        selected_tags, removed_excluded_implied = _filter_excluded_recommendation_tags(selected_tags)
+        implied_selected_tags = [
+            t for t in implied_selected_tags if not _is_excluded_recommendation_tag(t)
+        ]
+        if removed_excluded_implied:
+            log(
+                f"  Removed {len(removed_excluded_implied)} excluded tags after implications: "
+                f"{', '.join(removed_excluded_implied)}"
+            )
+
+        log("Step 4: Compose final prompt")
+        t0 = time.perf_counter()
+        final_prompt = compose_final_prompt(rewritten, selected_tags)
+        dt = time.perf_counter()-t0
+        _record_timing("prompt_composition", dt)
+        log(f"Prompt composition: {dt:.2f}s")
+
         log("Step 5: Build ranked group/category display")
         t0 = time.perf_counter()
         seed_terms = []
@@ -3398,7 +3398,7 @@ def rag_pipeline_ui(
         seed_terms.extend(classifier_auto_tags or [])
         seed_terms.extend(classifier_candidate_tags or [])
         seed_terms.extend(selected_tags)
-        seed_terms = list(dict.fromkeys(seed_terms))
+        seed_terms = list(dict.fromkeys(seed_terms))
 
         active_selected_tags = list(dict.fromkeys(selected_tags))
         structural_set = {_norm_tag_for_lookup(t) for t in (structural_tags or []) if t}
@@ -3514,8 +3514,8 @@ def rag_pipeline_ui(
             suggested_prompt_text=_format_user_facing_error(e),
         )
         return
-
-    
+
+    
 _startup_profile_mark("ui.blocks_build_begin")
 with gr.Blocks(css=css, js=client_js) as app:
     with gr.Row():
@@ -3615,21 +3615,21 @@ with gr.Blocks(css=css, js=client_js) as app:
                 visible=False,
                 interactive=False,
             )
-
-    with gr.Accordion("Display Settings", open=False):
-        with gr.Row():
-            display_top_groups = gr.Number(
-                value=display_top_groups_default,
-                precision=0,
-                label="Rows (Top Groups/Categories)",
-                minimum=1,
-            )
-            display_top_tags_per_group = gr.Number(
-                value=display_top_tags_per_group_default,
-                precision=0,
-                label="Top Tags Shown Per Row",
-                minimum=1,
-            )
+
+    with gr.Accordion("Display Settings", open=False):
+        with gr.Row():
+            display_top_groups = gr.Number(
+                value=display_top_groups_default,
+                precision=0,
+                label="Rows (Top Groups/Categories)",
+                minimum=1,
+            )
+            display_top_tags_per_group = gr.Number(
+                value=display_top_tags_per_group_default,
+                precision=0,
+                label="Top Tags Shown Per Row",
+                minimum=1,
+            )
             display_rank_top_k = gr.Number(
                 value=display_rank_top_k_default,
                 precision=0,
@@ -3710,7 +3710,7 @@ with gr.Blocks(css=css, js=client_js) as app:
         *row_checkboxes,
     ]
     run_outputs_with_rewrite = [*run_outputs, rewrite_state]
-
+
     image_tags.change(
         _update_run_button_visibility,
         inputs=[image_tags, last_run_prompt_state],
@@ -3785,7 +3785,7 @@ with gr.Blocks(css=css, js=client_js) as app:
         show_progress="minimal",
         show_progress_on=[mascot_img],
     )
-
+
     for idx, row_cb in enumerate(row_checkboxes):
         row_cb.change(
             fn=lambda changed_values, selected_state, rows_dirty, row_defs, row_values, i=idx: _on_toggle_row(

models/finetune/tag-classifier-modernbert-full/classifier_precision_thresholds_calibrated.csv

@@ -0,0 +1,131 @@
+tag,target_precision,threshold,val_gold_n,val_pred_n,val_tp,val_precision,val_recall,test_gold_n,test_pred_n,test_tp,test_fp,test_fn,test_precision,test_recall,test_f1
+solo,0.95,0.61279296875,1842,1784,1695,0.9501121076233184,0.9201954397394136,1785,1768,1649,119,136,0.9326923076923077,0.9238095238095239,0.9282296650717704
+solo,0.9,0.1419677734375,1842,2010,1809,0.9,0.9820846905537459,1785,2009,1746,263,39,0.8690890990542558,0.9781512605042016,0.9204006325777543
+solo,0.8,0.056976318359375,1842,2292,1834,0.800174520069808,0.995656894679696,1785,2269,1770,499,15,0.7800793301013662,0.9915966386554622,0.8732116428219043
+solo,0.5,0.0022430419921875,1842,3000,1842,0.614,1.0,1785,2993,1785,1208,0,0.5963915803541597,1.0,0.7471745500209293
+solo,0.2,0.0022430419921875,1842,3000,1842,0.614,1.0,1785,2993,1785,1208,0,0.5963915803541597,1.0,0.7471745500209293
+anthro,0.95,0.81591796875,1968,1010,960,0.9504950495049505,0.4878048780487805,1926,997,944,53,982,0.9468405215646941,0.490134994807892,0.6459117345193295
+anthro,0.9,0.47412109375,1968,1873,1686,0.9001601708489055,0.8567073170731707,1926,1811,1607,204,319,0.8873550524572059,0.8343717549325026,0.8600481669788601
+anthro,0.8,0.1680908203125,1968,2372,1898,0.8001686340640809,0.9644308943089431,1926,2360,1856,504,70,0.7864406779661017,0.9636552440290758,0.8660755949603359
+anthro,0.5,0.0045928955078125,1968,3000,1968,0.656,1.0,1926,3000,1926,1074,0,0.642,1.0,0.781973203410475
+anthro,0.2,0.0045928955078125,1968,3000,1968,0.656,1.0,1926,3000,1926,1074,0,0.642,1.0,0.781973203410475
+mammal,0.95,0.46044921875,2200,2070,1967,0.9502415458937198,0.894090909090909,2222,2064,1973,91,249,0.9559108527131783,0.8879387938793879,0.9206719552029864
+mammal,0.9,0.1807861328125,2200,2355,2120,0.9002123142250531,0.9636363636363636,2222,2368,2120,248,102,0.8952702702702703,0.9540954095409541,0.9237472766884531
+mammal,0.8,0.033966064453125,2200,2727,2183,0.8005133846718006,0.9922727272727273,2222,2741,2210,531,12,0.8062750820868296,0.9945994599459946,0.8905903687285917
+mammal,0.5,0.0003554821014404297,2200,3000,2200,0.7333333333333333,1.0,2222,3000,2222,778,0,0.7406666666666667,1.0,0.8510149368058215
+mammal,0.2,0.0003554821014404297,2200,3000,2200,0.7333333333333333,1.0,2222,3000,2222,778,0,0.7406666666666667,1.0,0.8510149368058215
+clothing,0.95,0.9404296875,1774,224,213,0.9508928571428571,0.12006764374295378,1730,195,187,8,1543,0.958974358974359,0.10809248554913295,0.1942857142857143
+clothing,0.9,0.76953125,1774,1064,958,0.900375939849624,0.5400225479143179,1730,999,910,89,820,0.9109109109109109,0.5260115606936416,0.6669109563942837
+clothing,0.8,0.298095703125,1774,2082,1666,0.8001921229586936,0.939120631341601,1730,2065,1614,451,116,0.7815980629539951,0.9329479768786128,0.850592885375494
+clothing,0.5,0.0038394927978515625,1774,3000,1774,0.5913333333333334,1.0,1730,2997,1730,1267,0,0.5772439105772439,1.0,0.7319653056907128
+clothing,0.2,0.0038394927978515625,1774,3000,1774,0.5913333333333334,1.0,1730,2997,1730,1267,0,0.5772439105772439,1.0,0.7319653056907128
+clothed,0.95,,1001,0,0,,,997,0,0,0,997,0.0,0.0,0.0
+clothed,0.9,,1001,0,0,,,997,0,0,0,997,0.0,0.0,0.0
+clothed,0.8,,1001,0,0,,,997,0,0,0,997,0.0,0.0,0.0
+clothed,0.5,0.429931640625,1001,1760,880,0.5,0.8791208791208791,997,1691,860,831,137,0.5085748078060319,0.8625877632898696,0.6398809523809523
+clothed,0.2,0.006511688232421875,1001,3000,1001,0.33366666666666667,1.0,997,3000,997,2003,0,0.3323333333333333,1.0,0.4988741556167125
+fur,0.95,0.908203125,1270,67,64,0.9552238805970149,0.050393700787401574,1250,70,62,8,1188,0.8857142857142857,0.0496,0.09393939393939393
+fur,0.9,0.89111328125,1270,101,91,0.900990099009901,0.07165354330708662,1250,108,97,11,1153,0.8981481481481481,0.0776,0.14285714285714285
+fur,0.8,0.8017578125,1270,370,296,0.8,0.23307086614173228,1250,376,316,60,934,0.8404255319148937,0.2528,0.3886838868388684
+fur,0.5,0.159423828125,1270,2506,1253,0.5,0.9866141732283464,1250,2452,1224,1228,26,0.499184339314845,0.9792,0.6612641815235007
+fur,0.2,0.01078033447265625,1270,3000,1270,0.42333333333333334,1.0,1250,2998,1250,1748,0,0.41694462975316876,1.0,0.588512241054614
+hair,0.95,0.9326171875,1148,63,60,0.9523809523809523,0.05226480836236934,1169,72,65,7,1104,0.9027777777777778,0.055603079555175364,0.10475423045930701
+hair,0.9,0.8955078125,1148,171,154,0.9005847953216374,0.13414634146341464,1169,187,161,26,1008,0.8609625668449198,0.1377245508982036,0.23746312684365783
+hair,0.8,0.79931640625,1148,526,421,0.8003802281368821,0.3667247386759582,1169,529,434,95,735,0.8204158790170132,0.3712574850299401,0.5111896348645466
+hair,0.5,0.2115478515625,1148,2206,1103,0.5,0.960801393728223,1169,2185,1117,1068,52,0.5112128146453089,0.9555175363558597,0.6660703637447822
+hair,0.2,0.0087127685546875,1148,3000,1148,0.38266666666666665,1.0,1169,2995,1169,1826,0,0.3903171953255426,1.0,0.5614793467819404
+blue_eyes,0.95,0.9482421875,431,37,36,0.972972972972973,0.08352668213457076,390,34,28,6,362,0.8235294117647058,0.07179487179487179,0.1320754716981132
+blue_eyes,0.9,0.93359375,431,54,49,0.9074074074074074,0.1136890951276102,390,53,44,9,346,0.8301886792452831,0.11282051282051282,0.1986455981941309
+blue_eyes,0.8,0.833984375,431,199,160,0.8040201005025126,0.37122969837587005,390,184,141,43,249,0.7663043478260869,0.36153846153846153,0.4912891986062717
+blue_eyes,0.5,0.60205078125,431,556,278,0.5,0.6450116009280742,390,548,258,290,132,0.4708029197080292,0.6615384615384615,0.5501066098081023
+blue_eyes,0.2,0.218994140625,431,2050,410,0.2,0.951276102088167,390,2003,370,1633,20,0.18472291562656015,0.9487179487179487,0.3092352695361471
+canid,0.95,0.92236328125,889,706,671,0.9504249291784702,0.7547806524184477,824,663,630,33,194,0.9502262443438914,0.7645631067961165,0.847343644922663
+canid,0.9,0.70654296875,889,864,778,0.9004629629629629,0.875140607424072,824,810,723,87,101,0.8925925925925926,0.8774271844660194,0.8849449204406364
+canid,0.8,0.3583984375,889,1036,829,0.8001930501930502,0.9325084364454443,824,983,765,218,59,0.7782299084435402,0.9283980582524272,0.8467072495849475
+canid,0.5,0.053009033203125,889,1750,875,0.5,0.984251968503937,824,1724,812,912,12,0.4709976798143852,0.9854368932038835,0.6373626373626374
+canid,0.2,2.6047229766845703e-05,889,3000,889,0.29633333333333334,1.0,824,3000,824,2176,0,0.27466666666666667,1.0,0.4309623430962343
+canine,0.95,0.94384765625,866,615,585,0.9512195121951219,0.6755196304849884,804,580,554,26,250,0.9551724137931035,0.6890547263681592,0.800578034682081
+canine,0.9,0.69921875,866,843,759,0.900355871886121,0.8764434180138568,804,806,709,97,95,0.8796526054590571,0.8818407960199005,0.8807453416149069
+canine,0.8,0.366943359375,866,1014,812,0.8007889546351085,0.9376443418013857,804,972,748,224,56,0.7695473251028807,0.9303482587064676,0.8423423423423424
+canine,0.5,0.058990478515625,866,1706,853,0.5,0.9849884526558892,804,1669,791,878,13,0.47393648891551826,0.9838308457711443,0.639708855640922
+canine,0.2,0.00010389089584350586,866,3000,866,0.2886666666666667,1.0,804,3000,804,2196,0,0.268,1.0,0.42271293375394325
+simple_background,0.95,0.95361328125,981,6,6,1.0,0.0061162079510703364,979,5,5,0,974,1.0,0.005107252298263534,0.01016260162601626
+simple_background,0.9,0.8837890625,981,81,73,0.9012345679012346,0.0744138634046891,979,93,79,14,900,0.8494623655913979,0.08069458631256383,0.14738805970149252
+simple_background,0.8,0.8173828125,981,227,182,0.801762114537445,0.18552497451580022,979,258,201,57,778,0.7790697674418605,0.20531154239019409,0.3249797898140663
+simple_background,0.5,0.388427734375,981,1701,851,0.5002939447383892,0.8674821610601428,979,1738,852,886,127,0.4902186421173763,0.8702757916241062,0.6271623113728377
+simple_background,0.2,0.008575439453125,981,3000,981,0.327,1.0,979,2996,979,2017,0,0.3267690253671562,1.0,0.49257861635220124
+male,0.95,0.9306640625,1276,312,297,0.9519230769230769,0.23275862068965517,1301,290,278,12,1023,0.9586206896551724,0.21368178324365872,0.349465744814582
+male,0.9,0.84521484375,1276,593,535,0.9021922428330523,0.41927899686520376,1301,576,529,47,772,0.9184027777777778,0.40661029976940816,0.5636654235482151
+male,0.8,0.5595703125,1276,1215,974,0.8016460905349794,0.7633228840125392,1301,1234,992,242,309,0.8038897893030794,0.7624903920061491,0.7826429980276134
+male,0.5,0.0616455078125,1276,2528,1264,0.5,0.9905956112852664,1301,2520,1294,1226,7,0.5134920634920634,0.994619523443505,0.6773096048154932
+male,0.2,0.0026531219482421875,1276,3000,1276,0.42533333333333334,1.0,1301,3000,1301,1699,0,0.43366666666666664,1.0,0.6049755870727738
+female,0.95,0.78857421875,1540,1151,1094,0.950477845351868,0.7103896103896103,1504,1163,1090,73,414,0.937231298366294,0.7247340425531915,0.817397825271841
+female,0.9,0.56640625,1540,1420,1278,0.9,0.8298701298701299,1504,1423,1268,155,236,0.8910751932536893,0.8430851063829787,0.8664161257259992
+female,0.8,0.2783203125,1540,1785,1428,0.8,0.9272727272727272,1504,1805,1408,397,96,0.7800554016620499,0.9361702127659575,0.8510123904502872
+female,0.5,0.0008296966552734375,1540,3000,1540,0.5133333333333333,1.0,1504,2998,1504,1494,0,0.5016677785190127,1.0,0.6681474900044425
+female,0.2,0.0008296966552734375,1540,3000,1540,0.5133333333333333,1.0,1504,2998,1504,1494,0,0.5016677785190127,1.0,0.6681474900044425
+ambiguous_gender,0.95,,488,0,0,,,497,0,0,0,497,0.0,0.0,0.0
+ambiguous_gender,0.9,,488,0,0,,,497,0,0,0,497,0.0,0.0,0.0
+ambiguous_gender,0.8,0.93359375,488,115,92,0.8,0.1885245901639344,497,98,75,23,422,0.7653061224489796,0.15090543259557343,0.25210084033613445
+ambiguous_gender,0.5,0.59375,488,734,367,0.5,0.7520491803278688,497,726,365,361,132,0.5027548209366391,0.7344064386317908,0.5968928863450532
+ambiguous_gender,0.2,0.07574462890625,488,2430,486,0.2,0.9959016393442623,497,2445,494,1951,3,0.20204498977505111,0.993963782696177,0.335825968728756
+text,0.95,0.9365234375,831,260,247,0.95,0.2972322503008424,782,285,265,20,517,0.9298245614035088,0.3388746803069054,0.4967197750702905
+text,0.9,0.84716796875,831,424,382,0.9009433962264151,0.45968712394705175,782,436,387,49,395,0.8876146788990825,0.4948849104859335,0.6354679802955665
+text,0.8,0.67724609375,831,637,511,0.8021978021978022,0.614921780986763,782,612,497,115,285,0.8120915032679739,0.6355498721227621,0.7130559540889526
+text,0.5,0.343994140625,831,1342,671,0.5,0.8074608904933814,782,1344,655,689,127,0.48735119047619047,0.8375959079283888,0.6161806208842898
+text,0.2,0.0240936279296875,831,3000,831,0.277,1.0,782,3000,782,2218,0,0.26066666666666666,1.0,0.41353781068217876
+looking_at_viewer,0.95,,566,0,0,,,525,0,0,0,525,0.0,0.0,0.0
+looking_at_viewer,0.9,,566,0,0,,,525,0,0,0,525,0.0,0.0,0.0
+looking_at_viewer,0.8,,566,0,0,,,525,0,0,0,525,0.0,0.0,0.0
+looking_at_viewer,0.5,0.84912109375,566,79,40,0.5063291139240507,0.0706713780918728,525,78,40,38,485,0.5128205128205128,0.0761904761904762,0.1326699834162521
+looking_at_viewer,0.2,0.0953369140625,566,2809,562,0.20007119971520113,0.9929328621908127,525,2780,522,2258,3,0.18776978417266188,0.9942857142857143,0.3158850226928896
+open_mouth,0.95,,555,0,0,,,547,0,0,0,547,0.0,0.0,0.0
+open_mouth,0.9,,555,0,0,,,547,0,0,0,547,0.0,0.0,0.0
+open_mouth,0.8,0.92529296875,555,12,10,0.8333333333333334,0.018018018018018018,547,9,8,1,539,0.8888888888888888,0.014625228519195612,0.028776978417266185
+open_mouth,0.5,0.69775390625,555,402,201,0.5,0.3621621621621622,547,382,172,210,375,0.450261780104712,0.31444241316270566,0.37029063509149623
+open_mouth,0.2,0.16357421875,555,2730,546,0.2,0.9837837837837838,547,2707,538,2169,9,0.19874399704469892,0.9835466179159049,0.3306699446834665
+smile,0.95,0.94970703125,774,9,9,1.0,0.011627906976744186,768,4,4,0,764,1.0,0.005208333333333333,0.010362694300518135
+smile,0.9,0.94873046875,774,10,9,0.9,0.011627906976744186,768,4,4,0,764,1.0,0.005208333333333333,0.010362694300518135
+smile,0.8,0.84375,774,143,115,0.8041958041958042,0.14857881136950904,768,156,120,36,648,0.7692307692307693,0.15625,0.2597402597402597
+smile,0.5,0.55322265625,774,1062,531,0.5,0.686046511627907,768,1016,486,530,282,0.47834645669291337,0.6328125,0.5448430493273543
+smile,0.2,0.0257110595703125,774,3000,774,0.258,1.0,768,2995,768,2227,0,0.25642737896494155,1.0,0.4081849588094605
+group,0.95,0.98974609375,323,24,23,0.9583333333333334,0.07120743034055728,347,33,33,0,314,1.0,0.09510086455331412,0.17368421052631577
+group,0.9,0.9794921875,323,62,56,0.9032258064516129,0.17337461300309598,347,69,64,5,283,0.927536231884058,0.1844380403458213,0.30769230769230765
+group,0.8,0.9130859375,323,212,170,0.8018867924528302,0.5263157894736842,347,236,192,44,155,0.8135593220338984,0.553314121037464,0.6586620926243568
+group,0.5,0.54638671875,323,566,283,0.5,0.8761609907120743,347,588,291,297,56,0.49489795918367346,0.8386167146974063,0.6224598930481283
+group,0.2,0.0794677734375,323,1589,318,0.2001258653241032,0.9845201238390093,347,1613,340,1273,7,0.21078735275883448,0.9798270893371758,0.3469387755102041
+duo,0.95,0.97802734375,683,123,117,0.9512195121951219,0.17130307467057102,708,146,134,12,574,0.9178082191780822,0.18926553672316385,0.31381733021077285
+duo,0.9,0.9482421875,683,260,234,0.9,0.34260614934114203,708,289,254,35,454,0.8788927335640139,0.3587570621468927,0.5095285857572719
+duo,0.8,0.7666015625,683,666,533,0.8003003003003003,0.780380673499268,708,681,555,126,153,0.8149779735682819,0.7838983050847458,0.7991360691144708
+duo,0.5,0.1884765625,683,1302,651,0.5,0.9531478770131772,708,1257,674,583,34,0.5361972951471758,0.9519774011299436,0.6860050890585242
+duo,0.2,0.0023784637451171875,683,3000,683,0.22766666666666666,1.0,708,3000,708,2292,0,0.236,1.0,0.38187702265372164
+feral,0.95,0.97802734375,612,80,76,0.95,0.12418300653594772,611,92,80,12,531,0.8695652173913043,0.1309328968903437,0.22759601706970128
+feral,0.9,0.95947265625,612,193,174,0.9015544041450777,0.28431372549019607,611,178,157,21,454,0.8820224719101124,0.2569558101472995,0.3979721166032953
+feral,0.8,0.88525390625,612,397,318,0.801007556675063,0.5196078431372549,611,407,327,80,284,0.8034398034398035,0.5351882160392799,0.6424361493123772
+feral,0.5,0.35009765625,612,1140,570,0.5,0.9313725490196079,611,1162,557,605,54,0.4793459552495697,0.911620294599018,0.6283135927805978
+feral,0.2,0.0009436607360839844,612,3000,612,0.204,1.0,611,2999,611,2388,0,0.20373457819273091,1.0,0.33850415512465376
+red_eyes,0.95,0.978515625,196,5,5,1.0,0.025510204081632654,182,4,4,0,178,1.0,0.02197802197802198,0.043010752688172046
+red_eyes,0.9,0.978515625,196,5,5,1.0,0.025510204081632654,182,4,4,0,178,1.0,0.02197802197802198,0.043010752688172046
+red_eyes,0.8,0.86474609375,196,86,69,0.8023255813953488,0.3520408163265306,182,84,53,31,129,0.6309523809523809,0.29120879120879123,0.39849624060150374
+red_eyes,0.5,0.62158203125,196,243,122,0.5020576131687243,0.6224489795918368,182,277,105,172,77,0.37906137184115524,0.5769230769230769,0.45751633986928103
+red_eyes,0.2,0.34765625,196,790,158,0.2,0.8061224489795918,182,796,142,654,40,0.17839195979899497,0.7802197802197802,0.29038854805725967
+topwear,0.95,0.9736328125,603,5,5,1.0,0.008291873963515755,602,1,1,0,601,1.0,0.0016611295681063123,0.0033167495854063015
+topwear,0.9,0.9736328125,603,5,5,1.0,0.008291873963515755,602,1,1,0,601,1.0,0.0016611295681063123,0.0033167495854063015
+topwear,0.8,0.939453125,603,40,32,0.8,0.05306799336650083,602,39,33,6,569,0.8461538461538461,0.054817275747508304,0.1029641185647426
+topwear,0.5,0.61669921875,603,826,413,0.5,0.6849087893864013,602,779,418,361,184,0.5365853658536586,0.6943521594684385,0.6053584359160029
+topwear,0.2,0.0037364959716796875,603,3000,603,0.201,1.0,602,2999,602,2397,0,0.20073357785928642,1.0,0.3343515690086087
+bottomwear,0.95,0.970703125,423,5,5,1.0,0.01182033096926714,439,7,5,2,434,0.7142857142857143,0.011389521640091117,0.022421524663677136
+bottomwear,0.9,0.970703125,423,5,5,1.0,0.01182033096926714,439,7,5,2,434,0.7142857142857143,0.011389521640091117,0.022421524663677136
+bottomwear,0.8,0.97021484375,423,6,5,0.8333333333333334,0.01182033096926714,439,7,5,2,434,0.7142857142857143,0.011389521640091117,0.022421524663677136
+bottomwear,0.5,0.73095703125,423,478,240,0.502092050209205,0.5673758865248227,439,421,208,213,231,0.49406175771971494,0.47380410022779046,0.48372093023255813
+bottomwear,0.2,0.1458740234375,423,2060,412,0.2,0.9739952718676123,439,2047,430,1617,9,0.21006350757205666,0.979498861047836,0.34593724859211583
+teeth,0.95,,492,0,0,,,494,0,0,0,494,0.0,0.0,0.0
+teeth,0.9,0.96435546875,492,12,11,0.9166666666666666,0.022357723577235773,494,7,6,1,488,0.8571428571428571,0.012145748987854251,0.023952095808383235
+teeth,0.8,0.892578125,492,71,57,0.8028169014084507,0.11585365853658537,494,69,48,21,446,0.6956521739130435,0.09716599190283401,0.17051509769094136
+teeth,0.5,0.68408203125,492,442,221,0.5,0.4491869918699187,494,430,195,235,299,0.45348837209302323,0.39473684210526316,0.422077922077922
+teeth,0.2,0.22021484375,492,2410,482,0.2,0.9796747967479674,494,2426,471,1955,23,0.1941467436108821,0.9534412955465587,0.3226027397260274
+wings,0.95,0.97998046875,377,118,113,0.9576271186440678,0.29973474801061006,404,122,121,1,283,0.9918032786885246,0.2995049504950495,0.4600760456273764
+wings,0.9,0.9404296875,377,253,228,0.9011857707509882,0.6047745358090185,404,236,213,23,191,0.902542372881356,0.5272277227722773,0.665625
+wings,0.8,0.8583984375,377,356,285,0.800561797752809,0.7559681697612732,404,343,275,68,129,0.8017492711370262,0.6806930693069307,0.7362784471218207
+wings,0.5,0.3701171875,377,694,347,0.5,0.9204244031830239,404,709,375,334,29,0.5289139633286318,0.9282178217821783,0.6738544474393531
+wings,0.2,0.0386962890625,377,1855,371,0.2,0.9840848806366048,404,1800,402,1398,2,0.22333333333333333,0.995049504950495,0.3647912885662432

models/finetune/tag-classifier-modernbert-full/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f120a4119c4805ff55ecafe08f1b5b5683b0aebbd7a6809fabc41bcd6f0fec0
+size 309059422

models/finetune/tag-classifier-modernbert-full/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/finetune/tag-classifier-modernbert-full/tokenizer_config.json

@@ -0,0 +1,945 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50281": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50282": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50283": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50284": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50285": {
+      "content": "[unused0]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "[unused1]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "[unused2]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "[unused3]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "[unused4]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "[unused5]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "[unused6]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "[unused7]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "[unused8]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "[unused9]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50295": {
+      "content": "[unused10]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50296": {
+      "content": "[unused11]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50297": {
+      "content": "[unused12]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50298": {
+      "content": "[unused13]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50299": {
+      "content": "[unused14]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50300": {
+      "content": "[unused15]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50301": {
+      "content": "[unused16]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50302": {
+      "content": "[unused17]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50303": {
+      "content": "[unused18]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50304": {
+      "content": "[unused19]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50305": {
+      "content": "[unused20]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50306": {
+      "content": "[unused21]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50307": {
+      "content": "[unused22]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50308": {
+      "content": "[unused23]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50309": {
+      "content": "[unused24]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50310": {
+      "content": "[unused25]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50311": {
+      "content": "[unused26]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50312": {
+      "content": "[unused27]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50313": {
+      "content": "[unused28]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50314": {
+      "content": "[unused29]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50315": {
+      "content": "[unused30]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50316": {
+      "content": "[unused31]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50317": {
+      "content": "[unused32]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50318": {
+      "content": "[unused33]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50319": {
+      "content": "[unused34]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50320": {
+      "content": "[unused35]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50321": {
+      "content": "[unused36]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50322": {
+      "content": "[unused37]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50323": {
+      "content": "[unused38]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50324": {
+      "content": "[unused39]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50325": {
+      "content": "[unused40]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50326": {
+      "content": "[unused41]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50327": {
+      "content": "[unused42]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50328": {
+      "content": "[unused43]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50329": {
+      "content": "[unused44]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50330": {
+      "content": "[unused45]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50331": {
+      "content": "[unused46]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50332": {
+      "content": "[unused47]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50333": {
+      "content": "[unused48]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50334": {
+      "content": "[unused49]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50335": {
+      "content": "[unused50]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50336": {
+      "content": "[unused51]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50337": {
+      "content": "[unused52]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50338": {
+      "content": "[unused53]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50339": {
+      "content": "[unused54]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50340": {
+      "content": "[unused55]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50341": {
+      "content": "[unused56]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50342": {
+      "content": "[unused57]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50343": {
+      "content": "[unused58]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50344": {
+      "content": "[unused59]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50345": {
+      "content": "[unused60]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50346": {
+      "content": "[unused61]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50347": {
+      "content": "[unused62]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50348": {
+      "content": "[unused63]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50349": {
+      "content": "[unused64]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50350": {
+      "content": "[unused65]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50351": {
+      "content": "[unused66]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50352": {
+      "content": "[unused67]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50353": {
+      "content": "[unused68]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50354": {
+      "content": "[unused69]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50355": {
+      "content": "[unused70]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50356": {
+      "content": "[unused71]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50357": {
+      "content": "[unused72]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50358": {
+      "content": "[unused73]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50359": {
+      "content": "[unused74]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50360": {
+      "content": "[unused75]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50361": {
+      "content": "[unused76]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50362": {
+      "content": "[unused77]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50363": {
+      "content": "[unused78]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50364": {
+      "content": "[unused79]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50365": {
+      "content": "[unused80]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50366": {
+      "content": "[unused81]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50367": {
+      "content": "[unused82]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "[UNK]"
+}