Add n30 caption-evident set and per-group display blend overrides

data/analysis/per_group_weight_tuning_caption_evident_n30.csv

@@ -0,0 +1,24 @@
+group_name,support_samples,best_w_tfidf,best_w_fasttext,best_ndcg,ndcg_fasttext_only,ndcg_tfidf_only,delta_best_minus_fasttext,delta_tfidf_minus_fasttext
+clothing,1,1.0,0.0,1.0,0.386853,1.0,0.613147,0.613147
+expression_detail,11,0.2,0.8,0.818522,0.797766,0.677285,0.020756,-0.120481
+body_type,18,1.0,0.0,0.979496,0.958992,0.979496,0.020504,0.020504
+pose_action_detail,7,0.8,0.2,0.973933,0.959446,0.971103,0.014486,0.011657
+gender,17,0.7,0.3,0.93487,0.927168,0.905458,0.007702,-0.02171
+gaze_detail,4,0.1,0.9,0.8125,0.811163,0.633429,0.001337,-0.177734
+hair,11,0.0,1.0,1.0,1.0,0.966448,0.0,-0.033552
+general_activity_if_any,3,1.0,0.0,1.0,1.0,1.0,0.0,0.0
+gaze,3,0.5,0.5,1.0,1.0,0.833333,0.0,-0.166667
+fur_style,1,0.9,0.1,1.0,1.0,0.63093,0.0,-0.36907
+limbs,1,1.0,0.0,1.0,1.0,1.0,0.0,0.0
+posture,3,1.0,0.0,1.0,1.0,1.0,0.0,0.0
+body_decor,3,0.9,0.1,1.0,1.0,0.876977,0.0,-0.123023
+franchise_series,2,1.0,0.0,1.0,1.0,1.0,0.0,0.0
+count,27,1.0,0.0,0.986331,0.986331,0.986331,0.0,0.0
+species,16,0.0,1.0,0.957772,0.957772,0.896342,0.0,-0.06143
+expression,7,0.5,0.5,0.894551,0.894551,0.723087,0.0,-0.171465
+clothing_detail,12,0.0,1.0,0.856749,0.856749,0.658325,0.0,-0.198423
+color_markings,12,0.0,1.0,0.838306,0.838306,0.772345,0.0,-0.065961
+anatomy_features,12,0.0,1.0,0.803319,0.803319,0.728669,0.0,-0.074649
+objects_props,6,0.0,1.0,0.645258,0.645258,0.570952,0.0,-0.074306
+text,1,0.6,0.4,0.63093,0.63093,0.5,0.0,-0.13093
+background_composition,4,0.0,1.0,0.593497,0.593497,0.409213,0.0,-0.184284

psq_rag/ui/group_ranked_display.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import csv
+import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
@@ -9,7 +10,7 @@ from typing import Dict, List, Sequence, Tuple
 import numpy as np
 
 from psq_rag.retrieval.psq_retrieval import construct_pseudo_vector, _norm_tag_for_lookup
-from psq_rag.retrieval.state import get_tfidf_components, get_tfidf_tag_vectors
+from psq_rag.retrieval.state import get_fasttext_model, get_tfidf_components, get_tfidf_tag_vectors
 
 
 @dataclass
@@ -19,6 +20,92 @@ class GroupRankingRow:
     tags: List[Tuple[str, float]]
 
 
+def _resolve_display_weights() -> Tuple[float, float]:
+    tfidf_w = float(os.environ.get("PSQ_DISPLAY_TFIDF_WEIGHT", "0.0"))
+    fasttext_w = float(os.environ.get("PSQ_DISPLAY_FASTTEXT_WEIGHT", "1.0"))
+    tfidf_w, fasttext_w = _normalize_weights(tfidf_w, fasttext_w)
+    return tfidf_w, fasttext_w
+
+
+def _normalize_weights(tfidf_w: float, fasttext_w: float) -> Tuple[float, float]:
+    tfidf_w = max(0.0, float(tfidf_w))
+    fasttext_w = max(0.0, float(fasttext_w))
+    total = tfidf_w + fasttext_w
+    if total <= 1e-8:
+        return 1.0, 0.0
+    return tfidf_w / total, fasttext_w / total
+
+
+@lru_cache(maxsize=1)
+def _load_group_weight_overrides() -> Dict[str, Tuple[float, float]]:
+    csv_path = Path(
+        os.environ.get(
+            "PSQ_DISPLAY_GROUP_WEIGHT_PATH",
+            "data/analysis/per_group_weight_tuning_caption_evident_n30.csv",
+        )
+    )
+    if not csv_path.exists():
+        return {}
+
+    min_support = int(os.environ.get("PSQ_DISPLAY_GROUP_WEIGHT_MIN_SUPPORT", "5"))
+    min_delta = float(os.environ.get("PSQ_DISPLAY_GROUP_WEIGHT_MIN_DELTA", "0.005"))
+    out: Dict[str, Tuple[float, float]] = {}
+
+    with csv_path.open("r", encoding="utf-8", newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            group = (row.get("group_name") or "").strip()
+            if not group:
+                continue
+            try:
+                support = int(float(row.get("support_samples") or "0"))
+                delta = float(row.get("delta_best_minus_fasttext") or "0")
+                tfidf_w = float(row.get("best_w_tfidf") or "0")
+                fasttext_w = float(row.get("best_w_fasttext") or "0")
+            except Exception:
+                continue
+            if support < min_support or delta < min_delta:
+                continue
+            out[group] = _normalize_weights(tfidf_w, fasttext_w)
+    return out
+
+
+def _safe_unit_vector(vec: np.ndarray) -> np.ndarray:
+    v = np.asarray(vec, dtype=np.float32).reshape(-1)
+    n = float(np.linalg.norm(v))
+    if n <= 1e-12:
+        return np.zeros_like(v, dtype=np.float32)
+    return (v / n).astype(np.float32)
+
+
+@lru_cache(maxsize=2)
+def _fasttext_tag_matrix(tags: Tuple[str, ...]) -> np.ndarray:
+    ft = get_fasttext_model()
+    rows = [_safe_unit_vector(ft.get_vector(tag)) for tag in tags]
+    if not rows:
+        return np.zeros((0, 0), dtype=np.float32)
+    return np.vstack(rows).astype(np.float32)
+
+
+def _build_fasttext_score_by_tag(tags: Sequence[str], query_terms: Sequence[str]) -> Dict[str, float]:
+    if not tags or not query_terms:
+        return {}
+
+    ft = get_fasttext_model()
+    query_rows = [_safe_unit_vector(ft.get_vector(term)) for term in query_terms]
+    query_rows = [r for r in query_rows if np.any(r)]
+    if not query_rows:
+        return {}
+    query_matrix = np.vstack(query_rows).astype(np.float32)
+
+    tag_matrix = _fasttext_tag_matrix(tuple(tags))
+    if tag_matrix.size == 0:
+        return {}
+    sims = (tag_matrix @ query_matrix.T).astype(np.float32)
+    best = np.max(sims, axis=1)
+    return {tag: float(score) for tag, score in zip(tags, best)}
+
+
 @lru_cache(maxsize=1)
 def _load_enabled_groups() -> Dict[str, List[str]]:
     csv_path = Path("data/analysis/category_registry.csv")
@@ -83,6 +170,22 @@ def _calibrate_probabilities(scores: Dict[str, float]) -> Dict[str, float]:
     return probs
 
 
+def _blend_prob(
+    tag: str,
+    tfidf_w: float,
+    fasttext_w: float,
+    prob_by_tag_tfidf: Dict[str, float],
+    prob_by_tag_fasttext: Dict[str, float],
+) -> float:
+    p_tfidf = prob_by_tag_tfidf.get(tag)
+    p_fasttext = prob_by_tag_fasttext.get(tag)
+    if tfidf_w <= 0.0 or p_tfidf is None:
+        return float(p_fasttext or 0.0)
+    if fasttext_w <= 0.0 or p_fasttext is None:
+        return float(p_tfidf or 0.0)
+    return float(tfidf_w * p_tfidf + fasttext_w * p_fasttext)
+
+
 def rank_groups_from_tfidf(
     seed_terms: Sequence[str],
     *,
@@ -94,55 +197,87 @@ def rank_groups_from_tfidf(
     if not groups:
         return []
 
-    components = get_tfidf_components()
-    tag_vectors = get_tfidf_tag_vectors()
-    idf = components["idf"]
-    term_to_col = components["tag_to_column_index"]
-    svd = components["svd_model"]
-    tag_to_row = tag_vectors["tag_to_row_index"]
-    mat_norm = tag_vectors["reduced_matrix_norm"]
-
-    pseudo_doc: Dict[str, float] = {}
-    for term in seed_terms:
-        key = _norm_tag_for_lookup(str(term))
-        if key in term_to_col:
-            pseudo_doc[key] = pseudo_doc.get(key, 0.0) + 1.0
-    if not pseudo_doc:
-        return []
-
-    pseudo_vec = construct_pseudo_vector(pseudo_doc, idf, term_to_col)
-    q = svd.transform(pseudo_vec).reshape(-1).astype(np.float32)
-    qn = float(np.linalg.norm(q))
-    if qn <= 0.0:
-        return []
-    q = q / qn
+    default_tfidf_w, default_fasttext_w = _resolve_display_weights()
+    per_group_weights = _load_group_weight_overrides()
+    use_tfidf = default_tfidf_w > 0.0 or any(w[0] > 0.0 for w in per_group_weights.values())
+    use_fasttext = default_fasttext_w > 0.0 or any(w[1] > 0.0 for w in per_group_weights.values())
 
     all_tags: List[str] = []
     for tags in groups.values():
         all_tags.extend(tags)
     all_tags = list(dict.fromkeys(all_tags))
 
-    scored_tags: List[str] = []
-    rows: List[int] = []
-    for tag in all_tags:
-        idx = tag_to_row.get(tag)
-        if idx is None:
-            continue
-        scored_tags.append(tag)
-        rows.append(int(idx))
-    if not rows:
+    query_terms = [_norm_tag_for_lookup(str(t)) for t in seed_terms if str(t).strip()]
+    query_terms = list(dict.fromkeys(query_terms))
+    if not query_terms:
         return []
 
-    sims = (mat_norm[np.asarray(rows, dtype=np.int32)] @ q).astype(np.float32)
-    score_by_tag: Dict[str, float] = {t: float(s) for t, s in zip(scored_tags, sims)}
-    prob_by_tag = _calibrate_probabilities(score_by_tag)
+    scored_tags = list(all_tags)
+
+    prob_by_tag_tfidf: Dict[str, float] = {}
+    if use_tfidf:
+        components = get_tfidf_components()
+        tag_vectors = get_tfidf_tag_vectors()
+        idf = components["idf"]
+        term_to_col = components["tag_to_column_index"]
+        svd = components["svd_model"]
+        tag_to_row = tag_vectors["tag_to_row_index"]
+        mat_norm = tag_vectors["reduced_matrix_norm"]
+
+        pseudo_doc: Dict[str, float] = {}
+        for term in query_terms:
+            if term in term_to_col:
+                pseudo_doc[term] = pseudo_doc.get(term, 0.0) + 1.0
+
+        if pseudo_doc:
+            tfidf_tags: List[str] = []
+            tfidf_rows: List[int] = []
+            for tag in all_tags:
+                idx = tag_to_row.get(tag)
+                if idx is None:
+                    continue
+                tfidf_tags.append(tag)
+                tfidf_rows.append(int(idx))
+
+            if tfidf_rows:
+                pseudo_vec = construct_pseudo_vector(pseudo_doc, idf, term_to_col)
+                q = svd.transform(pseudo_vec).reshape(-1).astype(np.float32)
+                qn = float(np.linalg.norm(q))
+                if qn > 0.0:
+                    q = q / qn
+                    sims = (mat_norm[np.asarray(tfidf_rows, dtype=np.int32)] @ q).astype(np.float32)
+                    tfidf_score_by_tag: Dict[str, float] = {t: float(s) for t, s in zip(tfidf_tags, sims)}
+                    prob_by_tag_tfidf = _calibrate_probabilities(tfidf_score_by_tag)
+
+    prob_by_tag_fasttext: Dict[str, float] = {}
+    if use_fasttext:
+        fasttext_score_by_tag = _build_fasttext_score_by_tag(scored_tags, query_terms)
+        if fasttext_score_by_tag:
+            prob_by_tag_fasttext = _calibrate_probabilities(fasttext_score_by_tag)
+
+    if not prob_by_tag_tfidf and not prob_by_tag_fasttext:
+        return []
 
     rows_out: List[GroupRankingRow] = []
     rank_k = max(1, int(group_rank_top_k))
     display_k = max(1, int(top_tags_per_group))
 
     for group_name, tags in groups.items():
-        scored = [(t, prob_by_tag[t]) for t in tags if t in prob_by_tag]
+        group_tfidf_w, group_fasttext_w = per_group_weights.get(
+            group_name,
+            (default_tfidf_w, default_fasttext_w),
+        )
+        scored: List[Tuple[str, float]] = []
+        for t in tags:
+            p = _blend_prob(
+                tag=t,
+                tfidf_w=group_tfidf_w,
+                fasttext_w=group_fasttext_w,
+                prob_by_tag_tfidf=prob_by_tag_tfidf,
+                prob_by_tag_fasttext=prob_by_tag_fasttext,
+            )
+            if p > 0.0:
+                scored.append((t, p))
         if not scored:
             continue
         scored.sort(key=lambda x: x[1], reverse=True)