psq_rag/pipeline/preproc.py

import re

def extract_user_provided_tags_upto_3_words(prompt_in: str) -> list[str]:
    """
    Heuristic:
    - split on '.' and ','
    - strip leading/trailing whitespace
    - split on whitespace
    - keep items with <= 3 tokens
    """
    if not prompt_in:
        return []

    parts = re.split(r"[.,]+", prompt_in)

    out: list[str] = []
    seen = set()

    for raw in parts:
        item = raw.strip()
        if not item:
            continue

        tokens = item.split()
        if len(tokens) <= 3:
            key = item.lower()
            if key not in seen:
                seen.add(key)
                out.append(item)

    return out


if __name__ == "__main__":
    print("preproc.py imports ok")