Spaces:
Running
Running
File size: 2,906 Bytes
e6404d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | from difflib import SequenceMatcher
from src.synonym_data import _load_groups, _find_synonym_group
def _dedup_exact(tags: list[str]) -> list[str]:
seen = set()
out = []
for t in tags:
k = t.lower().strip()
if k and k not in seen:
seen.add(k)
out.append(t)
return out
def _dedup_synonym_groups(tags: list[str]) -> list[str]:
result = []
for t in tags:
tl = t.lower().strip()
group = _find_synonym_group(t)
if group is None:
result.append(t)
continue
conflict = False
for existing in result:
el = existing.lower().strip()
if el in group and el != tl:
conflict = True
break
if not conflict:
result.append(t)
return result
def _ngram_similarity(a: str, b: str) -> float:
return SequenceMatcher(None, a.lower(), b.lower()).ratio()
def _is_attribute_variant(a: str, b: str) -> bool:
"""Two tags that share a head or a tail but differ in the other part denote
different attributes (e.g. 'blue eyes' vs 'blue hair', 'red dress' vs 'blue
dress') and must both be kept — fuzzy-merging them would drop a distinct
booru attribute."""
aw = a.lower().split()
bw = b.lower().split()
if len(aw) < 2 or len(bw) < 2 or len(aw) != len(bw):
return False
head_same = aw[:-1] == bw[:-1]
tail_same = aw[-1] == bw[-1]
# exactly one side differs -> different attribute, same concept
return head_same != tail_same
def _dedup_fuzzy(tags: list[str], threshold: float = 0.85) -> list[str]:
# Sort by length so a more specific (longer) tag survives its shorter fuzzy
# twin (e.g. "very long flowing red hair" beats "long red hair").
sorted_tags = sorted(tags, key=len, reverse=True)
result = []
for t in sorted_tags:
if len(t) < 4:
result.append(t)
continue
is_dup = False
for existing in result:
if len(existing) < 4:
continue
if _is_attribute_variant(t, existing):
continue
# Cheap upper bound first — a length mismatch alone can decide.
ratio = SequenceMatcher(None, t.lower(), existing.lower()).ratio()
if ratio >= threshold:
is_dup = True
break
if not is_dup:
result.append(t)
return result
def smart_dedup(tags: list[str], model: str = "anima") -> list[str]:
if not tags:
return []
no_exact = _dedup_exact(tags)
no_synonym = _dedup_synonym_groups(no_exact)
threshold = 0.75 if model == "anima" else 0.85
no_fuzzy = _dedup_fuzzy(no_synonym, threshold=threshold)
return no_fuzzy
def reload_groups():
from src.synonym_data import reload_synonym_groups
reload_synonym_groups()
_load_groups()
|