File size: 4,149 Bytes
7ebb6d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | """Build per-character file index from English_JSON.
For each character in Contacts.json, produces character_data/{id}.json with:
mentions - direct conversation/chat files (filename matches character)
references - other files where the character's name appears in content
"""
import json
import os
import re
from pathlib import Path
ROOT = Path(__file__).parent
JSON_DIR = ROOT / "English_JSON"
OUT_DIR = ROOT / "character_data"
def _normalise_names(info: dict) -> list[str]:
names = info.get("name", [])
if isinstance(names, str):
names = [names] if names else []
detective = info.get("detective_name") or ""
if detective:
names = list(names) + [detective]
return [n for n in names if n]
def load_characters() -> dict:
"""Merge all Contacts.json files into {id: {name, search_patterns, file_ids}}."""
chars: dict = {}
for contacts_path in sorted(JSON_DIR.rglob("Contacts.json")):
data = json.loads(contacts_path.read_text(encoding="utf-8"))
for char_id, info in data.items():
if char_id in chars:
continue
names = _normalise_names(info)
chars[char_id] = {
"name": names,
# Patterns for content search (word-boundary for plain words, substring otherwise)
"search_patterns": [_make_pattern(n) for n in names],
# Tokens to look for in filenames (ID + name variants, all lowercase)
"file_ids": {char_id.lower()} | {n.lower() for n in names},
}
return chars
def _make_pattern(name: str) -> re.Pattern:
if re.match(r"^[A-Za-z]+$", name):
return re.compile(r"\b" + re.escape(name) + r"\b", re.IGNORECASE)
return re.compile(re.escape(name), re.IGNORECASE)
def _is_chat_file(path: Path) -> bool:
return any(p in ("Conversations", "Filler Chats") for p in path.parts)
def _filename_matches(stem: str, file_ids: set[str]) -> bool:
stem_lower = stem.lower()
# Try each identifier: exact match, ends-with, or space-separated token
tokens = set(re.split(r"[^a-z0-9]+", stem_lower))
for fid in file_ids:
if fid in tokens or stem_lower == fid or stem_lower.endswith(fid):
return True
return False
def _flatten_strings(obj) -> list[str]:
if isinstance(obj, str):
return [obj]
if isinstance(obj, list):
out = []
for item in obj:
out.extend(_flatten_strings(item))
return out
if isinstance(obj, dict):
out = []
for k, v in obj.items():
out.append(k)
out.extend(_flatten_strings(v))
return out
return []
def _content_matches(text: str, patterns: list[re.Pattern]) -> bool:
return any(p.search(text) for p in patterns)
def main() -> None:
OUT_DIR.mkdir(exist_ok=True)
chars = load_characters()
results = {
char_id: {"id": char_id, "name": info["name"], "mentions": [], "references": []}
for char_id, info in chars.items()
}
for json_file in sorted(JSON_DIR.rglob("*.json")):
rel_path = str(json_file.relative_to(ROOT))
is_chat = _is_chat_file(json_file)
stem = json_file.stem
try:
data = json.loads(json_file.read_text(encoding="utf-8"))
except Exception:
continue
# Flatten content once per file; reuse across all character checks
flat_text = " ".join(_flatten_strings(data))
for char_id, info in chars.items():
if is_chat and _filename_matches(stem, info["file_ids"]):
results[char_id]["mentions"].append(rel_path)
elif info["search_patterns"] and _content_matches(flat_text, info["search_patterns"]):
results[char_id]["references"].append(rel_path)
for char_id, char_data in results.items():
out_path = OUT_DIR / f"{char_id}.json"
out_path.write_text(
json.dumps(char_data, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(f"Written {len(results)} character files to {OUT_DIR}/")
if __name__ == "__main__":
main()
|