Upload 2 files

build_character_data.py

@@ -0,0 +1,126 @@
+"""Build per-character file index from English_JSON.
+
+For each character in Contacts.json, produces character_data/{id}.json with:
+  mentions  - direct conversation/chat files (filename matches character)
+  references - other files where the character's name appears in content
+"""
+
+import json
+import os
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).parent
+JSON_DIR = ROOT / "English_JSON"
+OUT_DIR = ROOT / "character_data"
+
+
+def _normalise_names(info: dict) -> list[str]:
+    names = info.get("name", [])
+    if isinstance(names, str):
+        names = [names] if names else []
+    detective = info.get("detective_name") or ""
+    if detective:
+        names = list(names) + [detective]
+    return [n for n in names if n]
+
+
+def load_characters() -> dict:
+    """Merge all Contacts.json files into {id: {name, search_patterns, file_ids}}."""
+    chars: dict = {}
+    for contacts_path in sorted(JSON_DIR.rglob("Contacts.json")):
+        data = json.loads(contacts_path.read_text(encoding="utf-8"))
+        for char_id, info in data.items():
+            if char_id in chars:
+                continue
+            names = _normalise_names(info)
+            chars[char_id] = {
+                "name": names,
+                # Patterns for content search (word-boundary for plain words, substring otherwise)
+                "search_patterns": [_make_pattern(n) for n in names],
+                # Tokens to look for in filenames (ID + name variants, all lowercase)
+                "file_ids": {char_id.lower()} | {n.lower() for n in names},
+            }
+    return chars
+
+
+def _make_pattern(name: str) -> re.Pattern:
+    if re.match(r"^[A-Za-z]+$", name):
+        return re.compile(r"\b" + re.escape(name) + r"\b", re.IGNORECASE)
+    return re.compile(re.escape(name), re.IGNORECASE)
+
+
+def _is_chat_file(path: Path) -> bool:
+    return any(p in ("Conversations", "Filler Chats") for p in path.parts)
+
+
+def _filename_matches(stem: str, file_ids: set[str]) -> bool:
+    stem_lower = stem.lower()
+    # Try each identifier: exact match, ends-with, or space-separated token
+    tokens = set(re.split(r"[^a-z0-9]+", stem_lower))
+    for fid in file_ids:
+        if fid in tokens or stem_lower == fid or stem_lower.endswith(fid):
+            return True
+    return False
+
+
+def _flatten_strings(obj) -> list[str]:
+    if isinstance(obj, str):
+        return [obj]
+    if isinstance(obj, list):
+        out = []
+        for item in obj:
+            out.extend(_flatten_strings(item))
+        return out
+    if isinstance(obj, dict):
+        out = []
+        for k, v in obj.items():
+            out.append(k)
+            out.extend(_flatten_strings(v))
+        return out
+    return []
+
+
+def _content_matches(text: str, patterns: list[re.Pattern]) -> bool:
+    return any(p.search(text) for p in patterns)
+
+
+def main() -> None:
+    OUT_DIR.mkdir(exist_ok=True)
+    chars = load_characters()
+
+    results = {
+        char_id: {"id": char_id, "name": info["name"], "mentions": [], "references": []}
+        for char_id, info in chars.items()
+    }
+
+    for json_file in sorted(JSON_DIR.rglob("*.json")):
+        rel_path = str(json_file.relative_to(ROOT))
+        is_chat = _is_chat_file(json_file)
+        stem = json_file.stem
+
+        try:
+            data = json.loads(json_file.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+
+        # Flatten content once per file; reuse across all character checks
+        flat_text = " ".join(_flatten_strings(data))
+
+        for char_id, info in chars.items():
+            if is_chat and _filename_matches(stem, info["file_ids"]):
+                results[char_id]["mentions"].append(rel_path)
+            elif info["search_patterns"] and _content_matches(flat_text, info["search_patterns"]):
+                results[char_id]["references"].append(rel_path)
+
+    for char_id, char_data in results.items():
+        out_path = OUT_DIR / f"{char_id}.json"
+        out_path.write_text(
+            json.dumps(char_data, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+
+    print(f"Written {len(results)} character files to {OUT_DIR}/")
+
+
+if __name__ == "__main__":
+    main()

build_character_wikis.py

@@ -0,0 +1,265 @@
+"""Build per-character voice wikis from English_JSON dialogue via a local Ollama model.
+
+For each character indexed in character_data/{id}.json, processes the character's
+"mentions" files one at a time. Each file gets its own stateless LLM call (a fresh
+context window): the model receives the current wiki plus one file and returns the
+updated wiki. The wiki is saved after every file and progress is tracked in
+character_wikis/.progress.json, so an interrupted run resumes where it left off.
+
+Usage:
+    python build_character_wikis.py                  # all characters (resumes)
+    python build_character_wikis.py --character brad # single character
+    python build_character_wikis.py --force          # rebuild ignoring progress
+"""
+
+import argparse
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).parent
+CHAR_DIR = ROOT / "character_data"
+OUT_DIR = ROOT / "character_wikis"
+PROGRESS_PATH = OUT_DIR / ".progress.json"
+
+OLLAMA_URL = "http://localhost:11434/api/chat"
+MODEL = "gemma4:12b-mlx"
+# Gemma's recommended sampling parameters. 32k ctx comfortably fits the largest
+# mention file (~13k tokens incl. prompt + wiki) without over-allocating KV cache.
+MODEL_OPTIONS = {"temperature": 1, "top_k": 64, "top_p": 0.95, "num_ctx": 32768}
+REQUEST_TIMEOUT_S = 1800
+MAX_ATTEMPTS = 3
+RETRY_BACKOFF_S = 5
+
+SYSTEM_PROMPT = """\
+You are building a character voice wiki for {name}{aka}, a character in "Riverstone", \
+a narrative mystery mobile game told through phone chats and calls. Purpose of the \
+wiki: a translator LLM will later receive a translated line of {name}'s dialogue plus \
+this wiki, and adjust the translation to sound like {name} rather than a generic, \
+robotic rendering. Record only what serves that goal.
+
+You are given the current wiki and one game file. Update the wiki with any NEW \
+evidence the file provides about:
+- Tone & style: formal/casual, warmth, sarcasm, energy, politeness.
+- Speech patterns: slang, catchphrases, filler words, abbreviations, \
+emoji/punctuation habits, typical message length, grammar quirks.
+- Personality & relationships, insofar as they shape how {name} speaks and to whom.
+- Example lines: a few short verbatim quotes that best capture the voice.
+
+Rules:
+- Use only evidence from the file. Never invent details.
+- Strings starting with "system::" are engine keys, and "#0"-style keys, "next" and \
+"options_next" values are navigation IDs - not dialogue. Ignore them.
+- If the file adds nothing new about {name}, return the wiki unchanged.
+- Keep the wiki under 400 words, in markdown, under exactly these headings:
+  # {name}
+  ## Role & Relationships
+  ## Personality
+  ## Tone & Style
+  ## Speech Patterns
+  ## Example Lines
+- Output ONLY the complete updated wiki. No commentary before or after.
+"""
+
+USER_TEMPLATE = """\
+CURRENT WIKI:
+{wiki}
+
+FILE: {path}
+{hint}
+
+FILE CONTENT:
+{content}
+"""
+
+EMPTY_WIKI_PLACEHOLDER = "(empty - this is the first file)"
+
+
+def attribution_hint(rel_path: str, name: str) -> str:
+    """Tell the model who speaks which fields, based on the file's location."""
+    parts = Path(rel_path).parts
+    if "Adam_s Phone" in parts:
+        owner = "Detective Adam (the player)"
+    elif "Zoey_s Phone" in parts:
+        owner = "Zoey"
+    else:
+        owner = "the player"
+
+    if "Conversations" in parts:
+        return (
+            f'In this branching dialogue, "message" values are spoken by {name}; '
+            f'"options" values are reply choices spoken by {owner}, the phone\'s '
+            f"owner - not {name}."
+        )
+    if "Filler Chats" in parts:
+        return (
+            f"This is a chat log on {owner}'s phone between {owner} and {name} "
+            f'(group chats may include others). Different "type" values mark '
+            f"different speakers; attribute a line to {name} only when context "
+            f"makes the speaker clear."
+        )
+    return (
+        f"Extract only what this file reveals about {name}; do not attribute "
+        f"other speakers' lines to them."
+    )
+
+
+def call_ollama(system_prompt: str, user_msg: str) -> str:
+    payload = json.dumps(
+        {
+            "model": MODEL,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_msg},
+            ],
+            "stream": False,
+            # gemma4 has thinking enabled by default, which multiplies generation
+            # time several-fold per call; the wiki task doesn't need it.
+            "think": False,
+            "options": MODEL_OPTIONS,
+        }
+    ).encode("utf-8")
+
+    last_error: Exception | None = None
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        request = urllib.request.Request(
+            OLLAMA_URL, data=payload, headers={"Content-Type": "application/json"}
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_S) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+            return data["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, KeyError, json.JSONDecodeError) as exc:
+            last_error = exc
+            if attempt < MAX_ATTEMPTS:
+                print(f"    request failed ({exc}); retrying in {RETRY_BACKOFF_S}s...")
+                time.sleep(RETRY_BACKOFF_S)
+    raise RuntimeError(
+        f"Ollama request failed after {MAX_ATTEMPTS} attempts: {last_error}. "
+        f"Is the server running at {OLLAMA_URL}? Progress is saved; rerun to resume."
+    )
+
+
+def clean_reply(text: str) -> str:
+    """Strip whitespace and an optional wrapping markdown code fence."""
+    text = text.strip()
+    if text.startswith("```") and text.endswith("```"):
+        first_newline = text.find("\n")
+        if first_newline != -1:
+            text = text[first_newline + 1 : -3].strip()
+    return text
+
+
+def load_progress() -> dict:
+    if PROGRESS_PATH.exists():
+        return json.loads(PROGRESS_PATH.read_text(encoding="utf-8"))
+    return {}
+
+
+def save_progress(progress: dict) -> None:
+    PROGRESS_PATH.write_text(
+        json.dumps(progress, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+
+
+def process_character(char_data: dict, progress: dict, force: bool) -> dict:
+    """Run the per-file wiki-update loop for one character; returns new progress."""
+    char_id = char_data["id"]
+    names = char_data.get("name") or []
+    display_name = names[0] if names else char_id
+    aka = f' (also appearing as {", ".join(names[1:])})' if len(names) > 1 else ""
+    mentions = char_data.get("mentions", [])
+
+    if not mentions:
+        print(f"[{char_id}] no mention files - skipped")
+        return progress
+
+    wiki_path = OUT_DIR / f"{char_id}.md"
+    done = [] if force else list(progress.get(char_id, []))
+    wiki = ""
+    if not force and wiki_path.exists():
+        wiki = wiki_path.read_text(encoding="utf-8").strip()
+
+    pending = [m for m in mentions if m not in done]
+    if not pending:
+        print(f"[{char_id}] already complete ({len(mentions)} files)")
+        return progress
+
+    system_prompt = SYSTEM_PROMPT.format(name=display_name, aka=aka)
+    print(f"[{char_id}] {len(pending)} file(s) to process")
+
+    for i, rel_path in enumerate(pending, 1):
+        file_path = ROOT / rel_path
+        try:
+            data = json.loads(file_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            print(f"  ({i}/{len(pending)}) WARNING: cannot read {rel_path}: {exc}")
+            done = done + [rel_path]
+            progress = {**progress, char_id: done}
+            save_progress(progress)
+            continue
+
+        user_msg = USER_TEMPLATE.format(
+            wiki=wiki or EMPTY_WIKI_PLACEHOLDER,
+            path=rel_path.removeprefix("English_JSON/"),
+            hint=attribution_hint(rel_path, display_name),
+            content=json.dumps(data, indent=2, ensure_ascii=False),
+        )
+
+        started = time.monotonic()
+        reply = clean_reply(call_ollama(system_prompt, user_msg))
+        elapsed = time.monotonic() - started
+
+        if reply.startswith("#"):
+            wiki = reply
+            wiki_path.write_text(wiki + "\n", encoding="utf-8")
+            print(f"  ({i}/{len(pending)}) {rel_path} - updated ({elapsed:.0f}s)")
+        else:
+            print(
+                f"  ({i}/{len(pending)}) {rel_path} - WARNING: malformed reply, "
+                f"keeping previous wiki ({elapsed:.0f}s)"
+            )
+
+        done = done + [rel_path]
+        progress = {**progress, char_id: done}
+        save_progress(progress)
+
+    return progress
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    parser.add_argument("--character", help="process a single character id (e.g. brad)")
+    parser.add_argument(
+        "--force", action="store_true", help="rebuild from scratch, ignoring saved progress"
+    )
+    args = parser.parse_args()
+    sys.stdout.reconfigure(line_buffering=True)
+
+    char_files = sorted(CHAR_DIR.glob("*.json"))
+    if args.character:
+        char_files = [CHAR_DIR / f"{args.character}.json"]
+        if not char_files[0].exists():
+            sys.exit(f"No such character: {args.character} (expected {char_files[0]})")
+    if not char_files:
+        sys.exit(f"No character files found in {CHAR_DIR}; run build_character_data.py first.")
+
+    OUT_DIR.mkdir(exist_ok=True)
+    progress = load_progress()
+
+    try:
+        for char_file in char_files:
+            char_data = json.loads(char_file.read_text(encoding="utf-8"))
+            progress = process_character(char_data, progress, args.force)
+    except KeyboardInterrupt:
+        print("\nInterrupted - progress saved; rerun to resume.")
+        sys.exit(130)
+
+    print(f"Done. Wikis written to {OUT_DIR}/")
+
+
+if __name__ == "__main__":
+    main()