Upload folder using huggingface_hub

main/pipeline/build_file_context.py

@@ -0,0 +1,67 @@
+"""Parse FILE_MAPPING.md into file_context.json.
+
+FILE_MAPPING.md (teammate-generated) has one "### N. filename" section per
+source file with bullet fields. We extract per file: content label, inferred
+role, and detailed summary, keyed by the file's path inside the JSON pack
+(e.g. "Initial/Gameplay/Adam_s Phone/Messages/General.json").
+
+Usage:
+    python -m pipeline.build_file_context
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+import config
+
+_SECTION_RE = re.compile(r"^### \d+\. `", re.MULTILINE)
+_FIELD_RES = {
+    "rel_path": re.compile(r"^- Relative path: `(.+?)`", re.MULTILINE),
+    "label": re.compile(r"^- Content label: `(.+?)`", re.MULTILINE),
+    "role": re.compile(r"^- Inferred role: (.+?)$", re.MULTILINE),
+    "summary": re.compile(r"^- Detailed summary: (.+?)$", re.MULTILINE),
+}
+
+
+def normalize_key(rel_path: str) -> str:
+    """Canonical lookup key: strip stray spaces around each path component."""
+    return "/".join(part.strip() for part in rel_path.replace("\\", "/").split("/"))
+
+
+def _pack_key(rel_path: str) -> str:
+    """'English/Initial/.../General.docx' -> 'Initial/.../General.json'."""
+    path = rel_path.replace("\\", "/").removeprefix("English/")
+    return normalize_key(str(Path(path).with_suffix(".json")))
+
+
+def parse_file_mapping(markdown: str) -> dict[str, dict[str, str]]:
+    context: dict[str, dict[str, str]] = {}
+    for section in _SECTION_RE.split(markdown)[1:]:
+        fields = {}
+        for name, pattern in _FIELD_RES.items():
+            match = pattern.search(section)
+            fields[name] = match.group(1).strip() if match else ""
+        if not fields["rel_path"]:
+            continue
+        context[_pack_key(fields["rel_path"])] = {
+            "label": fields["label"],
+            "role": fields["role"],
+            "summary": fields["summary"],
+        }
+    return context
+
+
+def main() -> None:
+    markdown = config.FILE_MAPPING_PATH.read_text(encoding="utf-8")
+    context = parse_file_mapping(markdown)
+    config.FILE_CONTEXT_PATH.write_text(
+        json.dumps(context, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+    print(f"Wrote context for {len(context)} files to {config.FILE_CONTEXT_PATH.name}")
+
+
+if __name__ == "__main__":
+    main()

main/pipeline/clients.py

@@ -0,0 +1,414 @@
+"""In-process Hugging Face clients for the two translation models.
+
+Stage 1 - `translate`: google/translategemma-12b-it via its official
+TranslateGemma chat template.
+
+Stage 2 - `adjust_tone`: google/gemma-4-12B-it via the normal Gemma 4 chat
+template, rewriting a draft translation in a character's voice using their
+wiki.
+"""
+
+from __future__ import annotations
+
+import gc
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+import config
+
+try:
+    import spaces
+except ImportError:  # Local/dev installs do not need the Spaces runtime.
+    spaces = None
+
+
+# Runtime-injected tokens like "Person1" that must survive translation verbatim.
+_PLACEHOLDER_RE = re.compile(r"\b([A-Za-z]+)(\d+)\b")
+
+TONE_SYSTEM_TEMPLATE = """\
+You are a localization editor for "Riverstone", a narrative mystery mobile game \
+told through phone chats. Below is the voice wiki for {name}, a game character.
+
+{wiki}
+
+You will receive one of {name}'s chat lines in English and a draft {language} \
+translation. Rewrite the {language} draft so it reads like {name} texting in \
+{language}: match the wiki's tone, register, slang level, emoji and punctuation \
+habits.
+
+Rules:
+- Keep the exact meaning of the English line; never add or drop information.
+- Keep placeholder tokens (e.g. Person1), proper names, emoji and special \
+symbols exactly as written.
+- Use the formality level the wiki implies for {name} (casual characters use \
+informal address).
+- If the draft already sounds right, return it unchanged.
+- Output ONLY the final {language} line - no quotes, no commentary.
+"""
+
+TONE_USER_TEMPLATE = """\
+{context}English line: {source}
+Draft {language} translation: {draft}
+Final {language} line:"""
+
+WIKI_MAX_NEW_TOKENS = 1200
+
+
+@dataclass
+class _LoadedModel:
+    processor: Any
+    model: Any
+
+
+_MODELS: dict[str, _LoadedModel] = {}
+_MODEL_IDS = {
+    "translate": config.TRANSLATE_MODEL_ID,
+    "tone": config.TONE_MODEL_ID,
+}
+
+
+def _gpu(fn):
+    if spaces is None:
+        return fn
+    return spaces.GPU(
+        duration=config.ZERO_GPU_DURATION_S,
+        size=config.ZERO_GPU_SIZE,
+    )(fn)
+
+
+def _torch():
+    try:
+        import torch
+    except ImportError as exc:
+        raise RuntimeError(
+            "Hugging Face inference requires torch. Install the Space/runtime "
+            "dependencies from requirements.txt."
+        ) from exc
+    return torch
+
+
+def _hf_classes():
+    try:
+        from transformers import AutoModelForMultimodalLM, AutoProcessor
+
+        return AutoProcessor, AutoModelForMultimodalLM
+    except ImportError:
+        try:
+            from transformers import AutoModelForImageTextToText, AutoProcessor
+
+            return AutoProcessor, AutoModelForImageTextToText
+        except ImportError as exc:
+            raise RuntimeError(
+                "Hugging Face inference requires a recent transformers release "
+                "with Gemma multimodal model support."
+            ) from exc
+
+
+def _from_pretrained(model_id: str):
+    AutoProcessor, AutoModel = _hf_classes()
+    kwargs: dict[str, Any] = {}
+    if config.HF_DEVICE_MAP is not None:
+        kwargs["device_map"] = config.HF_DEVICE_MAP
+    if config.HF_DTYPE is not None:
+        kwargs["dtype"] = config.HF_DTYPE
+    if config.HF_ATTN_IMPLEMENTATION:
+        kwargs["attn_implementation"] = config.HF_ATTN_IMPLEMENTATION
+
+    processor = AutoProcessor.from_pretrained(model_id)
+    try:
+        model = AutoModel.from_pretrained(model_id, **kwargs)
+    except TypeError:
+        # Older Transformers used torch_dtype instead of dtype.
+        if "dtype" in kwargs:
+            kwargs["torch_dtype"] = kwargs.pop("dtype")
+        model = AutoModel.from_pretrained(model_id, **kwargs)
+    model.eval()
+    return _LoadedModel(processor=processor, model=model)
+
+
+def release_models(except_key: str | None = None) -> None:
+    """Free loaded HF models, optionally keeping one active model resident."""
+    for key in list(_MODELS):
+        if key != except_key:
+            del _MODELS[key]
+    gc.collect()
+    try:
+        torch = _torch()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except RuntimeError:
+        pass
+
+
+def warm_models(*keys: str) -> None:
+    """Preload selected models, useful from a Space module at startup."""
+    for key in keys:
+        _get_model(key)
+
+
+def _get_model(key: str) -> _LoadedModel:
+    if key not in _MODEL_IDS:
+        raise ValueError(f"Unknown model key: {key}")
+    if key not in _MODELS:
+        if not config.HF_KEEP_BOTH_MODELS:
+            release_models(except_key=key)
+        _MODELS[key] = _from_pretrained(_MODEL_IDS[key])
+    return _MODELS[key]
+
+
+def _model_device(model):
+    device = getattr(model, "device", None)
+    if device is not None:
+        return device
+    try:
+        return next(model.parameters()).device
+    except StopIteration:
+        return None
+
+
+def _model_dtype(model):
+    try:
+        for parameter in model.parameters():
+            if parameter.is_floating_point():
+                return parameter.dtype
+    except StopIteration:
+        return None
+    return None
+
+
+def _move_inputs(inputs, model):
+    device = _model_device(model)
+    dtype = _model_dtype(model)
+    if device is None:
+        return inputs
+    if dtype is not None:
+        try:
+            return inputs.to(device, dtype=dtype)
+        except TypeError:
+            pass
+    return inputs.to(device)
+
+
+def _apply_chat_template(processor, messages, *, enable_thinking: bool = False):
+    kwargs = {
+        "tokenize": True,
+        "add_generation_prompt": True,
+        "return_dict": True,
+        "return_tensors": "pt",
+    }
+    try:
+        return processor.apply_chat_template(
+            messages,
+            enable_thinking=enable_thinking,
+            **kwargs,
+        )
+    except TypeError:
+        return processor.apply_chat_template(messages, **kwargs)
+
+
+def _generate_text(
+    key: str,
+    messages: list[dict],
+    *,
+    max_new_tokens: int,
+    enable_thinking: bool = False,
+    **generation_kwargs,
+) -> str:
+    loaded = _get_model(key)
+    torch = _torch()
+    inputs = _apply_chat_template(
+        loaded.processor,
+        messages,
+        enable_thinking=enable_thinking,
+    )
+    inputs = _move_inputs(inputs, loaded.model)
+    input_len = inputs["input_ids"].shape[-1]
+
+    with torch.inference_mode():
+        output = loaded.model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            **generation_kwargs,
+        )
+
+    generated = output[0][input_len:]
+    decoded = loaded.processor.decode(generated, skip_special_tokens=False)
+    return _parse_response(loaded.processor, decoded)
+
+
+def _parse_response(processor, decoded: str) -> str:
+    if hasattr(processor, "parse_response"):
+        try:
+            parsed = processor.parse_response(decoded)
+            if isinstance(parsed, str):
+                return parsed.strip()
+            if isinstance(parsed, dict):
+                for key in ("content", "text", "response", "answer"):
+                    value = parsed.get(key)
+                    if isinstance(value, str):
+                        return value.strip()
+        except Exception:
+            pass
+
+    text = decoded
+    text = re.sub(r"<\|channel\>thought.*?<channel\|>", "", text, flags=re.DOTALL)
+    text = re.sub(r"<[^>]+>", "", text)
+    return text.strip()
+
+
+def restore_placeholders(source: str, translated: str) -> str:
+    """Undo model damage to PersonN-style tokens (e.g. 'Person 1' -> 'Person1')."""
+    result = translated
+    for word, num in _PLACEHOLDER_RE.findall(source):
+        token = f"{word}{num}"
+        if token in result:
+            continue
+        spaced = re.compile(rf"\b{re.escape(word)}\s+{num}\b", re.IGNORECASE)
+        result = spaced.sub(token, result)
+    return result
+
+
+def _translate_text(text: str) -> str:
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "source_lang_code": config.SOURCE_LANG_CODE,
+                    "target_lang_code": config.TARGET_LANG_CODE,
+                    "text": text,
+                }
+            ],
+        }
+    ]
+    reply = _generate_text(
+        "translate",
+        messages,
+        max_new_tokens=config.TRANSLATE_MAX_NEW_TOKENS,
+        do_sample=False,
+    )
+    return restore_placeholders(text, reply)
+
+
+def _adjust_tone_text(
+    source: str,
+    draft: str,
+    wiki: str,
+    char_name: str,
+    context_lines: list[str] | None = None,
+) -> str:
+    """Rewrite a draft translation in the character's voice via Gemma 4."""
+    context = ""
+    if context_lines:
+        joined = "\n".join(f"  {line}" for line in context_lines)
+        context = f"Preceding lines in this chat (English, for context only):\n{joined}\n\n"
+    messages = [
+        {
+            "role": "system",
+            "content": TONE_SYSTEM_TEMPLATE.format(
+                name=char_name, wiki=wiki, language=config.TARGET_LANG_NAME
+            ),
+        },
+        {
+            "role": "user",
+            "content": TONE_USER_TEMPLATE.format(
+                context=context,
+                source=source,
+                draft=draft,
+                language=config.TARGET_LANG_NAME,
+            ),
+        },
+    ]
+    reply = _generate_text(
+        "tone",
+        messages,
+        max_new_tokens=config.TONE_MAX_NEW_TOKENS,
+        enable_thinking=False,
+        **config.TONE_GENERATION_KWARGS,
+    )
+    reply = reply.strip().strip('"').strip()
+    return restore_placeholders(source, reply) if reply else draft
+
+
+@_gpu
+def translate(text: str) -> str:
+    """Translate one string with TranslateGemma (greedy)."""
+    return _translate_text(text)
+
+
+@_gpu
+def adjust_tone(
+    source: str,
+    draft: str,
+    wiki: str,
+    char_name: str,
+    context_lines: list[str] | None = None,
+) -> str:
+    """Rewrite a draft translation in the character's voice via Gemma 4."""
+    return _adjust_tone_text(source, draft, wiki, char_name, context_lines)
+
+
+@_gpu
+def translate_and_tone_items(
+    items: list[dict],
+    cache: dict,
+    wiki: str | None,
+    char_name: str | None,
+    context_window: int = 2,
+) -> tuple[list[dict], dict, dict]:
+    """Translate and tone a full review record in one GPU allocation."""
+    cache = dict(cache)
+    translated = 0
+    toned = 0
+
+    for item in items:
+        if item.get("mt") is not None:
+            continue
+        source = item["source"]
+        cached = cache.get(source)
+        if cached is not None:
+            item["mt"] = cached
+        else:
+            item["mt"] = _translate_text(source)
+            cache[source] = item["mt"]
+            translated += 1
+
+    if wiki and char_name:
+        sources = [item["source"] for item in items]
+        for i, item in enumerate(items):
+            if item.get("kind") != "dialogue" or item.get("toned") is not None:
+                continue
+            if item.get("mt") is None:
+                continue
+            context = sources[max(0, i - context_window) : i]
+            item["toned"] = _adjust_tone_text(
+                item["source"], item["mt"], wiki, char_name, context
+            )
+            toned += 1
+
+    return items, cache, {"translated": translated, "toned": toned}
+
+
+@_gpu
+def update_character_wiki(system_prompt: str, user_msg: str) -> str:
+    """Update a character wiki via the same in-process Gemma runtime as tone pass."""
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_msg},
+    ]
+    return _generate_text(
+        "tone",
+        messages,
+        max_new_tokens=WIKI_MAX_NEW_TOKENS,
+        enable_thinking=False,
+        **config.TONE_GENERATION_KWARGS,
+    ).strip()
+
+
+if getattr(config, "HF_PRELOAD_MODELS", ()):
+    warm_models(*config.HF_PRELOAD_MODELS)

main/pipeline/dialogue_map.py

@@ -0,0 +1,68 @@
+"""Map chat files to the character (or group) whose lines they contain.
+
+Built from character_data/{id}.json "mentions" (produced by
+build_character_data.py), keeping only characters that have a voice wiki in
+character_wikis/. When several characters claim the same file (group chats,
+flashback variants), the character whose id matches the filename stem wins,
+then the longest id as tie-break.
+
+Speaker attribution within a mapped file (see build_character_wikis.py):
+- Conversations: "message" = the mapped character, "options" = player/owner.
+- Filler Chats: type "1" = player/owner, type "-1" = system, anything else =
+  the mapped character/group.
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+import config
+
+
+def _stem_key(rel_path: str) -> str:
+    """Normalised filename stem: 'Flashback 1 Ralph.json' -> 'flashback1ralph'."""
+    return re.sub(r"[^a-z0-9]", "", Path(rel_path).stem.lower())
+
+
+def build_dialogue_map() -> dict[str, str]:
+    """Return {rel_path inside the source pack: character_id}."""
+    claims: dict[str, list[str]] = {}
+    for char_file in sorted(config.CHAR_DATA_DIR.glob("*.json")):
+        char_id = char_file.stem
+        if not (config.CHAR_WIKI_DIR / f"{char_id}.md").exists():
+            continue
+        data = json.loads(char_file.read_text(encoding="utf-8"))
+        for mention in data.get("mentions", []):
+            rel = mention.removeprefix("English_JSON/")
+            claims.setdefault(rel, []).append(char_id)
+
+    mapping: dict[str, str] = {}
+    for rel, char_ids in claims.items():
+        stem = _stem_key(rel)
+        exact = [c for c in char_ids if stem.endswith(c)]
+        candidates = exact or char_ids
+        mapping[rel] = max(candidates, key=len)
+    return mapping
+
+
+def load_wiki(char_id: str) -> str:
+    return (config.CHAR_WIKI_DIR / f"{char_id}.md").read_text(encoding="utf-8").strip()
+
+
+def display_name(char_id: str) -> str:
+    """Character display name from character_data (falls back to the id)."""
+    char_file = config.CHAR_DATA_DIR / f"{char_id}.json"
+    if char_file.exists():
+        names = json.loads(char_file.read_text(encoding="utf-8")).get("name") or []
+        if names:
+            return names[0]
+    return char_id
+
+
+if __name__ == "__main__":
+    dmap = build_dialogue_map()
+    print(f"{len(dmap)} chat files mapped to {len(set(dmap.values()))} characters")
+    for rel, cid in sorted(dmap.items())[:10]:
+        print(f"  {cid:15s} {rel}")

main/pipeline/export_pack.py

@@ -0,0 +1,83 @@
+"""Export the translated language pack (e.g. German_JSON/) from review records.
+
+Every file from the source pack is deep-copied with translations substituted at
+the recorded key paths; structure, IDs, and untranslatable values stay intact.
+Per string the best available text is used: reviewer-final > toned > machine
+translation > English source. A review-status summary is printed at the end.
+
+Usage:
+    python -m pipeline.export_pack
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+import config
+from pipeline.rules import get_at, set_many
+
+
+def best_text(item: dict) -> str | None:
+    return item.get("final") or item.get("toned") or item.get("mt")
+
+
+def export_file(source_data, record: dict) -> tuple[dict, dict]:
+    """Return (translated copy of source_data, status counts)."""
+    counts = {"approved": 0, "edited": 0, "rejected": 0, "pending": 0, "untranslated": 0}
+    updates = []
+    for item in record["items"]:
+        path = tuple(item["path"])
+        text = best_text(item)
+        if text is None:
+            counts["untranslated"] += 1
+            continue
+        if get_at(source_data, path) != item["source"]:
+            raise ValueError(f"source drift at {item['key']}: rerun translate_pack")
+        updates.append((path, text))
+        counts[item["status"]] += 1
+    return set_many(source_data, updates), counts
+
+
+def main() -> None:
+    if not config.TRANSLATIONS_DIR.exists():
+        sys.exit("No review records found - run pipeline/translate_pack.py first.")
+
+    totals = {"approved": 0, "edited": 0, "rejected": 0, "pending": 0, "untranslated": 0}
+    exported = 0
+    for source_file in sorted(config.SOURCE_DIR.rglob("*.json")):
+        rel = str(source_file.relative_to(config.SOURCE_DIR))
+        source_data = json.loads(source_file.read_text(encoding="utf-8"))
+        record_path = config.TRANSLATIONS_DIR / rel
+
+        if record_path.exists():
+            record = json.loads(record_path.read_text(encoding="utf-8"))
+            translated, counts = export_file(source_data, record)
+            for key, n in counts.items():
+                totals[key] += n
+        else:
+            translated = source_data  # nothing translatable or not yet processed
+
+        out_path = config.PACK_DIR / rel
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(
+            json.dumps(translated, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+        exported += 1
+
+    reviewed = totals["approved"] + totals["edited"]
+    translated_n = sum(totals.values()) - totals["untranslated"]
+    print(f"Exported {exported} files to {config.PACK_DIR.name}/")
+    print(
+        f"Strings: {translated_n} translated "
+        f"({totals['approved']} approved, {totals['edited']} edited, "
+        f"{totals['rejected']} rejected*, {totals['pending']} pending*, "
+        f"{totals['untranslated']} still English)."
+    )
+    print("* rejected/pending strings ship with the unreviewed machine translation.")
+    if translated_n:
+        print(f"Human-reviewed: {reviewed / translated_n:.0%}")
+
+
+if __name__ == "__main__":
+    main()

main/pipeline/rules.py

@@ -0,0 +1,147 @@
+"""Translatability rules: which strings in a language-pack JSON file get translated.
+
+The public API is `iter_translatable(data, rel_path)`, which yields `Item`s with a
+key path addressing the string inside the file, the source string, and a `kind`:
+
+  dialogue - a line spoken by the file's character (eligible for the tone pass)
+  player   - a line spoken by the phone owner / player (plain translation)
+  other    - UI copy, web text, notes, etc. (plain translation)
+
+`get_at` / `set_at` address values by the same key paths and are used by the
+exporter to substitute translations without touching structure.
+"""
+
+import copy
+import re
+from collections.abc import Iterator
+from typing import Any, NamedTuple
+
+# Field names that hold structural data, never prose.
+SKIP_FIELDS = {"id", "audio", "next", "options_next", "time", "type"}
+
+# Speaker codes in Filler Chats ("Schema B").
+TYPE_PLAYER = "1"
+TYPE_SYSTEM = "-1"
+
+_LETTER_RE = re.compile(r"[A-Za-z]")
+_URL_OR_FILE_RE = re.compile(r"^[\w.-]+\.[a-z][a-z0-9]{1,5}(/\S*)?$", re.IGNORECASE)
+
+
+class Item(NamedTuple):
+    path: tuple  # key path into the file's JSON structure
+    source: str
+    kind: str  # "dialogue" | "player" | "other"
+
+
+def is_translatable_value(value: Any) -> bool:
+    """Value-level filter applied to every candidate string."""
+    if not isinstance(value, str):
+        return False
+    text = value.strip()
+    if not text or text == "-":
+        return False
+    if text.startswith("system::"):
+        return False
+    if text.startswith(("http://", "https://")) or _URL_OR_FILE_RE.match(text):
+        return False
+    if not _LETTER_RE.search(text):  # numbers, emoji, punctuation only
+        return False
+    return True
+
+
+def detect_schema(data: Any, rel_path: str) -> str:
+    """Classify a file as 'conversation', 'chat_log', or 'generic'."""
+    parts = rel_path.replace("\\", "/").split("/")
+    if "Conversations" in parts and isinstance(data, dict):
+        if any(isinstance(v, dict) and "message" in v for v in data.values()):
+            return "conversation"
+    if "Filler Chats" in parts:
+        rows = data if isinstance(data, list) else list(data.values()) if isinstance(data, dict) else []
+        if any(isinstance(r, dict) and "text" in r for r in rows):
+            return "chat_log"
+    return "generic"
+
+
+def iter_translatable(data: Any, rel_path: str) -> Iterator[Item]:
+    schema = detect_schema(data, rel_path)
+    if schema == "conversation":
+        yield from _iter_conversation(data)
+    elif schema == "chat_log":
+        yield from _iter_chat_log(data)
+    else:
+        yield from _iter_generic(data, ())
+
+
+def _iter_conversation(data: dict) -> Iterator[Item]:
+    for msg_id, node in data.items():
+        if not isinstance(node, dict):
+            continue
+        message = node.get("message")
+        if is_translatable_value(message):
+            yield Item((msg_id, "message"), message, "dialogue")
+        options = node.get("options")
+        if isinstance(options, str):
+            if is_translatable_value(options):
+                yield Item((msg_id, "options"), options, "player")
+        elif isinstance(options, list):
+            for i, opt in enumerate(options):
+                if is_translatable_value(opt):
+                    yield Item((msg_id, "options", i), opt, "player")
+
+
+def _iter_chat_log(data: Any) -> Iterator[Item]:
+    rows = enumerate(data) if isinstance(data, list) else data.items()
+    for key, row in rows:
+        if not isinstance(row, dict):
+            continue
+        text = row.get("text")
+        if not is_translatable_value(text):
+            continue
+        row_type = str(row.get("type", ""))
+        if row_type == TYPE_PLAYER:
+            kind = "player"
+        elif row_type == TYPE_SYSTEM:
+            kind = "other"
+        else:  # "0" and any other code = a non-player chat participant
+            kind = "dialogue"
+        yield Item((key, "text"), text, kind)
+
+
+def _iter_generic(node: Any, path: tuple) -> Iterator[Item]:
+    if isinstance(node, dict):
+        for key, value in node.items():
+            if key in SKIP_FIELDS:
+                continue
+            yield from _iter_generic(value, path + (key,))
+    elif isinstance(node, list):
+        for i, value in enumerate(node):
+            yield from _iter_generic(value, path + (i,))
+    elif is_translatable_value(node):
+        yield Item(path, node, "other")
+
+
+def get_at(data: Any, path: tuple) -> Any:
+    node = data
+    for key in path:
+        node = node[key]
+    return node
+
+
+def set_at(data: Any, path: tuple, value: Any) -> Any:
+    """Return a deep copy of `data` with the value at `path` replaced."""
+    return set_many(data, [(path, value)])
+
+
+def set_many(data: Any, updates: list[tuple[tuple, Any]]) -> Any:
+    """Return a deep copy of `data` with every (path, value) update applied."""
+    result = copy.deepcopy(data)
+    for path, value in updates:
+        node = result
+        for key in path[:-1]:
+            node = node[key]
+        node[path[-1]] = value
+    return result
+
+
+def key_to_str(path: tuple) -> str:
+    return ".".join(str(p) for p in path)

main/pipeline/translate_pack.py

@@ -0,0 +1,196 @@
+"""Batch translation driver: English_JSON -> translations/<lang>/ review records.
+
+Stage 1 (translate): every translatable string through TranslateGemma.
+Stage 2 (tone): dialogue lines of mapped characters through the Ollama tone
+model using the character's voice wiki.
+
+One review record file is written per source file, mirroring the pack layout.
+Runs are resumable: existing records are merged by item key and only missing
+work is done. Repeated strings hit a shared translation cache.
+
+Usage:
+    python -m pipeline.translate_pack                       # full run, both stages
+    python -m pipeline.translate_pack --filter "Filler Chats/Brad"
+    python -m pipeline.translate_pack --stage translate     # stage 1 only
+    python -m pipeline.translate_pack --limit 5             # first 5 files
+    python -m pipeline.translate_pack --dry-run             # plan only, no LLM calls
+"""
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+import config
+from pipeline import clients
+from pipeline.dialogue_map import build_dialogue_map, display_name, load_wiki
+from pipeline.rules import iter_translatable, key_to_str
+
+SAVE_EVERY = 25  # persist record/cache after this many new translations
+CONTEXT_LINES = 2  # preceding source lines passed to the tone model
+
+
+def load_record(record_path: Path) -> dict:
+    if record_path.exists():
+        return json.loads(record_path.read_text(encoding="utf-8"))
+    return {}
+
+
+def save_json(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
+def build_items(data, rel: str, char_id: str | None, existing: dict) -> list[dict]:
+    """Fresh item list from the source file, merged with prior record state."""
+    prior = {item["key"]: item for item in existing.get("items", [])}
+    items = []
+    for found in iter_translatable(data, rel):
+        key = key_to_str(found.path)
+        old = prior.get(key)
+        if old and old["source"] == found.source:
+            items.append(old)
+            continue
+        items.append(
+            {
+                "key": key,
+                "path": list(found.path),
+                "source": found.source,
+                "kind": found.kind,
+                "character": char_id if found.kind == "dialogue" else None,
+                "mt": None,
+                "toned": None,
+                "status": "pending",
+                "final": None,
+            }
+        )
+    return items
+
+
+def run_translate_stage(record: dict, record_path: Path, cache: dict) -> int:
+    done = 0
+    for item in record["items"]:
+        if item["mt"] is not None:
+            continue
+        cached = cache.get(item["source"])
+        if cached is not None:
+            item["mt"] = cached
+        else:
+            item["mt"] = clients.translate(item["source"])
+            cache[item["source"]] = item["mt"]
+            done += 1
+            if done % SAVE_EVERY == 0:
+                save_json(record_path, record)
+                save_json(config.CACHE_PATH, cache)
+    return done
+
+
+def run_tone_stage(record: dict, record_path: Path) -> int:
+    char_id = record.get("character")
+    if not char_id:
+        return 0
+    wiki = load_wiki(char_id)
+    name = display_name(char_id)
+    sources = [item["source"] for item in record["items"]]
+
+    done = 0
+    for i, item in enumerate(record["items"]):
+        if item["kind"] != "dialogue" or item["toned"] is not None or item["mt"] is None:
+            continue
+        context = sources[max(0, i - CONTEXT_LINES) : i]
+        item["toned"] = clients.adjust_tone(item["source"], item["mt"], wiki, name, context)
+        done += 1
+        if done % SAVE_EVERY == 0:
+            save_json(record_path, record)
+    return done
+
+
+def needs_work(record: dict, stage: str) -> bool:
+    for item in record.get("items", []):
+        if stage in ("translate", "all") and item["mt"] is None:
+            return True
+        if (
+            stage in ("tone", "all")
+            and item["kind"] == "dialogue"
+            and record.get("character")
+            and item["mt"] is not None
+            and item["toned"] is None
+        ):
+            return True
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    parser.add_argument("--filter", default="", help="only files whose path contains this substring")
+    parser.add_argument("--limit", type=int, default=0, help="max number of files to process")
+    parser.add_argument("--stage", choices=["translate", "tone", "all"], default="all")
+    parser.add_argument("--dry-run", action="store_true", help="report planned work, no LLM calls")
+    args = parser.parse_args()
+    sys.stdout.reconfigure(line_buffering=True)
+
+    dmap = build_dialogue_map()
+    cache = json.loads(config.CACHE_PATH.read_text(encoding="utf-8")) if config.CACHE_PATH.exists() else {}
+
+    source_files = sorted(config.SOURCE_DIR.rglob("*.json"))
+    processed = 0
+    totals = {"translated": 0, "toned": 0, "files": 0}
+
+    try:
+        for source_file in source_files:
+            rel = str(source_file.relative_to(config.SOURCE_DIR))
+            if args.filter and args.filter not in rel:
+                continue
+            if args.limit and processed >= args.limit:
+                break
+
+            data = json.loads(source_file.read_text(encoding="utf-8"))
+            char_id = dmap.get(rel)
+            record_path = config.TRANSLATIONS_DIR / rel
+            record = load_record(record_path)
+            record = {
+                "file": rel,
+                "character": char_id,
+                "items": build_items(data, rel, char_id, record),
+            }
+            if not record["items"]:
+                continue
+            processed += 1
+            if not needs_work(record, args.stage):
+                continue
+
+            pending_mt = sum(1 for i in record["items"] if i["mt"] is None)
+            pending_tone = sum(
+                1
+                for i in record["items"]
+                if i["kind"] == "dialogue" and char_id and i["toned"] is None
+            )
+            print(f"[{rel}] strings={len(record['items'])} mt-pending={pending_mt} "
+                  f"tone-pending={pending_tone if char_id else 0}"
+                  + (f" character={char_id}" if char_id else ""))
+            if args.dry_run:
+                continue
+
+            started = time.monotonic()
+            if args.stage in ("translate", "all"):
+                totals["translated"] += run_translate_stage(record, record_path, cache)
+            if args.stage in ("tone", "all"):
+                totals["toned"] += run_tone_stage(record, record_path)
+            save_json(record_path, record)
+            save_json(config.CACHE_PATH, cache)
+            totals["files"] += 1
+            print(f"  done in {time.monotonic() - started:.0f}s")
+    except KeyboardInterrupt:
+        print("\nInterrupted - progress saved; rerun to resume.")
+        sys.exit(130)
+
+    print(
+        f"Finished: {totals['files']} file(s) updated, "
+        f"{totals['translated']} new translations, {totals['toned']} tone passes."
+    )
+
+
+if __name__ == "__main__":
+    main()