feat: add agent trace export

README.md

@@ -65,6 +65,10 @@ Local deterministic mode and app launch work without `HF_TOKEN`.
 
 ## Trace export
 
-Use the collapsed **Trace / Debug** panel to inspect a readable JSON preview of the current show trace. After creating a show, the **Download Trace JSON** button saves a file named `ai-puppet-theater-trace-<session_id>.json`.
+Every show keeps a lightweight agent trace in memory for the current Gradio session. The collapsed **Behind the Curtain** panel shows a compact human-readable trace summary with a copy button, plus the Director log. The collapsed **Trace / Debug** panel shows the full JSON preview.
 
-The trace includes session metadata, the premise, title, setting, selected actor and Director backends, model ids when used, Director decisions, actor responses, audience actions, validation status, latency, and fallback metadata. Trace output is sanitized for sharing: it does not include Hugging Face tokens, environment variable values, private local paths, raw tracebacks, actor secrets, or hidden reasoning.
+After creating a show, use **Download Trace JSON** to save `ai-puppet-theater-trace-<session_id>.json`. The file contains a stable envelope with `app_name`, `trace_version`, `session_id`, `created_at`, premise, title, setting, `show_length_mode`, min/target/max beats, actor backend, Director mode, model id when used, and normalized events.
+
+Trace events use a consistent public shape with `event_type`, `timestamp` or `step`, beat/story phase metadata, speaker, Director decisions, actor intent, actor memory updates, tool requests and results, audience actions, backend/model metadata, latency, validation status, fallback usage, and short fallback reasons where available.
+
+Traces are sanitized so they are safe to publish later as a dataset: they do not include Hugging Face tokens, environment variable values, private local paths, raw tracebacks, hidden reasoning, or private secrets. Errors are reduced to short summaries and model failures are recorded as fallback events.

app.py

@@ -27,7 +27,7 @@ from puppet_theater import (
     throw_prop,
     warm_up_openbmb,
 )
-from puppet_theater.trace import add_trace_event, render_trace_json, write_trace_json_file
+from puppet_theater.trace import add_trace_event, render_trace_json, render_trace_summary, write_trace_json_file
 
 
 EMPTY_STAGE = """
@@ -53,6 +53,7 @@ EMPTY_TRANSCRIPT = '<div class="conversation-empty">No show yet. The transcript
 EMPTY_AGENT_STATE = "<div class=\"agent-state-empty\">No agents on stage yet.</div>"
 EMPTY_DIRECTOR_LOG = "No director notes yet."
 EMPTY_TRACE = "No trace events yet."
+EMPTY_TRACE_SUMMARY = "No trace events yet."
 EMPTY_BACKEND = (
     "Active backend: deterministic\n"
     "Director mode: deterministic\n"
@@ -2453,6 +2454,10 @@ def render_trace(session: TheaterSession | None) -> str:
     return render_trace_json(session)
 
 
+def render_trace_summary_text(session: TheaterSession | None) -> str:
+    return render_trace_summary(session)
+
+
 def normalize_backend_name(backend_name: str | None) -> str:
     return backend_name if backend_name in BACKEND_VALUES else "deterministic"
 
@@ -2574,6 +2579,7 @@ def render_outputs(session: TheaterSession | None, voice_mode: str | None = DEFA
         f'<div class="voice-status">{escape(tts_status)}</div>',
         render_agent_state(session),
         render_director_log(session),
+        render_trace_summary_text(session),
         render_trace(session),
         write_trace_json_file(session),
         render_backend_settings(session),
@@ -2606,6 +2612,7 @@ def create_show(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(
@@ -2648,6 +2655,7 @@ def reset_show():
         f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
         EMPTY_AGENT_STATE,
         EMPTY_DIRECTOR_LOG,
+        EMPTY_TRACE_SUMMARY,
         EMPTY_TRACE,
         None,
         DEFAULT_SHOW_LENGTH,
@@ -2680,6 +2688,7 @@ def advance_one_beat(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(None, backend_name, director_mode, max_new_tokens, temperature),
@@ -2709,6 +2718,7 @@ def advance_full_act(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(None, backend_name, director_mode, max_new_tokens, temperature),
@@ -2778,6 +2788,7 @@ def throw_audience_prop(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(None, backend_name, director_mode, max_new_tokens, temperature),
@@ -2807,6 +2818,7 @@ def summon_audience_actor(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(None, backend_name, director_mode, max_new_tokens, temperature),
@@ -2835,6 +2847,7 @@ def request_audience_finale(
             f'<div class="voice-status">{escape(EMPTY_TTS_STATUS)}</div>',
             EMPTY_AGENT_STATE,
             EMPTY_DIRECTOR_LOG,
+            EMPTY_TRACE_SUMMARY,
             EMPTY_TRACE,
             None,
             render_backend_settings(None, backend_name, director_mode, max_new_tokens, temperature),
@@ -2886,6 +2899,7 @@ def warm_up_backend(
     return (
         session,
         render_director_log(session),
+        render_trace_summary_text(session),
         render_trace(session),
         write_trace_json_file(session),
         render_backend_settings(
@@ -3012,6 +3026,14 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             elem_classes=["no-field-label"],
         )
     with gr.Accordion("Behind the Curtain", open=False):
+        trace_summary_output = gr.Textbox(
+            value=EMPTY_TRACE_SUMMARY,
+            label="Trace Summary",
+            lines=8,
+            interactive=False,
+            buttons=["copy"],
+            elem_classes=["no-field-label"],
+        )
         director_output = gr.Textbox(
             value=EMPTY_DIRECTOR_LOG,
             label="Director Log",
@@ -3097,6 +3119,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3121,6 +3144,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3146,6 +3170,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3171,6 +3196,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3196,6 +3222,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3220,6 +3247,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             backend_output,
@@ -3228,7 +3256,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
     warm_up_button.click(
         warm_up_backend,
         inputs=[session_state, max_new_tokens_input, temperature_input],
-        outputs=[session_state, director_output, trace_output, trace_download, backend_output],
+        outputs=[session_state, director_output, trace_summary_output, trace_output, trace_download, backend_output],
     )
     reset_button.click(
         reset_show,
@@ -3244,6 +3272,7 @@ with gr.Blocks(title="AI Puppet Theater") as app:
             tts_status_output,
             agent_state_output,
             director_output,
+            trace_summary_output,
             trace_output,
             trace_download,
             show_length_select,

puppet_theater/director.py

@@ -453,6 +453,18 @@ def run_one_beat(session: TheaterSession | None) -> TheaterSession | None:
             fallback_used=director_generation.fallback_used,
             fallback_reason=director_generation.error,
         )
+    if director_generation.fallback_used:
+        add_trace_event(
+            session,
+            "director_fallback",
+            speaker=speaker.name,
+            backend_name=director_generation.director_mode,
+            model_id=director_generation.model_id,
+            validation_status=director_generation.validation_status,
+            fallback_used=True,
+            fallback_reason=director_generation.error or director_generation.validation_status,
+            story_phase=current_phase,
+        )
 
     backend_generation = generate_actor_response(session, decision, speaker, prop)
     session.beat_index += 1
@@ -557,6 +569,19 @@ def run_one_beat(session: TheaterSession | None) -> TheaterSession | None:
             fallback_used=backend_generation.fallback_used,
             fallback_reason=backend_generation.error,
         )
+    if backend_generation.fallback_used:
+        add_trace_event(
+            session,
+            "actor_fallback",
+            beat_index=session.beat_index,
+            speaker=speaker.name,
+            backend_name=backend_generation.backend_name,
+            model_id=backend_generation.model_id,
+            validation_status=backend_generation.validation_status,
+            fallback_used=True,
+            fallback_reason=backend_generation.error or backend_generation.validation_status,
+            load_status=backend_generation.load_status,
+        )
 
     if decision.should_end_scene or decision.beat_type == "finale":
         session.beat_index = session.max_beats

puppet_theater/models.py

@@ -161,6 +161,7 @@ class TheaterSession:
     min_beats: int = 7
     target_beats: int = 10
     max_beats: int = 12
+    show_length_mode: str = "standard"
     transcript: list[Beat] = field(default_factory=list)
     props: list[str] = field(default_factory=list)
     latest_prop: str | None = None

puppet_theater/session.py

@@ -106,6 +106,7 @@ def create_show_from_premise(
         min_beats=min_beats,
         target_beats=target_beats,
         max_beats=max_beats,
+        show_length_mode=active_show_length,
         transcript=[],
         props=[],
         latest_prop=None,

puppet_theater/trace.py

@@ -13,6 +13,26 @@ from puppet_theater.models import TheaterSession
 APP_NAME = "AI Puppet Theater"
 TRACE_VERSION = "1.0"
 _MAX_TEXT_CHARS = 500
+_STANDARD_EVENT_KEYS = (
+    "event_type",
+    "timestamp",
+    "step",
+    "beat_index",
+    "story_phase",
+    "speaker",
+    "director_decision",
+    "actor_intent",
+    "actor_memory_update",
+    "tool_request",
+    "tool_result",
+    "audience_action",
+    "backend_metadata",
+    "latency_ms",
+    "validation_status",
+    "fallback_used",
+    "fallback_reason",
+    "summary",
+)
 _PRIVATE_PATH_PATTERNS = [
     re.compile(r"/Users/[^\s\"'<>:]+(?:/[^\s\"'<>:]+)*"),
     re.compile(r"/private/(?:tmp|var)/[^\s\"'<>:]+(?:/[^\s\"'<>:]+)*"),
@@ -22,7 +42,7 @@ _PRIVATE_PATH_PATTERNS = [
 
 def add_trace_event(session: TheaterSession, event_type: str, **fields: Any) -> None:
     event: dict[str, Any] = {
-        "type": sanitize_value(event_type),
+        "event_type": sanitize_value(event_type),
         "timestamp": datetime.now(timezone.utc).isoformat(),
         "step": len(session.trace_events) + 1,
     }
@@ -49,6 +69,7 @@ def export_trace(session: TheaterSession | None) -> dict[str, Any] | None:
         "premise": sanitize_value(session.premise),
         "title": sanitize_value(session.show_title),
         "setting": sanitize_value(session.setting),
+        "show_length_mode": sanitize_value(session.show_length_mode),
         "min_beats": session.min_beats,
         "target_beats": session.target_beats,
         "max_beats": session.max_beats,
@@ -88,22 +109,193 @@ def normalize_trace_events(session: TheaterSession) -> list[dict[str, Any]]:
     normalized: list[dict[str, Any]] = []
     for index, raw_event in enumerate(session.trace_events, start=1):
         if isinstance(raw_event, dict):
-            event = {str(key): sanitize_value(value) for key, value in raw_event.items() if value is not None}
-            event.setdefault("type", "event")
-            event.setdefault("step", index)
-            normalized.append(event)
+            normalized.append(_normalize_event_dict(raw_event, index))
             continue
 
         normalized.append(
             {
-                "type": "legacy_event",
+                "event_type": "legacy_event",
                 "step": index,
                 "message": sanitize_value(raw_event),
+                "summary": sanitize_value(raw_event),
             }
         )
     return normalized
 
 
+def render_trace_summary(session: TheaterSession | None, max_events: int = 14) -> str:
+    if session is None:
+        return "No trace events yet."
+
+    events = normalize_trace_events(session)
+    if not events:
+        return "No trace events yet."
+
+    lines = [
+        f"{session.show_title} ({session.session_id})",
+        (
+            f"{session.show_length_mode} show: {session.min_beats}/"
+            f"{session.target_beats}/{session.max_beats} beats, "
+            f"actor backend={session.backend_name}, director={session.director_mode}"
+        ),
+        "",
+    ]
+    for event in events[-max_events:]:
+        step = event.get("step", "?")
+        event_type = event.get("event_type", "event")
+        beat = event.get("beat_index")
+        prefix = f"{step}. {event_type}"
+        if beat is not None:
+            prefix += f" [beat {beat}]"
+        lines.append(f"{prefix}: {_event_summary(event)}")
+    return "\n".join(lines)
+
+
+def _normalize_event_dict(raw_event: dict[str, Any], index: int) -> dict[str, Any]:
+    cleaned = {str(key): sanitize_value(value) for key, value in raw_event.items() if value is not None}
+    event_type = str(cleaned.pop("event_type", cleaned.pop("type", "event")) or "event")
+    normalized: dict[str, Any] = {
+        "event_type": sanitize_value(event_type),
+        "step": cleaned.pop("step", index),
+    }
+    if "timestamp" in cleaned:
+        normalized["timestamp"] = cleaned.pop("timestamp")
+
+    for key in (
+        "beat_index",
+        "story_phase",
+        "speaker",
+        "latency_ms",
+        "validation_status",
+        "fallback_used",
+        "fallback_reason",
+    ):
+        if key in cleaned:
+            normalized[key] = cleaned.pop(key)
+
+    if event_type == "director_decision":
+        normalized["director_decision"] = {
+            key: cleaned.pop(key)
+            for key in (
+                "beat_type",
+                "reason_summary",
+                "progress",
+                "uses_prop",
+                "reveal_secret",
+                "should_end_scene",
+                "min_beats",
+                "target_beats",
+                "max_beats",
+            )
+            if key in cleaned
+        }
+    if event_type == "actor_intent" and "intent" in cleaned:
+        normalized["actor_intent"] = cleaned.pop("intent")
+    if event_type == "actor_memory_update" and "memory_update" in cleaned:
+        normalized["actor_memory_update"] = cleaned.pop("memory_update")
+    if event_type in {"tool_requested", "tool_executed", "tool_ignored"}:
+        normalized["tool_request"] = {
+            key: cleaned.pop(key)
+            for key in ("tool_name", "reason", "arguments")
+            if key in cleaned
+        }
+    if event_type == "tool_result":
+        normalized["tool_result"] = {
+            key: cleaned.pop(key)
+            for key in ("tool_name", "result", "stage_effect")
+            if key in cleaned
+        }
+    if event_type == "audience_action":
+        action = cleaned.pop("audience_action", None)
+        normalized["audience_action"] = {
+            key: value
+            for key, value in {
+                "action": action,
+                "summary": cleaned.pop("action_summary", None),
+                "prop": cleaned.pop("prop", None),
+                "summoned_actor": cleaned.pop("summoned_actor", None),
+            }.items()
+            if value is not None
+        }
+
+    backend_metadata = {
+        key: cleaned.pop(key)
+        for key in (
+            "backend_name",
+            "model_id",
+            "load_status",
+            "director_mode",
+            "tts_backend",
+            "voice_mode",
+            "voice_name",
+        )
+        if key in cleaned
+    }
+    if backend_metadata:
+        normalized["backend_metadata"] = backend_metadata
+
+    if "reason_summary" in cleaned:
+        normalized["summary"] = cleaned.pop("reason_summary")
+    elif "action_summary" in cleaned:
+        normalized["summary"] = cleaned.pop("action_summary")
+    elif "error_summary" in cleaned:
+        normalized["summary"] = cleaned.pop("error_summary")
+
+    for key, value in cleaned.items():
+        if key not in normalized:
+            normalized[key] = value
+
+    if "summary" not in normalized:
+        normalized["summary"] = _event_summary(normalized)
+    return {key: normalized[key] for key in _STANDARD_EVENT_KEYS if key in normalized} | {
+        key: value for key, value in normalized.items() if key not in _STANDARD_EVENT_KEYS
+    }
+
+
+def _event_summary(event: dict[str, Any]) -> str:
+    event_type = str(event.get("event_type") or "event")
+    if event_type == "show_created":
+        backend = _backend_label(event)
+        return f"Show created with {backend}."
+    if event_type == "actors_created":
+        return f"{event.get('actor_count', 'Actors')} actors created."
+    if event_type == "director_decision":
+        decision = event.get("director_decision") if isinstance(event.get("director_decision"), dict) else {}
+        beat_type = decision.get("beat_type") if isinstance(decision, dict) else None
+        speaker = event.get("speaker") or "next actor"
+        reason = decision.get("reason_summary") if isinstance(decision, dict) else None
+        return f"Director chose {speaker} for {beat_type or 'a beat'}" + (f": {reason}" if reason else ".")
+    if event_type in {"actor_response", "backend_error", "backend_warmup"}:
+        status = event.get("validation_status") or "status unknown"
+        fallback = " with fallback" if event.get("fallback_used") else ""
+        return f"Backend returned {status}{fallback}."
+    if event_type in {"actor_fallback", "director_fallback"}:
+        reason = event.get("fallback_reason") or event.get("validation_status") or "fallback used"
+        return f"Fallback used: {reason}."
+    if event_type == "actor_intent":
+        return str(event.get("actor_intent") or "Actor intent recorded.")
+    if event_type == "actor_memory_update":
+        return str(event.get("actor_memory_update") or "Actor memory updated.")
+    if event_type == "tool_requested":
+        tool = _tool_label(event.get("tool_request"))
+        return f"Tool requested: {tool}."
+    if event_type == "tool_executed":
+        tool = _tool_label(event.get("tool_request"))
+        return f"Tool executed: {tool}."
+    if event_type == "tool_result":
+        result = event.get("tool_result") if isinstance(event.get("tool_result"), dict) else {}
+        return str(result.get("result") or f"Tool result from {result.get('tool_name', 'tool')}.")
+    if event_type == "audience_action":
+        action = event.get("audience_action") if isinstance(event.get("audience_action"), dict) else {}
+        return str(action.get("summary") or action.get("action") or "Audience action recorded.")
+    if event_type in {"tts_requested", "tts_generated", "tts_fallback"}:
+        metadata = event.get("backend_metadata") if isinstance(event.get("backend_metadata"), dict) else {}
+        return f"TTS event via {metadata.get('tts_backend', 'voice backend')}."
+    if event_type == "scene_completed":
+        return str(event.get("summary") or "Scene completed.")
+    return str(event.get("summary") or event_type.replace("_", " ").title())
+
+
 def sanitize_value(value: Any) -> Any:
     if isinstance(value, dict):
         return {
@@ -123,20 +315,41 @@ def sanitize_value(value: Any) -> Any:
 def _sanitize_text(value: str) -> str:
     text = " ".join(value.split())
     if "Traceback (most recent call last)" in text:
-        text = "Traceback redacted"
+        text = "Error details redacted"
+    for key, secret in os.environ.items():
+        if not _is_sensitive_key(key) or not secret:
+            continue
+        text = text.replace(secret, "[redacted]")
     for secret_name in ("HF_TOKEN", "HUGGINGFACEHUB_API_TOKEN"):
         secret = os.getenv(secret_name)
         if secret:
             text = text.replace(secret, "[redacted]")
-    text = re.sub(r"\b(HF_TOKEN|HUGGINGFACEHUB_API_TOKEN)\s*=\s*\S+", r"\1=[redacted]", text)
+    text = re.sub(
+        r"\b([A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|CREDENTIAL|API_KEY|ACCESS_KEY)[A-Z0-9_]*)\s*=\s*\S+",
+        r"\1=[redacted]",
+        text,
+    )
     for pattern in _PRIVATE_PATH_PATTERNS:
         text = pattern.sub("[redacted-path]", text)
     return text[:_MAX_TEXT_CHARS].rstrip()
 
 
+def _backend_label(event: dict[str, Any]) -> str:
+    metadata = event.get("backend_metadata") if isinstance(event.get("backend_metadata"), dict) else {}
+    backend = metadata.get("backend_name") or metadata.get("director_mode") or event.get("backend_name") or "backend"
+    model = metadata.get("model_id") or event.get("model_id")
+    return f"{backend} ({model})" if model else str(backend)
+
+
+def _tool_label(tool_request: Any) -> str:
+    if isinstance(tool_request, dict):
+        return str(tool_request.get("tool_name") or "tool")
+    return "tool"
+
+
 def _is_sensitive_key(key: str) -> bool:
     lowered = key.lower()
-    return "token" in lowered or "secret" in lowered or "password" in lowered
+    return any(marker in lowered for marker in ("token", "secret", "password", "credential", "api_key", "access_key"))
 
 
 def _model_id_for_mode(mode: str | None) -> str | None: