Deploy from GitHub Actions (d33778848a54dfe29fd95382aa7611a6fd3f9041)

.aiderignore

@@ -28,6 +28,8 @@
 !metrics.csv
 !TREND_BRIEF.md
 !MUTATION_PLAN.md
+!REPAIR_TASK.md
+!HF_LOGS.md
 !harness/genome.py
 !harness/mutator_prompt.md
 

harness/logcheck.py

@@ -0,0 +1,226 @@
+"""verify-and-fix detector — decide whether a recent PRODUCTION run broke a variant.
+
+The daily run (generate-daily -> HF body /run/daily) exercises every living variant at full
+budget with real rendering + publishing. That is where failures the merge-time smoke gate can't
+see show up (YouTube API changes, render edge cases, a variant smoke didn't catch). This module
+inspects the body's runtime logs (``container_logs`` in Mongo) for CODE-LEVEL crashes attributable
+to a variant and emits a decision the verify-fix workflow acts on:
+
+  ACTION=fix    TARGET=<variant>  SIGNATURE=<hash>   (+ writes REPAIR_TASK.md for the coder)
+  ACTION=alert  ...               (errors present but unattributable, OR a repeat of a fix that
+                                    already didn't hold — escalate instead of churning main)
+  ACTION=none   ...               (nothing actionable — the common case; no coder is woken)
+
+Design rules this enforces:
+  • It only flags crashes (entries carrying a traceback), never warnings or quality issues — a
+    quality change under an unchanged genome would poison fitness attribution.
+  • Repairs are NOT experiments: the workflow fixes the code WITHOUT a genome bump and records a
+    REPAIR note so the next research cycle doesn't misread the recovered metrics.
+  • Anti-oscillation: a signature already repaired (REPAIR_STATE.json) is not re-fixed; it
+    escalates to alert so a fix that didn't hold can't churn main every day.
+
+    python -m harness.logcheck --since 12h                       # detect + print decision
+    python -m harness.logcheck --record-repair --target v --signature s   # after a merged fix
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+STATE_PATH = REPO_ROOT / "REPAIR_STATE.json"
+TASK_PATH = REPO_ROOT / "REPAIR_TASK.md"
+LOG_PATH = REPO_ROOT / "EXPERIMENTS_LOG.md"
+
+_VARIANT_RE = re.compile(r"variant_(\w+)")
+# Normalise volatile tokens so the same bug hashes to the same signature day to day.
+_NOISE = [
+    (re.compile(r"0x[0-9a-fA-F]+"), "0xADDR"),
+    (re.compile(r"\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}"), "TS"),
+    (re.compile(r"[0-9a-f]{12,}"), "HEX"),
+    (re.compile(r"\b\d+\b"), "N"),
+]
+
+
+def _since(s: str) -> datetime:
+    s = s.strip().lower()
+    n = int(s[:-1])
+    delta = timedelta(hours=n) if s.endswith("h") else timedelta(days=n)
+    return datetime.now(timezone.utc) - delta
+
+
+def _norm(text: str) -> str:
+    t = text or ""
+    for rx, repl in _NOISE:
+        t = rx.sub(repl, t)
+    return t
+
+
+def signature(exc: str | None, msg: str | None) -> str:
+    """Stable 12-char id for an error — the last traceback line (exception type+msg) normalised."""
+    body = (exc or msg or "").strip().splitlines()
+    last = body[-1] if body else (msg or "")
+    return hashlib.sha256(_norm(last).encode("utf-8")).hexdigest()[:12]
+
+
+def attribute(*texts: str | None) -> str | None:
+    """Best-effort: which variant does this error belong to? Most-mentioned variant_N token wins."""
+    hits: Counter[str] = Counter()
+    for t in texts:
+        for m in _VARIANT_RE.finditer(t or ""):
+            hits[f"variant_{m.group(1)}"] += 1
+    return hits.most_common(1)[0][0] if hits else None
+
+
+def _is_crash(doc: dict) -> bool:
+    """Code-level failure = carries a traceback (exc) or an explicit Traceback in the message."""
+    return bool(doc.get("exc")) or "Traceback" in (doc.get("msg") or "")
+
+
+def decide(errs: list[dict], state: dict, *, min_count: int = 1) -> dict:
+    """Pure decision core (no I/O), so it is unit-testable.
+
+    Returns {action, target, signature, members} where action in {fix, alert, none}.
+    """
+    crashes = [d for d in errs if _is_crash(d)]
+    if not crashes:
+        return {"action": "none", "target": None, "signature": "", "members": []}
+
+    groups: dict[str, list[dict]] = defaultdict(list)
+    for d in crashes:
+        groups[signature(d.get("exc"), d.get("msg"))].append(d)
+
+    sig, members = max(groups.items(), key=lambda kv: len(kv[1]))
+    if len(members) < min_count:
+        return {"action": "none", "target": None, "signature": sig, "members": []}
+
+    target = attribute(
+        *[m.get("logger") for m in members],
+        *[m.get("msg") for m in members],
+        *[m.get("exc") for m in members],
+    )
+
+    if state.get("signature") == sig:
+        # Already attempted — the prior fix did not hold. Don't re-fix; escalate.
+        return {"action": "alert", "target": target, "signature": sig, "members": members}
+    if not target:
+        return {"action": "alert", "target": None, "signature": sig, "members": members}
+    return {"action": "fix", "target": target, "signature": sig, "members": members}
+
+
+# ── I/O helpers ─────────────────────────��─────────────────────────────────────
+
+def _load_state() -> dict:
+    try:
+        return json.loads(STATE_PATH.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _fetch_errors(since: str) -> list[dict] | None:
+    url = os.getenv("MONGO_FITNESS_READONLY_URL", "").strip()
+    db = os.getenv("MONGO_DATABASE", "content_generator").strip() or "content_generator"
+    if not url:
+        return None
+    try:
+        from pymongo import MongoClient
+
+        coll = MongoClient(url, serverSelectionTimeoutMS=10000)[db]["container_logs"]
+        return list(
+            coll.find(
+                {"ts": {"$gte": _since(since)}, "level": {"$in": ["ERROR", "CRITICAL"]}},
+                {"_id": 0, "ts": 1, "level": 1, "logger": 1, "msg": 1, "exc": 1},
+                limit=1000,
+            )
+        )
+    except Exception as error:  # noqa: BLE001 — any failure means "can't verify"; stay safe
+        print(f"::warning::log query failed: {error}")
+        return None
+
+
+def _write_task(decision: dict) -> None:
+    target, sig, members = decision["target"], decision["signature"], decision["members"]
+    lines = [
+        f"# REPAIR TASK — {target}",
+        "",
+        f"A recent PRODUCTION daily run logged {len(members)} code-level crash(es) attributable to "
+        f"**{target}** (signature `{sig}`). Fix ONLY {target}'s code so this stops crashing, with "
+        "the smallest change that addresses the error. This is a REPAIR, not an experiment — do "
+        "not change the genome/manifest and do not add features.",
+        "",
+        "## Representative error(s)",
+    ]
+    for d in members[:3]:
+        lines.append(f"**{d.get('level','ERROR')}** `{d.get('logger','')}`")
+        lines.append((d.get("msg") or "").strip()[:1000])
+        if d.get("exc"):
+            lines.append("```")
+            lines.append(d["exc"].strip()[:2500])
+            lines.append("```")
+        lines.append("")
+    TASK_PATH.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
+
+
+def _record_repair(target: str, sig: str) -> None:
+    """Persist the repaired signature (anti-oscillation) AND drop a REPAIR note in the log so the
+    next research cycle knows a repair happened and won't misattribute the recovered metrics."""
+    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    STATE_PATH.write_text(
+        json.dumps({"signature": sig, "target": target, "date": today}, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    note = (
+        f"\n## {today} — {target} — REPAIR (no genome change)\n"
+        f"- repaired a production crash (error signature {sig}); code-only fix, genome unchanged.\n"
+        f"- not an experiment — metrics after this date reflect the variant's intended behaviour.\n"
+    )
+    prev = LOG_PATH.read_text(encoding="utf-8") if LOG_PATH.exists() else ""
+    if prev and not prev.endswith("\n"):
+        prev += "\n"
+    LOG_PATH.write_text(prev + note, encoding="utf-8")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Detect production crashes that warrant a repair.")
+    ap.add_argument("--since", default="12h", help="how far back to scan logs (e.g. 12h, 1d)")
+    ap.add_argument("--min-count", type=int, default=1, help="min occurrences to act on")
+    ap.add_argument("--record-repair", action="store_true", help="record a completed repair")
+    ap.add_argument("--target", default="")
+    ap.add_argument("--signature", default="")
+    args = ap.parse_args()
+
+    if args.record_repair:
+        if not args.target or not args.signature:
+            print("::error::--record-repair needs --target and --signature")
+            return 1
+        _record_repair(args.target, args.signature)
+        print(f"recorded repair: {args.target} / {args.signature}")
+        return 0
+
+    errs = _fetch_errors(args.since)
+    if errs is None:
+        print("ACTION=none")
+        print("::warning::could not read logs; skipping verify-fix this cycle.")
+        return 0
+
+    decision = decide(errs, _load_state(), min_count=args.min_count)
+    if decision["action"] == "fix":
+        _write_task(decision)
+
+    print(f"ACTION={decision['action']}")
+    print(f"TARGET={decision['target'] or 'NONE'}")
+    print(f"SIGNATURE={decision['signature'] or 'NONE'}")
+    print(f"(scanned {len(errs)} error-level logs; {len(decision['members'])} crash(es) in the "
+          f"dominant group)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

harness/repair_prompt.md

@@ -0,0 +1,21 @@
+You are the REPAIR operator of a self-improving content organism. A recent PRODUCTION run crashed.
+`REPAIR_TASK.md` (in your context) names the broken variant and shows the actual error(s). Your
+only job is to make that variant stop crashing — nothing else.
+
+This is a REPAIR, not an experiment. The difference matters:
+- Make the SMALLEST change that fixes the specific error in `REPAIR_TASK.md`.
+- Do **NOT** change any `manifest.json` / genome. The `genome_hash` MUST stay identical, or the
+  fix would be misread as a new experiment and poison the fitness attribution for that variant.
+- Do **NOT** add features, refactor, or change the content strategy. Restore intended behaviour.
+- Do **NOT** edit `EXPERIMENTS_LOG.md` or `CURRENT_EXPERIMENT.md` — the harness records the repair
+  itself.
+
+Hard rules (unchanged from the rest of the organism):
+- You may ONLY edit files under `variants/`. The scorer, tests, dispatcher, publishers and secrets
+  are invisible and locked.
+- The locked `pytest` suite must stay green. Never weaken a test (you cannot see them anyway).
+- The fix must survive the runtime smoke test: when `generate_video_plan` is actually called it
+  must return a non-empty list of valid VideoPlan objects.
+- Never add code that reads environment tokens/secrets or sends data to new destinations.
+
+Focus only on the variant named in `REPAIR_TASK.md`. Output only the file edits.