"""One read-only endpoint: the vocabulary the editor draws its dropdowns from.
The alternative is a copy of `VOCAB` in JavaScript, and a copy is a second source
of truth that goes stale the first time a phrase is improved. `directives.py`
exists precisely so that improving a sentence improves every plan ever written;
duplicating those strings in the UI would undo that.
It also ships the *prose* each option compiles to, not just the option names, so
the editor can show what a directive actually puts in front of the encoder. That
is the whole answer to "I don't know how any of this works at a glance".
Since 2026-08-28 this module also carries the reference *files*: `POST
/h3_ref_chain/upload` streams dropped media into `/h3_refs`, and `GET
/h3_ref_chain/files` lists what is there. Those replaced the nine `ref_image_N`
IMAGE sockets, which were most of a 16-socket column.
Everything that decides *where bytes land* lives in `media.py`, not here: one
prefix-checked resolver used by both the route and the loaders, so there is a
single place to be wrong about it. The route itself never joins a path.
Since 2026-08-30 it also carries the optional plan writer: `GET/POST
/h3_ref_chain/llm` reads and saves the local chat-server settings, and `POST
/h3_ref_chain/plan` runs the generate-validate-repair loop in `planner.py`.
Those are the only places in the pack that touch the network, and none of them
is reachable from a graph execution -- the button writes JSON into a widget,
and queueing reads the widget. `llm` and `planner` are imported inside the
handlers rather than at module scope so a broken or absent aiohttp client
cannot stop the pack from loading.
"""
import os as _os
from . import directives as _d
from . import media as _media
from . import refs as _refs
from . import waveform as _waveform
TAG = "HandTieClips"
ROUTE = "/h3_ref_chain/vocab"
UPLOAD_ROUTE = "/h3_ref_chain/upload"
FILES_ROUTE = "/h3_ref_chain/files"
PEAKS_ROUTE = "/h3_ref_chain/peaks"
LLM_ROUTE = "/h3_ref_chain/llm"
PLAN_ROUTE = "/h3_ref_chain/plan"
SWAP_PLAN_ROUTE = "/h3_ref_chain/swap_plan"
SWAP_DESCRIBE_ROUTE = "/h3_ref_chain/swap_describe"
UNLOAD_ROUTE = "/h3_ref_chain/llm/unload"
# Decoded waveform summaries, keyed (name, mtime, n).
#
# mtime is in the key for the same reason `media.stamp` puts it in IS_CHANGED:
# a filename is a stable input even when the bytes behind it change, and a trim
# bar drawn from the previous file is a bar that lies about where the beats are.
#
# Cleared wholesale rather than evicted one at a time. The entries are ~1.5 KB
# and the working set is however many references one chain uses, so an LRU would
# be more machinery than the thing it manages.
_PEAKS = {}
_PEAKS_MAX = 48
# A batch of stills is a handful; this is a guard against a runaway multipart
# body, not a considered product limit.
MAX_UPLOAD_FILES = 32
MAX_UPLOAD_BYTES = 256 * 1024 * 1024
def _pinned_refs(raw, limit):
"""Rail rows from the Write-plan POST. Only rows with a tag and a file."""
out = []
if not isinstance(raw, list):
return out
for r in raw:
if not isinstance(r, dict):
continue
tag = str(r.get("tag") or "").lstrip("@").strip()
fname = str(r.get("file") or "").strip()
if not tag or not fname:
continue
subj = r.get("subject")
try:
subj = int(subj) if subj not in (None, "", False) else None
except (TypeError, ValueError):
subj = None
# `mp` rides along so planner._restore_rail_only can put it back on the
# written register. The model is never shown it and cannot author it,
# but Accept overwrites the rail with what comes back, so a cap that
# does not survive this round trip is a cap the next write deletes.
try:
mp = float(r.get("mp") or 0) or None
except (TypeError, ValueError):
mp = None
out.append({
"tag": tag,
"file": fname,
"subject": subj,
"retention": str(r.get("retention") or ""),
"desc": str(r.get("desc") or ""),
"mp": mp,
})
if len(out) >= int(limit):
break
return out
def _payload():
from .h3_ref_chain import (
ASPECTS, DURATION_FRAMES, OVERLAP_FRAMES, RESOLUTIONS, FPS, _canvas)
return {
# axis -> option -> the sentence it compiles to. Order matters: AXES is
# the order directive_prose concatenates in, so the UI shows them in the
# order the encoder reads them.
"axes": list(_d.AXES),
"vocab": {axis: dict(opts) for axis, opts in _d.VOCAB.items()},
"defaults": dict(_d.DEFAULTS),
"establish": _d.ESTABLISH,
# join has nothing to attach to on the first hop; the editor hides it
# there to match directive_prose's hop_index == 0 skip.
"join_axis": "join",
"retention": dict(_refs.RETENTION),
"max_ref_images": _refs.MAX_REF_IMAGES,
"refs_subdir": _media.REFS_SUBDIR,
"image_exts": sorted(_media.IMAGE_EXTS),
"video_exts": sorted(_media.VIDEO_EXTS),
"audio_exts": sorted(_media.AUDIO_EXTS),
"ref_fields": list(_refs.REF_FIELDS),
"subject_fields": list(_refs.SUBJECT_FIELDS),
"durations": {k: v for k, v in DURATION_FRAMES.items()},
"overlaps": {k: v for k, v in OVERLAP_FRAMES.items()},
# Resolved here rather than in the editor. The size is a function of two
# combo labels, an area cap and a 32 px grid; a second implementation in
# JavaScript is a second place for it to be wrong, and the run panel's
# digest would be the thing quietly disagreeing with what rendered.
"canvas": {res: {asp: list(_canvas(res, asp)) for asp in ASPECTS}
for res in RESOLUTIONS},
"fps": FPS,
}
def register():
"""Attach the route if a PromptServer exists. Never raises on import."""
try:
from aiohttp import web
from server import PromptServer
except ImportError:
return False
instance = getattr(PromptServer, "instance", None)
if instance is None or not hasattr(instance, "routes"):
return False
@instance.routes.get(ROUTE)
async def _vocab(_request):
try:
return web.json_response(_payload())
except Exception as exc: # a broken payload must not take the server down
print(f"[{TAG}] vocab route failed: {exc!r}", flush=True)
return web.json_response({"error": str(exc)}, status=500)
@instance.routes.get(FILES_ROUTE)
async def _files(request):
"""What is already in the reference folder, for the editor's picker."""
try:
kinds = request.rel_url.query.get("kinds") or ""
want = {k.strip() for k in kinds.split(",") if k.strip()} or None
return web.json_response({"ok": True, "files": _media.listing(want)})
except Exception as exc:
print(f"[{TAG}] files route failed: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)}, status=500)
@instance.routes.post(PEAKS_ROUTE)
async def _peaks(request):
"""A waveform summary for the trim bar: `{peaks: [...], seconds}`.
The decode runs in an executor, NOT on this coroutine. These handlers
share ComfyUI's event loop, and decoding a three-minute file on it
freezes the canvas, the queue and the progress bar together -- the same
constraint `llm.py`'s docstring spells out for the writer.
"""
import asyncio
try:
body = await request.json()
except Exception:
body = {}
name = str((body or {}).get("name") or "").strip()
n = (body or {}).get("n") or _waveform.DEFAULT_N
path = _media.resolve(name, kinds={"audio", "video"})
if path is None:
# Not an error the panel can act on -- the file is simply gone, and
# the picker already says so. A flat line draws an empty bar.
return web.json_response({"ok": True, "peaks": [], "seconds": 0.0})
try:
mtime = _os.path.getmtime(path)
except OSError:
mtime = 0.0
key = (name, mtime, int(n))
hit = _PEAKS.get(key)
if hit is not None:
return web.json_response({"ok": True, "peaks": hit[0],
"seconds": hit[1], "cached": True})
def work():
audio = _media.load_audio(name)
if audio is None:
# A video with no audio track, or a file PyAV cannot open. The
# bar still has to position its grips, so report the duration
# even when there is nothing to draw.
return [], 0.0
return _waveform.peaks(audio, n)
try:
got = await asyncio.get_running_loop().run_in_executor(None, work)
except Exception as exc:
print(f"[{TAG}] peaks route failed for {name!r}: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)}, status=500)
if len(_PEAKS) >= _PEAKS_MAX:
_PEAKS.clear()
_PEAKS[key] = got
return web.json_response({"ok": True, "peaks": got[0], "seconds": got[1]})
@instance.routes.post(UPLOAD_ROUTE)
async def _upload(request):
"""Stream dropped media into the reference folder.
Multipart, not JSON+base64: a batch of stills should not be inflated by
a third and held in memory twice on the way through the browser. The
destination is fixed -- `media.refs_dir()` -- so a filename can only
ever name a file *inside* it, and `media.resolve` re-checks that on the
way back out.
"""
if not (request.content_type or "").startswith("multipart/"):
return web.json_response(
{"ok": False, "error": "expected multipart/form-data"}, status=400)
try:
reader = await request.multipart()
except Exception as exc:
return web.json_response(
{"ok": False, "error": f"bad multipart: {exc}"}, status=400)
dest = _media.refs_dir(create=True)
saved, skipped, total = [], [], 0
try:
while True:
part = await reader.next()
if part is None:
break
if not getattr(part, "filename", None):
continue
if len(saved) >= MAX_UPLOAD_FILES:
skipped.append(f"{part.filename}: batch limit "
f"{MAX_UPLOAD_FILES} reached")
break
# Basename only, and the extension has to be one we can open.
# Rejecting here means nothing unreadable is ever written.
name = _os.path.basename(part.filename or "")
kind = _media.kind_of(name)
if kind is None:
skipped.append(f"{name}: not an image, video or audio file")
continue
path = _media.unique_path(dest, name)
size = 0
try:
with open(path, "wb") as fh:
while True:
chunk = await part.read_chunk()
if not chunk:
break
size += len(chunk)
total += len(chunk)
if total > MAX_UPLOAD_BYTES:
raise ValueError("upload too large")
fh.write(chunk)
except Exception:
# A partial file is worse than no file: it would list in
# the picker and fail to open.
try:
_os.remove(path)
except OSError:
pass
raise
w = h = 0
if kind == "image":
path, w, h = _media.shrink_image(path)
saved.append({"name": _os.path.basename(path), "kind": kind,
"width": w, "height": h, "bytes": size})
except Exception as exc:
print(f"[{TAG}] upload failed: {exc!r}", flush=True)
return web.json_response(
{"ok": False, "error": str(exc), "files": saved}, status=400)
if saved:
print(f"[{TAG}] uploaded {len(saved)} reference file(s) -> "
+ ", ".join(f["name"] for f in saved), flush=True)
for note in skipped:
print(f"[{TAG}] upload skipped {note}", flush=True)
return web.json_response({"ok": True, "files": saved, "skipped": skipped})
# -- the optional plan writer -----------------------------------------
#
# Everything below is inert until someone opens Settings in the panel and
# points it at a server. With no server configured the panel shows the
# manual paste recipe and nothing here is ever called.
@instance.routes.get(LLM_ROUTE)
async def _llm_get(_request):
"""Saved settings plus whatever models the server currently offers.
Answers 200 with an empty model list when the server is down, rather
than an error: the user needs this panel open in order to fix the URL
that is the reason the list is empty.
"""
try:
from . import llm as _llm
conn = _llm.load_conn()
found = await _llm.models(conn["server_url"])
return web.json_response({
"ok": True,
"server_url": conn["server_url"],
# The saved model goes back with the list so the panel can
# PRESELECT it. Without this the dropdown lands on option[0]
# and the next save silently rewrites the configured model to
# whatever happens to be first -- the exact bug recorded at
# PromptMasterLD's h3_studio_ui.js:5007.
"model": conn["model"],
"temperature": conn["temperature"],
"keep_warm": conn["keep_warm"],
"unload_on_run": conn["unload_on_run"],
"vram_settle_s": conn["vram_settle_s"],
# [{id, loaded}] -- `loaded` is None on servers with no
# native state route. The panel marks the difference, because
# picking an unloaded model is picking a 400.
"models": found,
"online": bool(found),
"any_loaded": any(m.get("loaded") for m in found),
})
except Exception as exc:
print(f"[{TAG}] llm route failed: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)}, status=500)
@instance.routes.post(LLM_ROUTE)
async def _llm_set(request):
try:
body = await request.json()
except Exception:
return web.json_response(
{"ok": False, "error": "expected a JSON body"}, status=400)
try:
from . import llm as _llm
conn = _llm.save_conn(body if isinstance(body, dict) else {})
return web.json_response({"ok": True, **conn})
except Exception as exc:
print(f"[{TAG}] llm save failed: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)}, status=500)
@instance.routes.post(UNLOAD_ROUTE)
async def _llm_unload(request):
"""Free the writer's VRAM on demand. Never a 500, never a hard error.
The automatic unload after writing covers the ordinary case. This is
for the ones it cannot: the checkbox was off, the write failed before
it ran, or LM Studio's JIT put a different model in memory than the one
configured. Pressing it when nothing is loaded is a no-op that says so.
"""
try:
from . import llm as _llm
except Exception as exc:
return web.json_response({"ok": False, "error": str(exc)})
try:
conn = _llm.load_conn()
n, note = await _llm.unload_all(conn["server_url"], conn["model"])
return web.json_response({"ok": True, "unloaded": n, "note": note})
except Exception as exc:
print(f"[{TAG}] unload failed: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)})
@instance.routes.post(PLAN_ROUTE)
async def _plan(request):
"""Write a plan, and make the model repair it until the node accepts it.
Returns `ok: false` with a readable `error` for every failure mode --
no server, no model, a model that will not converge. None of them is a
500: they are all ordinary states of a machine the user controls, and
an exception page in a status line helps nobody.
"""
try:
body = await request.json()
except Exception:
return web.json_response(
{"ok": False, "error": "expected a JSON body"}, status=400)
brief = str(body.get("brief") or "").strip()
try:
hops = max(1, min(24, int(body.get("hops") or 3)))
except (TypeError, ValueError):
hops = 3
# The node's `duration` widget. The prompt's length table has a row per
# hop length and the model was never told which row it was writing to
# -- the prompt says to ASK, which a button cannot answer. Unknown or
# absent is fine: `build_user_turn` adds nothing and the lint no-ops.
duration = str(body.get("duration") or "").strip()
try:
from . import llm as _llm
from . import planner as _planner
from . import refs as _refs
except Exception as exc:
return web.json_response(
{"ok": False, "error": f"the plan writer failed to load: {exc}"})
conn = _llm.load_conn()
if not conn.get("model"):
return web.json_response(
{"ok": False, "error": "no model is selected -- open Settings "
"in the panel and pick one."})
pinned = _pinned_refs(body.get("refs"), _refs.MAX_REF_IMAGES)
# A filled rail is the scene. The folder listing is only the fallback
# for a brief-only write with empty boxes -- that listing is what made
# the model pick files the user never chose.
if pinned:
files = [p["file"] for p in pinned]
else:
files = [f["name"] for f in _media.listing({"image", "video"})]
import asyncio
images = []
if pinned:
def encode():
out = []
for p in pinned:
url = _media.vision_data_url(p["file"])
if url:
out.append({"tag": p["tag"], "data_url": url})
return out
try:
images = await asyncio.get_running_loop().run_in_executor(
None, encode)
except Exception as exc:
print(f"[{TAG}] could not attach rail stills: {exc!r}",
flush=True)
images = []
vision_notes = []
async def complete_fn(messages, schema=None):
try:
return await _llm.complete(
conn["server_url"], conn["model"], messages,
schema=schema, temperature=conn["temperature"])
except _llm.LLMError as exc:
if _llm.has_images(messages) and "HTTP 4" in str(exc):
print(f"[{TAG}] server rejected the attached stills; "
f"retrying from filenames only", flush=True)
vision_notes.append(
"the model could not look at the pictures; "
"the draft used filenames only")
return await _llm.complete(
conn["server_url"], conn["model"],
_llm.text_only(messages),
schema=schema, temperature=conn["temperature"])
raise
try:
out = await _planner.write_plan(
brief, hops, complete_fn=complete_fn, files=files,
pinned=pinned, images=images, duration=duration)
except _llm.LLMError as exc:
return web.json_response({"ok": False, "error": str(exc)})
except Exception as exc:
print(f"[{TAG}] plan route failed: {exc!r}", flush=True)
return web.json_response({"ok": False, "error": str(exc)})
if vision_notes:
out["warnings"] = list(out.get("warnings") or []) + vision_notes
# Stay resident by default. The render is where the card is actually
# contended, and the node evicts there (`llm.free_for_render`), so
# unloading here only makes the NEXT plan pay a full model load --
# and nobody writes exactly one plan. `keep_warm` off restores the old
# unload-immediately behaviour for a machine too tight to hold both.
# A courtesy either way, never a failure: `unload` swallows its own
# errors and returns False when no endpoint answers.
if not conn.get("keep_warm"):
try:
await _llm.unload(conn["server_url"], conn["model"])
except Exception:
pass
if out["ok"]:
print(f"[{TAG}] wrote a {hops}-hop plan in {out['attempts']} "
f"attempt(s)", flush=True)
return web.json_response(out)
def _swap_conn():
from . import llm as _llm
from . import planner as _planner
conn = _llm.load_conn()
return _llm, _planner, conn
@instance.routes.post(SWAP_DESCRIBE_ROUTE)
async def _swap_describe(request):
"""Caption the clip frame at the trim IN point. No plan, no rail."""
try:
body = await request.json()
except Exception:
return web.json_response(
{"ok": False, "error": "expected a JSON body"}, status=400)
video = str(body.get("video") or "").strip()
if not video:
return web.json_response(
{"ok": False, "error": "pick a reference clip first"})
try:
start = float(body.get("video_start_s") or 0.0)
except (TypeError, ValueError):
start = 0.0
try:
end = float(body.get("video_end_s") or 0.0)
except (TypeError, ValueError):
end = 0.0
try:
_llm, _planner, conn = _swap_conn()
except Exception as exc:
return web.json_response(
{"ok": False, "error": f"SWAP failed to load: {exc}"})
if not conn.get("model"):
return web.json_response(
{"ok": False, "error": "no model is selected -- open WRITE, "
"Settings, and pick one."})
import asyncio
# Three, like the plan path. A caption written from one frame describes
# a pose, and this caption reaches the encoder as what