Character Cartridges (.joypack v1.0): spec, packer, loader node, and WREN - the first portable video-character file (verified: same face + rhotic General American voice in an unseen scene, zero training)
Browse files- .gitattributes +1 -0
- JOYPACK_SPEC.md +94 -0
- __init__.py +13 -1
- joyecho_cartridge.py +104 -0
- joypack.py +208 -0
- joypacks/WREN.joypack +3 -0
- nodes.py +119 -6
- web/js/joyecho_widget_persistence.js +57 -0
.gitattributes
CHANGED
|
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
demo_one_room_q8.mp4 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
demo_one_room_q8.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
joypacks/WREN.joypack filter=lfs diff=lfs merge=lfs -text
|
JOYPACK_SPEC.md
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# .joypack — portable video-character cartridge (spec v1.0)
|
| 2 |
+
|
| 3 |
+
One file = one character: face, voice, canon description, and optionally the
|
| 4 |
+
LoRAs and environments that define them. Drop it into a JoyEcho workflow and
|
| 5 |
+
render new scenes with that exact character — same face, same voice — with
|
| 6 |
+
zero training. Designed to extend the roleplay ecosystem's Character Card
|
| 7 |
+
V3/CHARX lineage into photoreal joint audio-video generation.
|
| 8 |
+
|
| 9 |
+
## Container
|
| 10 |
+
|
| 11 |
+
A `.joypack` is a ZIP archive (store or deflate) with this layout:
|
| 12 |
+
|
| 13 |
+
```
|
| 14 |
+
manifest.json REQUIRED spec + inventory (see below)
|
| 15 |
+
persona/card.json optional CharacterCardV3-compatible JSON (chat interop)
|
| 16 |
+
voice/anchor.(mp4|wav) REQUIRED 4s+ clip of the character speaking.
|
| 17 |
+
Seeds the JoyEcho memory bank before
|
| 18 |
+
shot 1 - the voice continues instead of
|
| 19 |
+
being re-rolled. mp4 preferred (pairs
|
| 20 |
+
audio with a moving reference face).
|
| 21 |
+
refs/*.png|jpg REQUIRED 1-8 MSR identity reference stills,
|
| 22 |
+
front-lit face clearly visible.
|
| 23 |
+
prompts/dna.txt REQUIRED the canonical DNA sentence(s): the
|
| 24 |
+
verbatim identity text to carry in every
|
| 25 |
+
shot prompt, plus voice scaffold line
|
| 26 |
+
(timbre + accent binding).
|
| 27 |
+
loras/zimage/*.safetensors optional face LoRA for first-frame/still work
|
| 28 |
+
loras/ltx/*.safetensors optional LTX-2.3 LoRA(s) (ID-LoRA, style)
|
| 29 |
+
environment/*.png|jpg optional canonical location stills
|
| 30 |
+
environment/rooms.txt optional verbatim room description(s) - the
|
| 31 |
+
location-lock prose, one per paragraph
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## manifest.json
|
| 35 |
+
|
| 36 |
+
```json
|
| 37 |
+
{
|
| 38 |
+
"joypack": "1.0",
|
| 39 |
+
"name": "MARA",
|
| 40 |
+
"speaker_tag": "mara",
|
| 41 |
+
"display_name": "Detective Mara",
|
| 42 |
+
"authors": ["you"],
|
| 43 |
+
"license": "CC-BY-4.0",
|
| 44 |
+
"notes": "freeform",
|
| 45 |
+
"voice": {"file": "voice/anchor.mp4", "accent_line": "speaking in a casual American accent"},
|
| 46 |
+
"refs": ["refs/mara_01.png", "refs/mara_02.png"],
|
| 47 |
+
"dna": "prompts/dna.txt",
|
| 48 |
+
"loras": {
|
| 49 |
+
"zimage": [{"file": "loras/zimage/mara_z.safetensors", "trigger": "mara_z", "strength": 0.7}],
|
| 50 |
+
"ltx": [{"file": "loras/ltx/mara_id.safetensors", "strength": 1.0}]
|
| 51 |
+
},
|
| 52 |
+
"environment": {"stills": ["environment/apartment_01.png"], "rooms": "environment/rooms.txt"},
|
| 53 |
+
"render_law": {"video_fps": 24},
|
| 54 |
+
"sha256": {"voice/anchor.mp4": "..."}
|
| 55 |
+
}
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
Only `joypack`, `name`, `speaker_tag`, `voice`, `refs`, and `dna` are required.
|
| 59 |
+
Unknown keys MUST be ignored (forward compatibility). `render_law` records
|
| 60 |
+
generation constraints the character was authored under - loaders should warn
|
| 61 |
+
when the live workflow violates them (the fps/accent law above all).
|
| 62 |
+
|
| 63 |
+
## Loader behavior (reference implementation: JoyEcho_CartridgeLoader)
|
| 64 |
+
|
| 65 |
+
Loading MATERIALIZES the cartridge into the host's existing conventions -
|
| 66 |
+
the render path itself needs no changes:
|
| 67 |
+
|
| 68 |
+
1. Unpack to a cache dir keyed by content hash.
|
| 69 |
+
2. Copy `voice/anchor.*` to `input/joyecho_voices/<speaker_tag>/` (folder
|
| 70 |
+
auto-casting picks it up; every script whose speaker tag matches is voiced
|
| 71 |
+
by the anchor).
|
| 72 |
+
3. Copy `refs/*` to the workflow's refs root under `<speaker_tag>/` (RefPicker
|
| 73 |
+
finds them by character scan).
|
| 74 |
+
4. Copy LoRAs to `models/loras/joypack/<name>/` and report their names for the
|
| 75 |
+
LoRA stack.
|
| 76 |
+
5. Output the DNA text, accent line, room text, and trigger words as node
|
| 77 |
+
outputs for prompt assembly.
|
| 78 |
+
6. Verify sha256 entries when present; refuse a cartridge whose manifest is
|
| 79 |
+
missing required keys.
|
| 80 |
+
|
| 81 |
+
Removal = delete the materialized folders; a cartridge never modifies the
|
| 82 |
+
host beyond those drop-in locations.
|
| 83 |
+
|
| 84 |
+
## Security
|
| 85 |
+
|
| 86 |
+
A cartridge is DATA. Loaders MUST NOT execute anything from the archive, MUST
|
| 87 |
+
reject path traversal (entries containing `..` or absolute paths), and should
|
| 88 |
+
treat persona/card.json purely as chat-client payload. LoRA files are weights;
|
| 89 |
+
standard safetensors-only policy applies (no pickle formats).
|
| 90 |
+
|
| 91 |
+
## Versioning
|
| 92 |
+
|
| 93 |
+
`joypack: "1.0"` this document. Planned 1.1: serialized memory-bank state
|
| 94 |
+
(`bank/state.joybank`) for scene-warm characters; multi-character packs.
|
__init__.py
CHANGED
|
@@ -159,4 +159,16 @@ try:
|
|
| 159 |
except Exception as e:
|
| 160 |
print(f"[Rebels JE] joyecho_autofinish failed to load: {e!r}", flush=True)
|
| 161 |
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
except Exception as e:
|
| 160 |
print(f"[Rebels JE] joyecho_autofinish failed to load: {e!r}", flush=True)
|
| 161 |
|
| 162 |
+
try:
|
| 163 |
+
from .joyecho_cartridge import (
|
| 164 |
+
NODE_CLASS_MAPPINGS as _CG_CM,
|
| 165 |
+
NODE_DISPLAY_NAME_MAPPINGS as _CG_DM,
|
| 166 |
+
)
|
| 167 |
+
NODE_CLASS_MAPPINGS.update(_CG_CM)
|
| 168 |
+
NODE_DISPLAY_NAME_MAPPINGS.update(_CG_DM)
|
| 169 |
+
except Exception as _e:
|
| 170 |
+
print(f"[JoyAI-Echo] cartridge loader unavailable: {_e}")
|
| 171 |
+
|
| 172 |
+
WEB_DIRECTORY = "./web/js"
|
| 173 |
+
|
| 174 |
+
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
joyecho_cartridge.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""JoyEcho_CartridgeLoader - load a .joypack character cartridge.
|
| 2 |
+
|
| 3 |
+
Materializes the cartridge into the host's existing conventions (voice anchor
|
| 4 |
+
into input/joyecho_voices/<tag>/, reference stills into the refs root, LoRAs
|
| 5 |
+
into models/loras/joypack/<name>/) and outputs the character's texts for
|
| 6 |
+
prompt assembly. The render path needs no changes - after loading, any script
|
| 7 |
+
whose speaker tag matches is voiced and identified as the cartridge character.
|
| 8 |
+
|
| 9 |
+
See JOYPACK_SPEC.md for the format.
|
| 10 |
+
"""
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
try:
|
| 15 |
+
import folder_paths
|
| 16 |
+
except Exception:
|
| 17 |
+
folder_paths = None
|
| 18 |
+
|
| 19 |
+
from .joypack import materialize, read_manifest, JoypackError
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _packs_dir():
|
| 23 |
+
if folder_paths:
|
| 24 |
+
base = folder_paths.get_input_directory()
|
| 25 |
+
else:
|
| 26 |
+
base = os.path.join(os.getcwd(), "input")
|
| 27 |
+
d = os.path.join(base, "joypacks")
|
| 28 |
+
os.makedirs(d, exist_ok=True)
|
| 29 |
+
return d
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class JoyEcho_CartridgeLoader:
|
| 33 |
+
@classmethod
|
| 34 |
+
def INPUT_TYPES(cls):
|
| 35 |
+
packs = ["(none)"]
|
| 36 |
+
try:
|
| 37 |
+
packs += sorted(f for f in os.listdir(_packs_dir())
|
| 38 |
+
if f.lower().endswith(".joypack"))
|
| 39 |
+
except Exception:
|
| 40 |
+
pass
|
| 41 |
+
return {
|
| 42 |
+
"required": {
|
| 43 |
+
"cartridge": (packs, {
|
| 44 |
+
"tooltip": "A .joypack from input/joypacks/. Loading installs "
|
| 45 |
+
"the character's voice anchor, reference stills and "
|
| 46 |
+
"LoRAs into this workflow's normal folders - any "
|
| 47 |
+
"script whose speaker tag matches renders as this "
|
| 48 |
+
"character. See JOYPACK_SPEC.md."}),
|
| 49 |
+
"refs_root": ("STRING", {"default": "joyecho_refs", "tooltip":
|
| 50 |
+
"Where reference stills install (RefPicker's root). "
|
| 51 |
+
"Absolute path or relative to the input folder."}),
|
| 52 |
+
},
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING")
|
| 56 |
+
RETURN_NAMES = ("speaker_tag", "dna_text", "accent_line", "rooms_text", "report")
|
| 57 |
+
FUNCTION = "load"
|
| 58 |
+
CATEGORY = "JoyAI-Echo"
|
| 59 |
+
|
| 60 |
+
def load(self, cartridge, refs_root="joyecho_refs"):
|
| 61 |
+
if cartridge == "(none)":
|
| 62 |
+
return ("", "", "", "", "no cartridge selected")
|
| 63 |
+
pack_path = os.path.join(_packs_dir(), cartridge)
|
| 64 |
+
|
| 65 |
+
in_dir = (folder_paths.get_input_directory() if folder_paths
|
| 66 |
+
else os.path.join(os.getcwd(), "input"))
|
| 67 |
+
voices_dir = os.path.join(in_dir, "joyecho_voices")
|
| 68 |
+
refs_dir = (refs_root if os.path.isabs(refs_root)
|
| 69 |
+
else os.path.join(in_dir, refs_root))
|
| 70 |
+
if folder_paths:
|
| 71 |
+
loras_dir = os.path.join(folder_paths.models_dir, "loras", "joypack")
|
| 72 |
+
else:
|
| 73 |
+
loras_dir = os.path.join(os.getcwd(), "models", "loras", "joypack")
|
| 74 |
+
cache_dir = os.path.join(in_dir, "joypacks", "_cache")
|
| 75 |
+
|
| 76 |
+
try:
|
| 77 |
+
res = materialize(pack_path, voices_dir, refs_dir, loras_dir, cache_dir)
|
| 78 |
+
except (JoypackError, OSError) as e:
|
| 79 |
+
raise ValueError(f"joypack load failed: {e}")
|
| 80 |
+
|
| 81 |
+
law = res.get("render_law", {})
|
| 82 |
+
law_note = ""
|
| 83 |
+
if law.get("video_fps"):
|
| 84 |
+
law_note = (f" | render law: video_fps={law['video_fps']} "
|
| 85 |
+
f"(authored under the 24fps accent law)" if law["video_fps"] == 24
|
| 86 |
+
else f" | render law: video_fps={law['video_fps']}")
|
| 87 |
+
loras_note = ""
|
| 88 |
+
if res["ltx_loras"]:
|
| 89 |
+
loras_note = " | LTX loras: " + ", ".join(
|
| 90 |
+
f"{os.path.basename(l['path'])}@{l['strength']}" for l in res["ltx_loras"])
|
| 91 |
+
report = (f"loaded '{res['name']}' as speaker '{res['speaker_tag']}' - "
|
| 92 |
+
f"voice anchor + {sum(1 for p in res['installed_paths'] if refs_dir in p)} "
|
| 93 |
+
f"ref(s) installed{loras_note}{law_note}")
|
| 94 |
+
print(f"[JoyEcho] Cartridge: {report}", flush=True)
|
| 95 |
+
|
| 96 |
+
dna = res["dna"]
|
| 97 |
+
if res.get("triggers"):
|
| 98 |
+
dna = dna # triggers are for Z-Image prompts; reported, not injected
|
| 99 |
+
return (res["speaker_tag"], dna, res.get("accent_line", ""),
|
| 100 |
+
res.get("rooms", ""), report)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
NODE_CLASS_MAPPINGS = {"JoyEcho_CartridgeLoader": JoyEcho_CartridgeLoader}
|
| 104 |
+
NODE_DISPLAY_NAME_MAPPINGS = {"JoyEcho_CartridgeLoader": "JoyEcho Cartridge Loader (.joypack)"}
|
joypack.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""joypack - portable video-character cartridges (.joypack), spec v1.0.
|
| 2 |
+
|
| 3 |
+
Library + CLI. See JOYPACK_SPEC.md for the format. A .joypack is a plain zip;
|
| 4 |
+
this module packs one from a source folder and safely unpacks/materializes one
|
| 5 |
+
into the JoyEcho host conventions (joyecho_voices/<tag>/, refs root, loras).
|
| 6 |
+
|
| 7 |
+
CLI:
|
| 8 |
+
python joypack.py pack <source_dir> <out.joypack>
|
| 9 |
+
python joypack.py inspect <file.joypack>
|
| 10 |
+
|
| 11 |
+
Security: archives are DATA. No execution, no path traversal (rejected), no
|
| 12 |
+
pickle weight formats (safetensors only for loras/).
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import hashlib
|
| 17 |
+
import json
|
| 18 |
+
import os
|
| 19 |
+
import shutil
|
| 20 |
+
import sys
|
| 21 |
+
import zipfile
|
| 22 |
+
|
| 23 |
+
SPEC_VERSION = "1.0"
|
| 24 |
+
REQUIRED_KEYS = ("joypack", "name", "speaker_tag", "voice", "refs", "dna")
|
| 25 |
+
VOICE_EXTS = (".mp4", ".wav")
|
| 26 |
+
REF_EXTS = (".png", ".jpg", ".jpeg", ".webp")
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class JoypackError(ValueError):
|
| 30 |
+
pass
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _sha256(path):
|
| 34 |
+
h = hashlib.sha256()
|
| 35 |
+
with open(path, "rb") as f:
|
| 36 |
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
| 37 |
+
h.update(chunk)
|
| 38 |
+
return h.hexdigest()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _safe_name(entry_name):
|
| 42 |
+
"""Reject traversal and absolute paths; return normalized relative path."""
|
| 43 |
+
n = entry_name.replace("\\", "/")
|
| 44 |
+
if n.startswith("/") or ".." in n.split("/") or (len(n) > 1 and n[1] == ":"):
|
| 45 |
+
raise JoypackError(f"unsafe path in archive: {entry_name!r}")
|
| 46 |
+
return n
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def pack(source_dir, out_path):
|
| 50 |
+
"""Build a .joypack from a folder laid out per the spec.
|
| 51 |
+
|
| 52 |
+
If the folder has no manifest.json, a minimal one is synthesized from the
|
| 53 |
+
layout (name = folder name, speaker_tag = lowercased name).
|
| 54 |
+
"""
|
| 55 |
+
source_dir = os.path.abspath(source_dir)
|
| 56 |
+
man_path = os.path.join(source_dir, "manifest.json")
|
| 57 |
+
if os.path.isfile(man_path):
|
| 58 |
+
manifest = json.load(open(man_path, encoding="utf-8"))
|
| 59 |
+
else:
|
| 60 |
+
name = os.path.basename(source_dir.rstrip("\\/"))
|
| 61 |
+
manifest = {"joypack": SPEC_VERSION, "name": name,
|
| 62 |
+
"speaker_tag": name.lower()}
|
| 63 |
+
|
| 64 |
+
# discover required components if unlisted
|
| 65 |
+
def _find(sub, exts):
|
| 66 |
+
d = os.path.join(source_dir, sub)
|
| 67 |
+
if not os.path.isdir(d):
|
| 68 |
+
return []
|
| 69 |
+
return sorted(f"{sub}/{f}" for f in os.listdir(d)
|
| 70 |
+
if f.lower().endswith(exts))
|
| 71 |
+
|
| 72 |
+
if "voice" not in manifest or not manifest["voice"].get("file"):
|
| 73 |
+
vs = _find("voice", VOICE_EXTS)
|
| 74 |
+
if not vs:
|
| 75 |
+
raise JoypackError("no voice anchor found (voice/anchor.mp4|wav)")
|
| 76 |
+
manifest.setdefault("voice", {})["file"] = vs[0]
|
| 77 |
+
if not manifest.get("refs"):
|
| 78 |
+
manifest["refs"] = _find("refs", REF_EXTS)
|
| 79 |
+
if not manifest["refs"]:
|
| 80 |
+
raise JoypackError("no reference stills found (refs/*.png)")
|
| 81 |
+
if not manifest.get("dna"):
|
| 82 |
+
p = "prompts/dna.txt"
|
| 83 |
+
if not os.path.isfile(os.path.join(source_dir, p)):
|
| 84 |
+
raise JoypackError("no DNA text found (prompts/dna.txt)")
|
| 85 |
+
manifest["dna"] = p
|
| 86 |
+
manifest.setdefault("render_law", {"video_fps": 24})
|
| 87 |
+
|
| 88 |
+
# collect every real file under the spec dirs + the manifest
|
| 89 |
+
files = []
|
| 90 |
+
for root, _, names in os.walk(source_dir):
|
| 91 |
+
for fn in names:
|
| 92 |
+
full = os.path.join(root, fn)
|
| 93 |
+
rel = os.path.relpath(full, source_dir).replace("\\", "/")
|
| 94 |
+
if rel == "manifest.json":
|
| 95 |
+
continue
|
| 96 |
+
if rel.startswith("loras/") and not fn.lower().endswith(".safetensors"):
|
| 97 |
+
raise JoypackError(f"loras/ may only contain .safetensors: {rel}")
|
| 98 |
+
files.append(rel)
|
| 99 |
+
|
| 100 |
+
manifest["sha256"] = {manifest["voice"]["file"]:
|
| 101 |
+
_sha256(os.path.join(source_dir, manifest["voice"]["file"]))}
|
| 102 |
+
for k in REQUIRED_KEYS:
|
| 103 |
+
if k not in manifest:
|
| 104 |
+
raise JoypackError(f"manifest missing required key: {k}")
|
| 105 |
+
|
| 106 |
+
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z:
|
| 107 |
+
z.writestr("manifest.json", json.dumps(manifest, indent=1))
|
| 108 |
+
for rel in files:
|
| 109 |
+
z.write(os.path.join(source_dir, rel), rel)
|
| 110 |
+
return out_path, manifest
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def read_manifest(pack_path):
|
| 114 |
+
with zipfile.ZipFile(pack_path) as z:
|
| 115 |
+
try:
|
| 116 |
+
manifest = json.loads(z.read("manifest.json").decode("utf-8"))
|
| 117 |
+
except KeyError:
|
| 118 |
+
raise JoypackError("not a joypack: manifest.json missing")
|
| 119 |
+
for k in REQUIRED_KEYS:
|
| 120 |
+
if k not in manifest:
|
| 121 |
+
raise JoypackError(f"manifest missing required key: {k}")
|
| 122 |
+
return manifest
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def materialize(pack_path, voices_dir, refs_dir, loras_dir, cache_dir):
|
| 126 |
+
"""Unpack + install a cartridge into host conventions.
|
| 127 |
+
|
| 128 |
+
Returns a dict: {name, speaker_tag, dna, accent_line, rooms, triggers,
|
| 129 |
+
ltx_loras, zimage_loras, installed_paths}
|
| 130 |
+
"""
|
| 131 |
+
manifest = read_manifest(pack_path)
|
| 132 |
+
tag = manifest["speaker_tag"]
|
| 133 |
+
name = manifest["name"]
|
| 134 |
+
out = {"name": name, "speaker_tag": tag, "installed_paths": []}
|
| 135 |
+
|
| 136 |
+
with zipfile.ZipFile(pack_path) as z:
|
| 137 |
+
names = {_safe_name(n) for n in z.namelist()}
|
| 138 |
+
|
| 139 |
+
def _extract(member, dst_dir, dst_name=None):
|
| 140 |
+
member = _safe_name(member)
|
| 141 |
+
if member not in names:
|
| 142 |
+
raise JoypackError(f"manifest references missing file: {member}")
|
| 143 |
+
os.makedirs(dst_dir, exist_ok=True)
|
| 144 |
+
dst = os.path.join(dst_dir, dst_name or os.path.basename(member))
|
| 145 |
+
with z.open(member) as src, open(dst, "wb") as f:
|
| 146 |
+
shutil.copyfileobj(src, f)
|
| 147 |
+
out["installed_paths"].append(dst)
|
| 148 |
+
return dst
|
| 149 |
+
|
| 150 |
+
# voice anchor -> joyecho_voices/<tag>/
|
| 151 |
+
vfile = manifest["voice"]["file"]
|
| 152 |
+
want = manifest.get("sha256", {}).get(vfile)
|
| 153 |
+
vdst = _extract(vfile, os.path.join(voices_dir, tag))
|
| 154 |
+
if want and _sha256(vdst) != want:
|
| 155 |
+
os.remove(vdst)
|
| 156 |
+
raise JoypackError("voice anchor failed sha256 verification")
|
| 157 |
+
|
| 158 |
+
# refs -> <refs_root>/<NAME>/
|
| 159 |
+
for r in manifest["refs"]:
|
| 160 |
+
_extract(r, os.path.join(refs_dir, name))
|
| 161 |
+
|
| 162 |
+
# loras -> models/loras/joypack/<name>/
|
| 163 |
+
out["ltx_loras"], out["zimage_loras"] = [], []
|
| 164 |
+
loras = manifest.get("loras", {})
|
| 165 |
+
for fam, key in (("ltx", "ltx_loras"), ("zimage", "zimage_loras")):
|
| 166 |
+
for entry in loras.get(fam, []) or []:
|
| 167 |
+
dst = _extract(entry["file"], os.path.join(loras_dir, name))
|
| 168 |
+
out[key].append({"path": dst,
|
| 169 |
+
"strength": float(entry.get("strength", 1.0)),
|
| 170 |
+
"trigger": entry.get("trigger", "")})
|
| 171 |
+
|
| 172 |
+
# texts
|
| 173 |
+
out["dna"] = z.read(_safe_name(manifest["dna"])).decode("utf-8").strip()
|
| 174 |
+
out["accent_line"] = manifest.get("voice", {}).get("accent_line", "")
|
| 175 |
+
rooms = manifest.get("environment", {}).get("rooms")
|
| 176 |
+
out["rooms"] = (z.read(_safe_name(rooms)).decode("utf-8").strip()
|
| 177 |
+
if rooms and _safe_name(rooms) in names else "")
|
| 178 |
+
env_stills = manifest.get("environment", {}).get("stills", []) or []
|
| 179 |
+
for s in env_stills:
|
| 180 |
+
_extract(s, os.path.join(cache_dir, name, "environment"))
|
| 181 |
+
|
| 182 |
+
out["render_law"] = manifest.get("render_law", {})
|
| 183 |
+
out["triggers"] = [e.get("trigger", "") for e in
|
| 184 |
+
(manifest.get("loras", {}).get("zimage", []) or []) if e.get("trigger")]
|
| 185 |
+
return out
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def main():
|
| 189 |
+
if len(sys.argv) < 3:
|
| 190 |
+
print(__doc__)
|
| 191 |
+
return 1
|
| 192 |
+
cmd = sys.argv[1]
|
| 193 |
+
if cmd == "pack":
|
| 194 |
+
out, man = pack(sys.argv[2], sys.argv[3])
|
| 195 |
+
print(f"packed {man['name']} -> {out}")
|
| 196 |
+
print(f" speaker_tag={man['speaker_tag']} refs={len(man['refs'])} "
|
| 197 |
+
f"voice={man['voice']['file']}")
|
| 198 |
+
elif cmd == "inspect":
|
| 199 |
+
man = read_manifest(sys.argv[2])
|
| 200 |
+
print(json.dumps(man, indent=1))
|
| 201 |
+
else:
|
| 202 |
+
print(__doc__)
|
| 203 |
+
return 1
|
| 204 |
+
return 0
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
if __name__ == "__main__":
|
| 208 |
+
raise SystemExit(main())
|
joypacks/WREN.joypack
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:664ec73cbd0d52088765ae87f584407993c4ad308aea539f8e0c2f99b1ed5652
|
| 3 |
+
size 2682926
|
nodes.py
CHANGED
|
@@ -1224,7 +1224,7 @@ class JoyEcho_Generate:
|
|
| 1224 |
"model": ("JOYECHO_MODEL",),
|
| 1225 |
"conditioning": ("JOYECHO_COND",),
|
| 1226 |
"seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
|
| 1227 |
-
"num_frames": ("INT", {"default": 241, "min": 9, "max":
|
| 1228 |
"tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
|
| 1229 |
# Rebels local patch: portrait resolutions. Height was capped at
|
| 1230 |
# 1088 while width allowed 1920, which silently forbade portrait
|
|
@@ -1380,6 +1380,19 @@ class JoyEcho_Generate:
|
|
| 1380 |
"it) - use heights whose /32 is EVEN (768, not 736) or it "
|
| 1381 |
"smears one edge.",
|
| 1382 |
}),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1383 |
},
|
| 1384 |
}
|
| 1385 |
|
|
@@ -1449,6 +1462,7 @@ class JoyEcho_Generate:
|
|
| 1449 |
resident_blocks: int = 0,
|
| 1450 |
hires_factor: float = 1.0,
|
| 1451 |
hires_denoise: str = "subtle (1 step)",
|
|
|
|
| 1452 |
):
|
| 1453 |
from ltx_distillation.inference.bidirectional_pipeline import BidirectionalAVInferencePipeline
|
| 1454 |
from ltx_distillation.inference.memory_bidirectional_pipeline import BidirectionalMemoryAVInferencePipeline
|
|
@@ -1617,6 +1631,12 @@ class JoyEcho_Generate:
|
|
| 1617 |
_hires_audio_lats = []
|
| 1618 |
_hires_video_lats = [] # spatial hires mode only (~10MB/shot on CPU)
|
| 1619 |
_hires_spatial = str(hires_denoise).lower().startswith("spatial")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1620 |
|
| 1621 |
num_shots = len(conditioning)
|
| 1622 |
offloader = None
|
|
@@ -2050,6 +2070,23 @@ class JoyEcho_Generate:
|
|
| 2050 |
if device.type == "cuda":
|
| 2051 |
torch.cuda.synchronize()
|
| 2052 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2053 |
# Move VAE back to CPU
|
| 2054 |
_move(video_vae.decoder, "cpu")
|
| 2055 |
_move(audio_vae.decoder, "cpu")
|
|
@@ -2097,6 +2134,8 @@ class JoyEcho_Generate:
|
|
| 2097 |
_trim = max(_trim, _t1)
|
| 2098 |
if _trim > 0 and video_uint8.shape[0] > _trim + 16:
|
| 2099 |
video_uint8 = video_uint8[_trim:]
|
|
|
|
|
|
|
| 2100 |
if audio_waveform is not None:
|
| 2101 |
_cut = int(round(_trim / float(video_fps) * audio_sample_rate))
|
| 2102 |
if audio_waveform.shape[-1] > _cut:
|
|
@@ -2131,10 +2170,19 @@ class JoyEcho_Generate:
|
|
| 2131 |
|
| 2132 |
# Save per-shot video immediately for real-time preview (now trimmed,
|
| 2133 |
# so it matches the final output frame-for-frame).
|
| 2134 |
-
|
| 2135 |
-
|
| 2136 |
-
|
| 2137 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2138 |
|
| 2139 |
del video_latent, audio_latent, audio_memory_latent, video_uint8, audio_waveform
|
| 2140 |
_empty_cache()
|
|
@@ -2653,6 +2701,71 @@ class JoyEcho_Generate:
|
|
| 2653 |
_empty_cache()
|
| 2654 |
print(f"[JoyEcho] Hires refine pass done in {_time.time()-_t0:.0f}s.", flush=True)
|
| 2655 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2656 |
@staticmethod
|
| 2657 |
def _save_shot_video(video_uint8, audio_waveform, shot_idx, fps, audio_sr, prefix):
|
| 2658 |
"""Save a single shot as mp4 immediately after generation."""
|
|
@@ -2766,7 +2879,7 @@ class JoyEcho_SingleShotGenerate:
|
|
| 2766 |
"tooltip": "Single shot prompt text",
|
| 2767 |
}),
|
| 2768 |
"seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
|
| 2769 |
-
"num_frames": ("INT", {"default": 241, "min": 9, "max":
|
| 2770 |
"tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
|
| 2771 |
# Rebels local patch: portrait resolutions. Height was capped at
|
| 2772 |
# 1088 while width allowed 1920, which silently forbade portrait
|
|
|
|
| 1224 |
"model": ("JOYECHO_MODEL",),
|
| 1225 |
"conditioning": ("JOYECHO_COND",),
|
| 1226 |
"seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
|
| 1227 |
+
"num_frames": ("INT", {"default": 241, "min": 9, "max": 1441, "step": 8,
|
| 1228 |
"tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
|
| 1229 |
# Rebels local patch: portrait resolutions. Height was capped at
|
| 1230 |
# 1088 while width allowed 1920, which silently forbade portrait
|
|
|
|
| 1380 |
"it) - use heights whose /32 is EVEN (768, not 736) or it "
|
| 1381 |
"smears one edge.",
|
| 1382 |
}),
|
| 1383 |
+
"temporal_upscale": (["off", "2x (48 fps master)"], {
|
| 1384 |
+
"default": "off",
|
| 1385 |
+
"tooltip": "LTX temporal latent upsampler (x2) on each shot's OWN latents "
|
| 1386 |
+
"after sampling: per-shot files and the master come out at "
|
| 1387 |
+
"double fps (24 -> ~48) with the audio untouched - same "
|
| 1388 |
+
"duration, twice the motion samples, no optical-flow ghosting. "
|
| 1389 |
+
"The render itself stays at video_fps 24, so the accent/sync "
|
| 1390 |
+
"law is unaffected; the in-canvas preview also stays at base "
|
| 1391 |
+
"fps (only the saved files double). v1 limitation: requires "
|
| 1392 |
+
"hires_factor 1.0 (skipped with a warning otherwise). Keep OFF "
|
| 1393 |
+
"for found-footage looks - interpolated camcorder reads as "
|
| 1394 |
+
"soap opera.",
|
| 1395 |
+
}),
|
| 1396 |
},
|
| 1397 |
}
|
| 1398 |
|
|
|
|
| 1462 |
resident_blocks: int = 0,
|
| 1463 |
hires_factor: float = 1.0,
|
| 1464 |
hires_denoise: str = "subtle (1 step)",
|
| 1465 |
+
temporal_upscale: str = "off",
|
| 1466 |
):
|
| 1467 |
from ltx_distillation.inference.bidirectional_pipeline import BidirectionalAVInferencePipeline
|
| 1468 |
from ltx_distillation.inference.memory_bidirectional_pipeline import BidirectionalMemoryAVInferencePipeline
|
|
|
|
| 1631 |
_hires_audio_lats = []
|
| 1632 |
_hires_video_lats = [] # spatial hires mode only (~10MB/shot on CPU)
|
| 1633 |
_hires_spatial = str(hires_denoise).lower().startswith("spatial")
|
| 1634 |
+
_temporal_on = str(temporal_upscale).lower().startswith("2x")
|
| 1635 |
+
if _temporal_on and hires_factor > 1.0:
|
| 1636 |
+
print("[JoyEcho] temporal_upscale requires hires_factor 1.0 in this "
|
| 1637 |
+
"version - the hires passes re-save shots at base fps and would "
|
| 1638 |
+
"undo it. Temporal upscale SKIPPED.", flush=True)
|
| 1639 |
+
_temporal_on = False
|
| 1640 |
|
| 1641 |
num_shots = len(conditioning)
|
| 1642 |
offloader = None
|
|
|
|
| 2070 |
if device.type == "cuda":
|
| 2071 |
torch.cuda.synchronize()
|
| 2072 |
|
| 2073 |
+
# Temporal x2: second decode from the shot's OWN latents through the
|
| 2074 |
+
# LTX temporal upsampler while the decoder is still on GPU. Only the
|
| 2075 |
+
# SAVED shot files (and thus the worker-built master) get the doubled
|
| 2076 |
+
# fps; the bank, refs and in-graph frames stay base-fps native.
|
| 2077 |
+
_tu_uint8 = None
|
| 2078 |
+
if _temporal_on:
|
| 2079 |
+
try:
|
| 2080 |
+
_tu_uint8 = self._temporal_upsample_decode(
|
| 2081 |
+
video_vae, video_latent, _decode_tiling_config, device)
|
| 2082 |
+
print(f"[JoyEcho] temporal x2: shot {shot_idx+1} "
|
| 2083 |
+
f"{video_uint8.shape[0]}f -> {_tu_uint8.shape[0]}f "
|
| 2084 |
+
f"(saved at {video_fps * 2} fps).", flush=True)
|
| 2085 |
+
except Exception as _tu_e:
|
| 2086 |
+
print(f"[JoyEcho] temporal x2 FAILED on shot {shot_idx+1}: "
|
| 2087 |
+
f"{_tu_e} - saving base-fps shot instead.", flush=True)
|
| 2088 |
+
_tu_uint8 = None
|
| 2089 |
+
|
| 2090 |
# Move VAE back to CPU
|
| 2091 |
_move(video_vae.decoder, "cpu")
|
| 2092 |
_move(audio_vae.decoder, "cpu")
|
|
|
|
| 2134 |
_trim = max(_trim, _t1)
|
| 2135 |
if _trim > 0 and video_uint8.shape[0] > _trim + 16:
|
| 2136 |
video_uint8 = video_uint8[_trim:]
|
| 2137 |
+
if _tu_uint8 is not None:
|
| 2138 |
+
_tu_uint8 = _tu_uint8[2 * _trim:] # same cut in doubled frames
|
| 2139 |
if audio_waveform is not None:
|
| 2140 |
_cut = int(round(_trim / float(video_fps) * audio_sample_rate))
|
| 2141 |
if audio_waveform.shape[-1] > _cut:
|
|
|
|
| 2170 |
|
| 2171 |
# Save per-shot video immediately for real-time preview (now trimmed,
|
| 2172 |
# so it matches the final output frame-for-frame).
|
| 2173 |
+
if _tu_uint8 is not None:
|
| 2174 |
+
# doubled-fps shot file: the AutoFinish worker probes r_frame_rate
|
| 2175 |
+
# per shot, so the master assembles at the doubled rate untouched
|
| 2176 |
+
self._save_shot_video(
|
| 2177 |
+
_tu_uint8, audio_waveform, shot_idx,
|
| 2178 |
+
video_fps * 2, audio_sample_rate, output_prefix
|
| 2179 |
+
)
|
| 2180 |
+
del _tu_uint8
|
| 2181 |
+
else:
|
| 2182 |
+
self._save_shot_video(
|
| 2183 |
+
video_uint8, audio_waveform, shot_idx,
|
| 2184 |
+
video_fps, audio_sample_rate, output_prefix
|
| 2185 |
+
)
|
| 2186 |
|
| 2187 |
del video_latent, audio_latent, audio_memory_latent, video_uint8, audio_waveform
|
| 2188 |
_empty_cache()
|
|
|
|
| 2701 |
_empty_cache()
|
| 2702 |
print(f"[JoyEcho] Hires refine pass done in {_time.time()-_t0:.0f}s.", flush=True)
|
| 2703 |
|
| 2704 |
+
def _temporal_upsample_decode(self, video_vae, video_latent, tiling_config, device):
|
| 2705 |
+
"""Decode a shot's video latents through the LTX temporal x2 upsampler.
|
| 2706 |
+
|
| 2707 |
+
Mirrors the hires-spatial pass: pipeline latents [1,F,C,h,w] ->
|
| 2708 |
+
un-normalize -> reflect-pad T (the upsampler family corrupts sequence
|
| 2709 |
+
ends; pads absorb it) -> upsample -> crop pad -> re-normalize -> tiled
|
| 2710 |
+
decode. Deterministic, video-only - the audio lane is never touched.
|
| 2711 |
+
Returns uint8 frames with F_out ~= 2*F-1. Decoder must already be on
|
| 2712 |
+
`device` (call inside Phase B).
|
| 2713 |
+
"""
|
| 2714 |
+
import json as _json
|
| 2715 |
+
import folder_paths
|
| 2716 |
+
import comfy.utils as _cutils
|
| 2717 |
+
from comfy.ldm.lightricks.latent_upsampler import LatentUpsampler
|
| 2718 |
+
from ltx_core.model.video_vae import TemporalTilingConfig, TilingConfig
|
| 2719 |
+
from ltx_distillation.utils import decode_benchmark_sample
|
| 2720 |
+
|
| 2721 |
+
# The doubled sequence (~2x frames) MUST decode temporally tiled no
|
| 2722 |
+
# matter what the shot's own tiling resolved to - an untiled 449-frame
|
| 2723 |
+
# decode is exactly the uncatchable native cuDNN abort the tiled-decode
|
| 2724 |
+
# work fixed (killed the whole process on first try, 2026-07-30).
|
| 2725 |
+
tiling_config = TilingConfig(
|
| 2726 |
+
spatial_config=None,
|
| 2727 |
+
temporal_config=TemporalTilingConfig(tile_size_in_frames=64,
|
| 2728 |
+
tile_overlap_in_frames=24))
|
| 2729 |
+
|
| 2730 |
+
_dt = torch.bfloat16
|
| 2731 |
+
cls = type(self)
|
| 2732 |
+
if getattr(cls, "_tu_model", None) is None:
|
| 2733 |
+
path = folder_paths.get_full_path_or_raise(
|
| 2734 |
+
"latent_upscale_models",
|
| 2735 |
+
"ltx-2.3-temporal-upscaler-x2-1.0.safetensors")
|
| 2736 |
+
sd, metadata = _cutils.load_torch_file(
|
| 2737 |
+
path, safe_load=True, return_metadata=True)
|
| 2738 |
+
m = LatentUpsampler.from_config(
|
| 2739 |
+
_json.loads(metadata["config"])).to(dtype=_dt)
|
| 2740 |
+
m.load_state_dict(sd)
|
| 2741 |
+
m.eval()
|
| 2742 |
+
cls._tu_model = m
|
| 2743 |
+
del sd
|
| 2744 |
+
up_model = cls._tu_model.to(device)
|
| 2745 |
+
|
| 2746 |
+
stats = video_vae.decoder.per_channel_statistics # on device with decoder
|
| 2747 |
+
raw = video_latent.permute(0, 2, 1, 3, 4) # [1,C,F,h,w]
|
| 2748 |
+
raw = stats.un_normalize(raw.to(device=device, dtype=torch.float32))
|
| 2749 |
+
f_in = raw.shape[2]
|
| 2750 |
+
p = min(8, f_in - 1)
|
| 2751 |
+
if p > 0:
|
| 2752 |
+
raw = torch.cat([raw[:, :, 1:p + 1].flip(2), raw,
|
| 2753 |
+
raw[:, :, -(p + 1):-1].flip(2)], dim=2)
|
| 2754 |
+
with torch.no_grad():
|
| 2755 |
+
up = up_model(raw.to(dtype=_dt))
|
| 2756 |
+
_scale = up.shape[2] / raw.shape[2]
|
| 2757 |
+
del raw
|
| 2758 |
+
if p > 0:
|
| 2759 |
+
_po = int(round(p * _scale))
|
| 2760 |
+
up = up[:, :, _po:up.shape[2] - _po]
|
| 2761 |
+
up = stats.normalize(up.to(torch.float32)).to(_dt)
|
| 2762 |
+
up = up.permute(0, 2, 1, 3, 4).contiguous() # [1,F2,C,h,w]
|
| 2763 |
+
u8, _ = decode_benchmark_sample(video_vae, None, up.to(device), None,
|
| 2764 |
+
video_tiling_config=tiling_config)
|
| 2765 |
+
del up
|
| 2766 |
+
up_model.to("cpu")
|
| 2767 |
+
return u8
|
| 2768 |
+
|
| 2769 |
@staticmethod
|
| 2770 |
def _save_shot_video(video_uint8, audio_waveform, shot_idx, fps, audio_sr, prefix):
|
| 2771 |
"""Save a single shot as mp4 immediately after generation."""
|
|
|
|
| 2879 |
"tooltip": "Single shot prompt text",
|
| 2880 |
}),
|
| 2881 |
"seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
|
| 2882 |
+
"num_frames": ("INT", {"default": 241, "min": 9, "max": 1441, "step": 8,
|
| 2883 |
"tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
|
| 2884 |
# Rebels local patch: portrait resolutions. Height was capped at
|
| 2885 |
# 1088 while width allowed 1920, which silently forbade portrait
|
web/js/joyecho_widget_persistence.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// JoyEcho widget persistence - values survive node-layout changes.
|
| 2 |
+
//
|
| 3 |
+
// ComfyUI serializes widgets_values as a POSITIONAL array. Whenever a JoyEcho
|
| 4 |
+
// node gains/loses/reorders a widget between saves, every saved value shifts
|
| 5 |
+
// into the wrong slot or silently reverts to defaults - the "my settings reset
|
| 6 |
+
// after every update" complaint, and the reason the Generate node ships a
|
| 7 |
+
// scramble-guard error at all.
|
| 8 |
+
//
|
| 9 |
+
// Fix: on serialize, ALSO store {widgetName: value} in node.properties (a
|
| 10 |
+
// dict - layout-immune). On configure, after the stock positional restore,
|
| 11 |
+
// re-apply values BY NAME. Combo values that no longer exist in the current
|
| 12 |
+
// option list are skipped so a renamed dropdown option falls back to its
|
| 13 |
+
// default instead of erroring.
|
| 14 |
+
//
|
| 15 |
+
// First save under this extension writes the map; from then on, layout
|
| 16 |
+
// changes can never revert your settings again.
|
| 17 |
+
|
| 18 |
+
import { app } from "../../scripts/app.js";
|
| 19 |
+
|
| 20 |
+
const PREFIX = "JoyEcho_";
|
| 21 |
+
const PROP = "je_widget_values";
|
| 22 |
+
|
| 23 |
+
app.registerExtension({
|
| 24 |
+
name: "joyecho.widgetPersistence",
|
| 25 |
+
beforeRegisterNodeDef(nodeType, nodeData) {
|
| 26 |
+
if (!nodeData?.name?.startsWith(PREFIX)) return;
|
| 27 |
+
|
| 28 |
+
const origSerialize = nodeType.prototype.onSerialize;
|
| 29 |
+
nodeType.prototype.onSerialize = function (o) {
|
| 30 |
+
origSerialize?.apply(this, arguments);
|
| 31 |
+
if (!this.widgets?.length) return;
|
| 32 |
+
const map = {};
|
| 33 |
+
for (const w of this.widgets) {
|
| 34 |
+
if (w.name !== undefined && w.value !== undefined) {
|
| 35 |
+
map[w.name] = w.value;
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
o.properties = o.properties || {};
|
| 39 |
+
o.properties[PROP] = map;
|
| 40 |
+
};
|
| 41 |
+
|
| 42 |
+
const origConfigure = nodeType.prototype.onConfigure;
|
| 43 |
+
nodeType.prototype.onConfigure = function (o) {
|
| 44 |
+
origConfigure?.apply(this, arguments);
|
| 45 |
+
const saved = o?.properties?.[PROP];
|
| 46 |
+
if (!saved || !this.widgets?.length) return;
|
| 47 |
+
for (const w of this.widgets) {
|
| 48 |
+
if (!(w.name in saved)) continue;
|
| 49 |
+
const v = saved[w.name];
|
| 50 |
+
const opts = w.options?.values;
|
| 51 |
+
if (Array.isArray(opts) && !opts.includes(v)) continue; // renamed combo option
|
| 52 |
+
w.value = v;
|
| 53 |
+
w.callback?.(v);
|
| 54 |
+
}
|
| 55 |
+
};
|
| 56 |
+
},
|
| 57 |
+
});
|