joeygambino commited on
Commit
cb016a1
·
verified ·
1 Parent(s): 1d491ef

Character Cartridges (.joypack v1.0): spec, packer, loader node, and WREN - the first portable video-character file (verified: same face + rhotic General American voice in an unseen scene, zero training)

Browse files
.gitattributes CHANGED
@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  demo_one_room_q8.mp4 filter=lfs diff=lfs merge=lfs -text
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  demo_one_room_q8.mp4 filter=lfs diff=lfs merge=lfs -text
37
+ joypacks/WREN.joypack filter=lfs diff=lfs merge=lfs -text
JOYPACK_SPEC.md ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # .joypack — portable video-character cartridge (spec v1.0)
2
+
3
+ One file = one character: face, voice, canon description, and optionally the
4
+ LoRAs and environments that define them. Drop it into a JoyEcho workflow and
5
+ render new scenes with that exact character — same face, same voice — with
6
+ zero training. Designed to extend the roleplay ecosystem's Character Card
7
+ V3/CHARX lineage into photoreal joint audio-video generation.
8
+
9
+ ## Container
10
+
11
+ A `.joypack` is a ZIP archive (store or deflate) with this layout:
12
+
13
+ ```
14
+ manifest.json REQUIRED spec + inventory (see below)
15
+ persona/card.json optional CharacterCardV3-compatible JSON (chat interop)
16
+ voice/anchor.(mp4|wav) REQUIRED 4s+ clip of the character speaking.
17
+ Seeds the JoyEcho memory bank before
18
+ shot 1 - the voice continues instead of
19
+ being re-rolled. mp4 preferred (pairs
20
+ audio with a moving reference face).
21
+ refs/*.png|jpg REQUIRED 1-8 MSR identity reference stills,
22
+ front-lit face clearly visible.
23
+ prompts/dna.txt REQUIRED the canonical DNA sentence(s): the
24
+ verbatim identity text to carry in every
25
+ shot prompt, plus voice scaffold line
26
+ (timbre + accent binding).
27
+ loras/zimage/*.safetensors optional face LoRA for first-frame/still work
28
+ loras/ltx/*.safetensors optional LTX-2.3 LoRA(s) (ID-LoRA, style)
29
+ environment/*.png|jpg optional canonical location stills
30
+ environment/rooms.txt optional verbatim room description(s) - the
31
+ location-lock prose, one per paragraph
32
+ ```
33
+
34
+ ## manifest.json
35
+
36
+ ```json
37
+ {
38
+ "joypack": "1.0",
39
+ "name": "MARA",
40
+ "speaker_tag": "mara",
41
+ "display_name": "Detective Mara",
42
+ "authors": ["you"],
43
+ "license": "CC-BY-4.0",
44
+ "notes": "freeform",
45
+ "voice": {"file": "voice/anchor.mp4", "accent_line": "speaking in a casual American accent"},
46
+ "refs": ["refs/mara_01.png", "refs/mara_02.png"],
47
+ "dna": "prompts/dna.txt",
48
+ "loras": {
49
+ "zimage": [{"file": "loras/zimage/mara_z.safetensors", "trigger": "mara_z", "strength": 0.7}],
50
+ "ltx": [{"file": "loras/ltx/mara_id.safetensors", "strength": 1.0}]
51
+ },
52
+ "environment": {"stills": ["environment/apartment_01.png"], "rooms": "environment/rooms.txt"},
53
+ "render_law": {"video_fps": 24},
54
+ "sha256": {"voice/anchor.mp4": "..."}
55
+ }
56
+ ```
57
+
58
+ Only `joypack`, `name`, `speaker_tag`, `voice`, `refs`, and `dna` are required.
59
+ Unknown keys MUST be ignored (forward compatibility). `render_law` records
60
+ generation constraints the character was authored under - loaders should warn
61
+ when the live workflow violates them (the fps/accent law above all).
62
+
63
+ ## Loader behavior (reference implementation: JoyEcho_CartridgeLoader)
64
+
65
+ Loading MATERIALIZES the cartridge into the host's existing conventions -
66
+ the render path itself needs no changes:
67
+
68
+ 1. Unpack to a cache dir keyed by content hash.
69
+ 2. Copy `voice/anchor.*` to `input/joyecho_voices/<speaker_tag>/` (folder
70
+ auto-casting picks it up; every script whose speaker tag matches is voiced
71
+ by the anchor).
72
+ 3. Copy `refs/*` to the workflow's refs root under `<speaker_tag>/` (RefPicker
73
+ finds them by character scan).
74
+ 4. Copy LoRAs to `models/loras/joypack/<name>/` and report their names for the
75
+ LoRA stack.
76
+ 5. Output the DNA text, accent line, room text, and trigger words as node
77
+ outputs for prompt assembly.
78
+ 6. Verify sha256 entries when present; refuse a cartridge whose manifest is
79
+ missing required keys.
80
+
81
+ Removal = delete the materialized folders; a cartridge never modifies the
82
+ host beyond those drop-in locations.
83
+
84
+ ## Security
85
+
86
+ A cartridge is DATA. Loaders MUST NOT execute anything from the archive, MUST
87
+ reject path traversal (entries containing `..` or absolute paths), and should
88
+ treat persona/card.json purely as chat-client payload. LoRA files are weights;
89
+ standard safetensors-only policy applies (no pickle formats).
90
+
91
+ ## Versioning
92
+
93
+ `joypack: "1.0"` this document. Planned 1.1: serialized memory-bank state
94
+ (`bank/state.joybank`) for scene-warm characters; multi-character packs.
__init__.py CHANGED
@@ -159,4 +159,16 @@ try:
159
  except Exception as e:
160
  print(f"[Rebels JE] joyecho_autofinish failed to load: {e!r}", flush=True)
161
 
162
- __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  except Exception as e:
160
  print(f"[Rebels JE] joyecho_autofinish failed to load: {e!r}", flush=True)
161
 
162
+ try:
163
+ from .joyecho_cartridge import (
164
+ NODE_CLASS_MAPPINGS as _CG_CM,
165
+ NODE_DISPLAY_NAME_MAPPINGS as _CG_DM,
166
+ )
167
+ NODE_CLASS_MAPPINGS.update(_CG_CM)
168
+ NODE_DISPLAY_NAME_MAPPINGS.update(_CG_DM)
169
+ except Exception as _e:
170
+ print(f"[JoyAI-Echo] cartridge loader unavailable: {_e}")
171
+
172
+ WEB_DIRECTORY = "./web/js"
173
+
174
+ __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
joyecho_cartridge.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JoyEcho_CartridgeLoader - load a .joypack character cartridge.
2
+
3
+ Materializes the cartridge into the host's existing conventions (voice anchor
4
+ into input/joyecho_voices/<tag>/, reference stills into the refs root, LoRAs
5
+ into models/loras/joypack/<name>/) and outputs the character's texts for
6
+ prompt assembly. The render path needs no changes - after loading, any script
7
+ whose speaker tag matches is voiced and identified as the cartridge character.
8
+
9
+ See JOYPACK_SPEC.md for the format.
10
+ """
11
+ import json
12
+ import os
13
+
14
+ try:
15
+ import folder_paths
16
+ except Exception:
17
+ folder_paths = None
18
+
19
+ from .joypack import materialize, read_manifest, JoypackError
20
+
21
+
22
+ def _packs_dir():
23
+ if folder_paths:
24
+ base = folder_paths.get_input_directory()
25
+ else:
26
+ base = os.path.join(os.getcwd(), "input")
27
+ d = os.path.join(base, "joypacks")
28
+ os.makedirs(d, exist_ok=True)
29
+ return d
30
+
31
+
32
+ class JoyEcho_CartridgeLoader:
33
+ @classmethod
34
+ def INPUT_TYPES(cls):
35
+ packs = ["(none)"]
36
+ try:
37
+ packs += sorted(f for f in os.listdir(_packs_dir())
38
+ if f.lower().endswith(".joypack"))
39
+ except Exception:
40
+ pass
41
+ return {
42
+ "required": {
43
+ "cartridge": (packs, {
44
+ "tooltip": "A .joypack from input/joypacks/. Loading installs "
45
+ "the character's voice anchor, reference stills and "
46
+ "LoRAs into this workflow's normal folders - any "
47
+ "script whose speaker tag matches renders as this "
48
+ "character. See JOYPACK_SPEC.md."}),
49
+ "refs_root": ("STRING", {"default": "joyecho_refs", "tooltip":
50
+ "Where reference stills install (RefPicker's root). "
51
+ "Absolute path or relative to the input folder."}),
52
+ },
53
+ }
54
+
55
+ RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "STRING")
56
+ RETURN_NAMES = ("speaker_tag", "dna_text", "accent_line", "rooms_text", "report")
57
+ FUNCTION = "load"
58
+ CATEGORY = "JoyAI-Echo"
59
+
60
+ def load(self, cartridge, refs_root="joyecho_refs"):
61
+ if cartridge == "(none)":
62
+ return ("", "", "", "", "no cartridge selected")
63
+ pack_path = os.path.join(_packs_dir(), cartridge)
64
+
65
+ in_dir = (folder_paths.get_input_directory() if folder_paths
66
+ else os.path.join(os.getcwd(), "input"))
67
+ voices_dir = os.path.join(in_dir, "joyecho_voices")
68
+ refs_dir = (refs_root if os.path.isabs(refs_root)
69
+ else os.path.join(in_dir, refs_root))
70
+ if folder_paths:
71
+ loras_dir = os.path.join(folder_paths.models_dir, "loras", "joypack")
72
+ else:
73
+ loras_dir = os.path.join(os.getcwd(), "models", "loras", "joypack")
74
+ cache_dir = os.path.join(in_dir, "joypacks", "_cache")
75
+
76
+ try:
77
+ res = materialize(pack_path, voices_dir, refs_dir, loras_dir, cache_dir)
78
+ except (JoypackError, OSError) as e:
79
+ raise ValueError(f"joypack load failed: {e}")
80
+
81
+ law = res.get("render_law", {})
82
+ law_note = ""
83
+ if law.get("video_fps"):
84
+ law_note = (f" | render law: video_fps={law['video_fps']} "
85
+ f"(authored under the 24fps accent law)" if law["video_fps"] == 24
86
+ else f" | render law: video_fps={law['video_fps']}")
87
+ loras_note = ""
88
+ if res["ltx_loras"]:
89
+ loras_note = " | LTX loras: " + ", ".join(
90
+ f"{os.path.basename(l['path'])}@{l['strength']}" for l in res["ltx_loras"])
91
+ report = (f"loaded '{res['name']}' as speaker '{res['speaker_tag']}' - "
92
+ f"voice anchor + {sum(1 for p in res['installed_paths'] if refs_dir in p)} "
93
+ f"ref(s) installed{loras_note}{law_note}")
94
+ print(f"[JoyEcho] Cartridge: {report}", flush=True)
95
+
96
+ dna = res["dna"]
97
+ if res.get("triggers"):
98
+ dna = dna # triggers are for Z-Image prompts; reported, not injected
99
+ return (res["speaker_tag"], dna, res.get("accent_line", ""),
100
+ res.get("rooms", ""), report)
101
+
102
+
103
+ NODE_CLASS_MAPPINGS = {"JoyEcho_CartridgeLoader": JoyEcho_CartridgeLoader}
104
+ NODE_DISPLAY_NAME_MAPPINGS = {"JoyEcho_CartridgeLoader": "JoyEcho Cartridge Loader (.joypack)"}
joypack.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """joypack - portable video-character cartridges (.joypack), spec v1.0.
2
+
3
+ Library + CLI. See JOYPACK_SPEC.md for the format. A .joypack is a plain zip;
4
+ this module packs one from a source folder and safely unpacks/materializes one
5
+ into the JoyEcho host conventions (joyecho_voices/<tag>/, refs root, loras).
6
+
7
+ CLI:
8
+ python joypack.py pack <source_dir> <out.joypack>
9
+ python joypack.py inspect <file.joypack>
10
+
11
+ Security: archives are DATA. No execution, no path traversal (rejected), no
12
+ pickle weight formats (safetensors only for loras/).
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import hashlib
17
+ import json
18
+ import os
19
+ import shutil
20
+ import sys
21
+ import zipfile
22
+
23
+ SPEC_VERSION = "1.0"
24
+ REQUIRED_KEYS = ("joypack", "name", "speaker_tag", "voice", "refs", "dna")
25
+ VOICE_EXTS = (".mp4", ".wav")
26
+ REF_EXTS = (".png", ".jpg", ".jpeg", ".webp")
27
+
28
+
29
+ class JoypackError(ValueError):
30
+ pass
31
+
32
+
33
+ def _sha256(path):
34
+ h = hashlib.sha256()
35
+ with open(path, "rb") as f:
36
+ for chunk in iter(lambda: f.read(1 << 20), b""):
37
+ h.update(chunk)
38
+ return h.hexdigest()
39
+
40
+
41
+ def _safe_name(entry_name):
42
+ """Reject traversal and absolute paths; return normalized relative path."""
43
+ n = entry_name.replace("\\", "/")
44
+ if n.startswith("/") or ".." in n.split("/") or (len(n) > 1 and n[1] == ":"):
45
+ raise JoypackError(f"unsafe path in archive: {entry_name!r}")
46
+ return n
47
+
48
+
49
+ def pack(source_dir, out_path):
50
+ """Build a .joypack from a folder laid out per the spec.
51
+
52
+ If the folder has no manifest.json, a minimal one is synthesized from the
53
+ layout (name = folder name, speaker_tag = lowercased name).
54
+ """
55
+ source_dir = os.path.abspath(source_dir)
56
+ man_path = os.path.join(source_dir, "manifest.json")
57
+ if os.path.isfile(man_path):
58
+ manifest = json.load(open(man_path, encoding="utf-8"))
59
+ else:
60
+ name = os.path.basename(source_dir.rstrip("\\/"))
61
+ manifest = {"joypack": SPEC_VERSION, "name": name,
62
+ "speaker_tag": name.lower()}
63
+
64
+ # discover required components if unlisted
65
+ def _find(sub, exts):
66
+ d = os.path.join(source_dir, sub)
67
+ if not os.path.isdir(d):
68
+ return []
69
+ return sorted(f"{sub}/{f}" for f in os.listdir(d)
70
+ if f.lower().endswith(exts))
71
+
72
+ if "voice" not in manifest or not manifest["voice"].get("file"):
73
+ vs = _find("voice", VOICE_EXTS)
74
+ if not vs:
75
+ raise JoypackError("no voice anchor found (voice/anchor.mp4|wav)")
76
+ manifest.setdefault("voice", {})["file"] = vs[0]
77
+ if not manifest.get("refs"):
78
+ manifest["refs"] = _find("refs", REF_EXTS)
79
+ if not manifest["refs"]:
80
+ raise JoypackError("no reference stills found (refs/*.png)")
81
+ if not manifest.get("dna"):
82
+ p = "prompts/dna.txt"
83
+ if not os.path.isfile(os.path.join(source_dir, p)):
84
+ raise JoypackError("no DNA text found (prompts/dna.txt)")
85
+ manifest["dna"] = p
86
+ manifest.setdefault("render_law", {"video_fps": 24})
87
+
88
+ # collect every real file under the spec dirs + the manifest
89
+ files = []
90
+ for root, _, names in os.walk(source_dir):
91
+ for fn in names:
92
+ full = os.path.join(root, fn)
93
+ rel = os.path.relpath(full, source_dir).replace("\\", "/")
94
+ if rel == "manifest.json":
95
+ continue
96
+ if rel.startswith("loras/") and not fn.lower().endswith(".safetensors"):
97
+ raise JoypackError(f"loras/ may only contain .safetensors: {rel}")
98
+ files.append(rel)
99
+
100
+ manifest["sha256"] = {manifest["voice"]["file"]:
101
+ _sha256(os.path.join(source_dir, manifest["voice"]["file"]))}
102
+ for k in REQUIRED_KEYS:
103
+ if k not in manifest:
104
+ raise JoypackError(f"manifest missing required key: {k}")
105
+
106
+ with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z:
107
+ z.writestr("manifest.json", json.dumps(manifest, indent=1))
108
+ for rel in files:
109
+ z.write(os.path.join(source_dir, rel), rel)
110
+ return out_path, manifest
111
+
112
+
113
+ def read_manifest(pack_path):
114
+ with zipfile.ZipFile(pack_path) as z:
115
+ try:
116
+ manifest = json.loads(z.read("manifest.json").decode("utf-8"))
117
+ except KeyError:
118
+ raise JoypackError("not a joypack: manifest.json missing")
119
+ for k in REQUIRED_KEYS:
120
+ if k not in manifest:
121
+ raise JoypackError(f"manifest missing required key: {k}")
122
+ return manifest
123
+
124
+
125
+ def materialize(pack_path, voices_dir, refs_dir, loras_dir, cache_dir):
126
+ """Unpack + install a cartridge into host conventions.
127
+
128
+ Returns a dict: {name, speaker_tag, dna, accent_line, rooms, triggers,
129
+ ltx_loras, zimage_loras, installed_paths}
130
+ """
131
+ manifest = read_manifest(pack_path)
132
+ tag = manifest["speaker_tag"]
133
+ name = manifest["name"]
134
+ out = {"name": name, "speaker_tag": tag, "installed_paths": []}
135
+
136
+ with zipfile.ZipFile(pack_path) as z:
137
+ names = {_safe_name(n) for n in z.namelist()}
138
+
139
+ def _extract(member, dst_dir, dst_name=None):
140
+ member = _safe_name(member)
141
+ if member not in names:
142
+ raise JoypackError(f"manifest references missing file: {member}")
143
+ os.makedirs(dst_dir, exist_ok=True)
144
+ dst = os.path.join(dst_dir, dst_name or os.path.basename(member))
145
+ with z.open(member) as src, open(dst, "wb") as f:
146
+ shutil.copyfileobj(src, f)
147
+ out["installed_paths"].append(dst)
148
+ return dst
149
+
150
+ # voice anchor -> joyecho_voices/<tag>/
151
+ vfile = manifest["voice"]["file"]
152
+ want = manifest.get("sha256", {}).get(vfile)
153
+ vdst = _extract(vfile, os.path.join(voices_dir, tag))
154
+ if want and _sha256(vdst) != want:
155
+ os.remove(vdst)
156
+ raise JoypackError("voice anchor failed sha256 verification")
157
+
158
+ # refs -> <refs_root>/<NAME>/
159
+ for r in manifest["refs"]:
160
+ _extract(r, os.path.join(refs_dir, name))
161
+
162
+ # loras -> models/loras/joypack/<name>/
163
+ out["ltx_loras"], out["zimage_loras"] = [], []
164
+ loras = manifest.get("loras", {})
165
+ for fam, key in (("ltx", "ltx_loras"), ("zimage", "zimage_loras")):
166
+ for entry in loras.get(fam, []) or []:
167
+ dst = _extract(entry["file"], os.path.join(loras_dir, name))
168
+ out[key].append({"path": dst,
169
+ "strength": float(entry.get("strength", 1.0)),
170
+ "trigger": entry.get("trigger", "")})
171
+
172
+ # texts
173
+ out["dna"] = z.read(_safe_name(manifest["dna"])).decode("utf-8").strip()
174
+ out["accent_line"] = manifest.get("voice", {}).get("accent_line", "")
175
+ rooms = manifest.get("environment", {}).get("rooms")
176
+ out["rooms"] = (z.read(_safe_name(rooms)).decode("utf-8").strip()
177
+ if rooms and _safe_name(rooms) in names else "")
178
+ env_stills = manifest.get("environment", {}).get("stills", []) or []
179
+ for s in env_stills:
180
+ _extract(s, os.path.join(cache_dir, name, "environment"))
181
+
182
+ out["render_law"] = manifest.get("render_law", {})
183
+ out["triggers"] = [e.get("trigger", "") for e in
184
+ (manifest.get("loras", {}).get("zimage", []) or []) if e.get("trigger")]
185
+ return out
186
+
187
+
188
+ def main():
189
+ if len(sys.argv) < 3:
190
+ print(__doc__)
191
+ return 1
192
+ cmd = sys.argv[1]
193
+ if cmd == "pack":
194
+ out, man = pack(sys.argv[2], sys.argv[3])
195
+ print(f"packed {man['name']} -> {out}")
196
+ print(f" speaker_tag={man['speaker_tag']} refs={len(man['refs'])} "
197
+ f"voice={man['voice']['file']}")
198
+ elif cmd == "inspect":
199
+ man = read_manifest(sys.argv[2])
200
+ print(json.dumps(man, indent=1))
201
+ else:
202
+ print(__doc__)
203
+ return 1
204
+ return 0
205
+
206
+
207
+ if __name__ == "__main__":
208
+ raise SystemExit(main())
joypacks/WREN.joypack ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:664ec73cbd0d52088765ae87f584407993c4ad308aea539f8e0c2f99b1ed5652
3
+ size 2682926
nodes.py CHANGED
@@ -1224,7 +1224,7 @@ class JoyEcho_Generate:
1224
  "model": ("JOYECHO_MODEL",),
1225
  "conditioning": ("JOYECHO_COND",),
1226
  "seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
1227
- "num_frames": ("INT", {"default": 241, "min": 9, "max": 481, "step": 8,
1228
  "tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
1229
  # Rebels local patch: portrait resolutions. Height was capped at
1230
  # 1088 while width allowed 1920, which silently forbade portrait
@@ -1380,6 +1380,19 @@ class JoyEcho_Generate:
1380
  "it) - use heights whose /32 is EVEN (768, not 736) or it "
1381
  "smears one edge.",
1382
  }),
 
 
 
 
 
 
 
 
 
 
 
 
 
1383
  },
1384
  }
1385
 
@@ -1449,6 +1462,7 @@ class JoyEcho_Generate:
1449
  resident_blocks: int = 0,
1450
  hires_factor: float = 1.0,
1451
  hires_denoise: str = "subtle (1 step)",
 
1452
  ):
1453
  from ltx_distillation.inference.bidirectional_pipeline import BidirectionalAVInferencePipeline
1454
  from ltx_distillation.inference.memory_bidirectional_pipeline import BidirectionalMemoryAVInferencePipeline
@@ -1617,6 +1631,12 @@ class JoyEcho_Generate:
1617
  _hires_audio_lats = []
1618
  _hires_video_lats = [] # spatial hires mode only (~10MB/shot on CPU)
1619
  _hires_spatial = str(hires_denoise).lower().startswith("spatial")
 
 
 
 
 
 
1620
 
1621
  num_shots = len(conditioning)
1622
  offloader = None
@@ -2050,6 +2070,23 @@ class JoyEcho_Generate:
2050
  if device.type == "cuda":
2051
  torch.cuda.synchronize()
2052
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2053
  # Move VAE back to CPU
2054
  _move(video_vae.decoder, "cpu")
2055
  _move(audio_vae.decoder, "cpu")
@@ -2097,6 +2134,8 @@ class JoyEcho_Generate:
2097
  _trim = max(_trim, _t1)
2098
  if _trim > 0 and video_uint8.shape[0] > _trim + 16:
2099
  video_uint8 = video_uint8[_trim:]
 
 
2100
  if audio_waveform is not None:
2101
  _cut = int(round(_trim / float(video_fps) * audio_sample_rate))
2102
  if audio_waveform.shape[-1] > _cut:
@@ -2131,10 +2170,19 @@ class JoyEcho_Generate:
2131
 
2132
  # Save per-shot video immediately for real-time preview (now trimmed,
2133
  # so it matches the final output frame-for-frame).
2134
- self._save_shot_video(
2135
- video_uint8, audio_waveform, shot_idx,
2136
- video_fps, audio_sample_rate, output_prefix
2137
- )
 
 
 
 
 
 
 
 
 
2138
 
2139
  del video_latent, audio_latent, audio_memory_latent, video_uint8, audio_waveform
2140
  _empty_cache()
@@ -2653,6 +2701,71 @@ class JoyEcho_Generate:
2653
  _empty_cache()
2654
  print(f"[JoyEcho] Hires refine pass done in {_time.time()-_t0:.0f}s.", flush=True)
2655
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2656
  @staticmethod
2657
  def _save_shot_video(video_uint8, audio_waveform, shot_idx, fps, audio_sr, prefix):
2658
  """Save a single shot as mp4 immediately after generation."""
@@ -2766,7 +2879,7 @@ class JoyEcho_SingleShotGenerate:
2766
  "tooltip": "Single shot prompt text",
2767
  }),
2768
  "seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
2769
- "num_frames": ("INT", {"default": 241, "min": 9, "max": 481, "step": 8,
2770
  "tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
2771
  # Rebels local patch: portrait resolutions. Height was capped at
2772
  # 1088 while width allowed 1920, which silently forbade portrait
 
1224
  "model": ("JOYECHO_MODEL",),
1225
  "conditioning": ("JOYECHO_COND",),
1226
  "seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
1227
+ "num_frames": ("INT", {"default": 241, "min": 9, "max": 1441, "step": 8,
1228
  "tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
1229
  # Rebels local patch: portrait resolutions. Height was capped at
1230
  # 1088 while width allowed 1920, which silently forbade portrait
 
1380
  "it) - use heights whose /32 is EVEN (768, not 736) or it "
1381
  "smears one edge.",
1382
  }),
1383
+ "temporal_upscale": (["off", "2x (48 fps master)"], {
1384
+ "default": "off",
1385
+ "tooltip": "LTX temporal latent upsampler (x2) on each shot's OWN latents "
1386
+ "after sampling: per-shot files and the master come out at "
1387
+ "double fps (24 -> ~48) with the audio untouched - same "
1388
+ "duration, twice the motion samples, no optical-flow ghosting. "
1389
+ "The render itself stays at video_fps 24, so the accent/sync "
1390
+ "law is unaffected; the in-canvas preview also stays at base "
1391
+ "fps (only the saved files double). v1 limitation: requires "
1392
+ "hires_factor 1.0 (skipped with a warning otherwise). Keep OFF "
1393
+ "for found-footage looks - interpolated camcorder reads as "
1394
+ "soap opera.",
1395
+ }),
1396
  },
1397
  }
1398
 
 
1462
  resident_blocks: int = 0,
1463
  hires_factor: float = 1.0,
1464
  hires_denoise: str = "subtle (1 step)",
1465
+ temporal_upscale: str = "off",
1466
  ):
1467
  from ltx_distillation.inference.bidirectional_pipeline import BidirectionalAVInferencePipeline
1468
  from ltx_distillation.inference.memory_bidirectional_pipeline import BidirectionalMemoryAVInferencePipeline
 
1631
  _hires_audio_lats = []
1632
  _hires_video_lats = [] # spatial hires mode only (~10MB/shot on CPU)
1633
  _hires_spatial = str(hires_denoise).lower().startswith("spatial")
1634
+ _temporal_on = str(temporal_upscale).lower().startswith("2x")
1635
+ if _temporal_on and hires_factor > 1.0:
1636
+ print("[JoyEcho] temporal_upscale requires hires_factor 1.0 in this "
1637
+ "version - the hires passes re-save shots at base fps and would "
1638
+ "undo it. Temporal upscale SKIPPED.", flush=True)
1639
+ _temporal_on = False
1640
 
1641
  num_shots = len(conditioning)
1642
  offloader = None
 
2070
  if device.type == "cuda":
2071
  torch.cuda.synchronize()
2072
 
2073
+ # Temporal x2: second decode from the shot's OWN latents through the
2074
+ # LTX temporal upsampler while the decoder is still on GPU. Only the
2075
+ # SAVED shot files (and thus the worker-built master) get the doubled
2076
+ # fps; the bank, refs and in-graph frames stay base-fps native.
2077
+ _tu_uint8 = None
2078
+ if _temporal_on:
2079
+ try:
2080
+ _tu_uint8 = self._temporal_upsample_decode(
2081
+ video_vae, video_latent, _decode_tiling_config, device)
2082
+ print(f"[JoyEcho] temporal x2: shot {shot_idx+1} "
2083
+ f"{video_uint8.shape[0]}f -> {_tu_uint8.shape[0]}f "
2084
+ f"(saved at {video_fps * 2} fps).", flush=True)
2085
+ except Exception as _tu_e:
2086
+ print(f"[JoyEcho] temporal x2 FAILED on shot {shot_idx+1}: "
2087
+ f"{_tu_e} - saving base-fps shot instead.", flush=True)
2088
+ _tu_uint8 = None
2089
+
2090
  # Move VAE back to CPU
2091
  _move(video_vae.decoder, "cpu")
2092
  _move(audio_vae.decoder, "cpu")
 
2134
  _trim = max(_trim, _t1)
2135
  if _trim > 0 and video_uint8.shape[0] > _trim + 16:
2136
  video_uint8 = video_uint8[_trim:]
2137
+ if _tu_uint8 is not None:
2138
+ _tu_uint8 = _tu_uint8[2 * _trim:] # same cut in doubled frames
2139
  if audio_waveform is not None:
2140
  _cut = int(round(_trim / float(video_fps) * audio_sample_rate))
2141
  if audio_waveform.shape[-1] > _cut:
 
2170
 
2171
  # Save per-shot video immediately for real-time preview (now trimmed,
2172
  # so it matches the final output frame-for-frame).
2173
+ if _tu_uint8 is not None:
2174
+ # doubled-fps shot file: the AutoFinish worker probes r_frame_rate
2175
+ # per shot, so the master assembles at the doubled rate untouched
2176
+ self._save_shot_video(
2177
+ _tu_uint8, audio_waveform, shot_idx,
2178
+ video_fps * 2, audio_sample_rate, output_prefix
2179
+ )
2180
+ del _tu_uint8
2181
+ else:
2182
+ self._save_shot_video(
2183
+ video_uint8, audio_waveform, shot_idx,
2184
+ video_fps, audio_sample_rate, output_prefix
2185
+ )
2186
 
2187
  del video_latent, audio_latent, audio_memory_latent, video_uint8, audio_waveform
2188
  _empty_cache()
 
2701
  _empty_cache()
2702
  print(f"[JoyEcho] Hires refine pass done in {_time.time()-_t0:.0f}s.", flush=True)
2703
 
2704
+ def _temporal_upsample_decode(self, video_vae, video_latent, tiling_config, device):
2705
+ """Decode a shot's video latents through the LTX temporal x2 upsampler.
2706
+
2707
+ Mirrors the hires-spatial pass: pipeline latents [1,F,C,h,w] ->
2708
+ un-normalize -> reflect-pad T (the upsampler family corrupts sequence
2709
+ ends; pads absorb it) -> upsample -> crop pad -> re-normalize -> tiled
2710
+ decode. Deterministic, video-only - the audio lane is never touched.
2711
+ Returns uint8 frames with F_out ~= 2*F-1. Decoder must already be on
2712
+ `device` (call inside Phase B).
2713
+ """
2714
+ import json as _json
2715
+ import folder_paths
2716
+ import comfy.utils as _cutils
2717
+ from comfy.ldm.lightricks.latent_upsampler import LatentUpsampler
2718
+ from ltx_core.model.video_vae import TemporalTilingConfig, TilingConfig
2719
+ from ltx_distillation.utils import decode_benchmark_sample
2720
+
2721
+ # The doubled sequence (~2x frames) MUST decode temporally tiled no
2722
+ # matter what the shot's own tiling resolved to - an untiled 449-frame
2723
+ # decode is exactly the uncatchable native cuDNN abort the tiled-decode
2724
+ # work fixed (killed the whole process on first try, 2026-07-30).
2725
+ tiling_config = TilingConfig(
2726
+ spatial_config=None,
2727
+ temporal_config=TemporalTilingConfig(tile_size_in_frames=64,
2728
+ tile_overlap_in_frames=24))
2729
+
2730
+ _dt = torch.bfloat16
2731
+ cls = type(self)
2732
+ if getattr(cls, "_tu_model", None) is None:
2733
+ path = folder_paths.get_full_path_or_raise(
2734
+ "latent_upscale_models",
2735
+ "ltx-2.3-temporal-upscaler-x2-1.0.safetensors")
2736
+ sd, metadata = _cutils.load_torch_file(
2737
+ path, safe_load=True, return_metadata=True)
2738
+ m = LatentUpsampler.from_config(
2739
+ _json.loads(metadata["config"])).to(dtype=_dt)
2740
+ m.load_state_dict(sd)
2741
+ m.eval()
2742
+ cls._tu_model = m
2743
+ del sd
2744
+ up_model = cls._tu_model.to(device)
2745
+
2746
+ stats = video_vae.decoder.per_channel_statistics # on device with decoder
2747
+ raw = video_latent.permute(0, 2, 1, 3, 4) # [1,C,F,h,w]
2748
+ raw = stats.un_normalize(raw.to(device=device, dtype=torch.float32))
2749
+ f_in = raw.shape[2]
2750
+ p = min(8, f_in - 1)
2751
+ if p > 0:
2752
+ raw = torch.cat([raw[:, :, 1:p + 1].flip(2), raw,
2753
+ raw[:, :, -(p + 1):-1].flip(2)], dim=2)
2754
+ with torch.no_grad():
2755
+ up = up_model(raw.to(dtype=_dt))
2756
+ _scale = up.shape[2] / raw.shape[2]
2757
+ del raw
2758
+ if p > 0:
2759
+ _po = int(round(p * _scale))
2760
+ up = up[:, :, _po:up.shape[2] - _po]
2761
+ up = stats.normalize(up.to(torch.float32)).to(_dt)
2762
+ up = up.permute(0, 2, 1, 3, 4).contiguous() # [1,F2,C,h,w]
2763
+ u8, _ = decode_benchmark_sample(video_vae, None, up.to(device), None,
2764
+ video_tiling_config=tiling_config)
2765
+ del up
2766
+ up_model.to("cpu")
2767
+ return u8
2768
+
2769
  @staticmethod
2770
  def _save_shot_video(video_uint8, audio_waveform, shot_idx, fps, audio_sr, prefix):
2771
  """Save a single shot as mp4 immediately after generation."""
 
2879
  "tooltip": "Single shot prompt text",
2880
  }),
2881
  "seed": ("INT", {"default": 12345, "min": 0, "max": 2**31 - 1}),
2882
+ "num_frames": ("INT", {"default": 241, "min": 9, "max": 1441, "step": 8,
2883
  "tooltip": "Must be 1 + 8*k (e.g. 121, 241, 361)"}),
2884
  # Rebels local patch: portrait resolutions. Height was capped at
2885
  # 1088 while width allowed 1920, which silently forbade portrait
web/js/joyecho_widget_persistence.js ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // JoyEcho widget persistence - values survive node-layout changes.
2
+ //
3
+ // ComfyUI serializes widgets_values as a POSITIONAL array. Whenever a JoyEcho
4
+ // node gains/loses/reorders a widget between saves, every saved value shifts
5
+ // into the wrong slot or silently reverts to defaults - the "my settings reset
6
+ // after every update" complaint, and the reason the Generate node ships a
7
+ // scramble-guard error at all.
8
+ //
9
+ // Fix: on serialize, ALSO store {widgetName: value} in node.properties (a
10
+ // dict - layout-immune). On configure, after the stock positional restore,
11
+ // re-apply values BY NAME. Combo values that no longer exist in the current
12
+ // option list are skipped so a renamed dropdown option falls back to its
13
+ // default instead of erroring.
14
+ //
15
+ // First save under this extension writes the map; from then on, layout
16
+ // changes can never revert your settings again.
17
+
18
+ import { app } from "../../scripts/app.js";
19
+
20
+ const PREFIX = "JoyEcho_";
21
+ const PROP = "je_widget_values";
22
+
23
+ app.registerExtension({
24
+ name: "joyecho.widgetPersistence",
25
+ beforeRegisterNodeDef(nodeType, nodeData) {
26
+ if (!nodeData?.name?.startsWith(PREFIX)) return;
27
+
28
+ const origSerialize = nodeType.prototype.onSerialize;
29
+ nodeType.prototype.onSerialize = function (o) {
30
+ origSerialize?.apply(this, arguments);
31
+ if (!this.widgets?.length) return;
32
+ const map = {};
33
+ for (const w of this.widgets) {
34
+ if (w.name !== undefined && w.value !== undefined) {
35
+ map[w.name] = w.value;
36
+ }
37
+ }
38
+ o.properties = o.properties || {};
39
+ o.properties[PROP] = map;
40
+ };
41
+
42
+ const origConfigure = nodeType.prototype.onConfigure;
43
+ nodeType.prototype.onConfigure = function (o) {
44
+ origConfigure?.apply(this, arguments);
45
+ const saved = o?.properties?.[PROP];
46
+ if (!saved || !this.widgets?.length) return;
47
+ for (const w of this.widgets) {
48
+ if (!(w.name in saved)) continue;
49
+ const v = saved[w.name];
50
+ const opts = w.options?.values;
51
+ if (Array.isArray(opts) && !opts.includes(v)) continue; // renamed combo option
52
+ w.value = v;
53
+ w.callback?.(v);
54
+ }
55
+ };
56
+ },
57
+ });