fix: video RoPE clock uses the render fps (hardcoded 24 at 25fps renders = 4%/s AV rope skew = the ~10s lip-sync drift)
Browse files- libs/ltx_distillation/models/ltx_wrapper.py +7 -2
- nodes.py +9 -0
libs/ltx_distillation/models/ltx_wrapper.py
CHANGED
|
@@ -231,8 +231,13 @@ class LTX2DiffusionWrapper(nn.Module):
|
|
| 231 |
causal_fix=True,
|
| 232 |
).float()
|
| 233 |
|
| 234 |
-
# Convert temporal dimension from frames to seconds
|
| 235 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
pixel_coords[:, 0, ...] = pixel_coords[:, 0, ...] / self.VIDEO_FPS
|
| 237 |
|
| 238 |
if downscale_factor != 1:
|
|
|
|
| 231 |
causal_fix=True,
|
| 232 |
).float()
|
| 233 |
|
| 234 |
+
# Convert temporal dimension from frames to seconds. self.VIDEO_FPS
|
| 235 |
+
# MUST be the actual render fps - the JoyEcho nodes stamp it as an
|
| 236 |
+
# instance attribute before generation. The class default of 24.0 with
|
| 237 |
+
# 25fps renders skewed video rope-time 4% fast against audio's
|
| 238 |
+
# true-seconds rope: a growing mouth-ahead-of-audio drift that crossed
|
| 239 |
+
# visibility at ~9.6s into every shot (the "10s lip-sync cliff",
|
| 240 |
+
# diagnosed 2026-07-23).
|
| 241 |
pixel_coords[:, 0, ...] = pixel_coords[:, 0, ...] / self.VIDEO_FPS
|
| 242 |
|
| 243 |
if downscale_factor != 1:
|
nodes.py
CHANGED
|
@@ -1287,6 +1287,12 @@ class JoyEcho_Generate:
|
|
| 1287 |
generator.latent_height = video_height // 32
|
| 1288 |
generator.latent_width = video_width // 32
|
| 1289 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1290 |
|
| 1291 |
# Compute latent shapes
|
| 1292 |
video_shape, audio_shape = compute_latent_shapes(
|
|
@@ -1890,6 +1896,7 @@ class JoyEcho_Generate:
|
|
| 1890 |
generator.latent_height = th // 32
|
| 1891 |
generator.latent_width = tw // 32
|
| 1892 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
|
|
|
| 1893 |
|
| 1894 |
offl = None
|
| 1895 |
if sequential_offload:
|
|
@@ -2228,6 +2235,8 @@ class JoyEcho_SingleShotGenerate:
|
|
| 2228 |
generator.latent_height = video_height // 32
|
| 2229 |
generator.latent_width = video_width // 32
|
| 2230 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
|
|
|
|
|
|
| 2231 |
|
| 2232 |
# Compute latent shapes
|
| 2233 |
video_shape, audio_shape = compute_latent_shapes(
|
|
|
|
| 1287 |
generator.latent_height = video_height // 32
|
| 1288 |
generator.latent_width = video_width // 32
|
| 1289 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
| 1290 |
+
# RENDER fps must drive the video RoPE clock (wrapper class default is
|
| 1291 |
+
# 24.0). At 25fps the hardcoded 24 skewed video rope-time 4% fast vs
|
| 1292 |
+
# audio's true-seconds rope -> ~40ms/s growing mouth-ahead-of-audio
|
| 1293 |
+
# drift, crossing visibility at ~9.6s into every shot (the "10s lip
|
| 1294 |
+
# sync cliff", 2026-07-23).
|
| 1295 |
+
generator.VIDEO_FPS = float(video_fps)
|
| 1296 |
|
| 1297 |
# Compute latent shapes
|
| 1298 |
video_shape, audio_shape = compute_latent_shapes(
|
|
|
|
| 1896 |
generator.latent_height = th // 32
|
| 1897 |
generator.latent_width = tw // 32
|
| 1898 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
| 1899 |
+
generator.VIDEO_FPS = float(fps) # refine RoPE clock = render fps too
|
| 1900 |
|
| 1901 |
offl = None
|
| 1902 |
if sequential_offload:
|
|
|
|
| 2235 |
generator.latent_height = video_height // 32
|
| 2236 |
generator.latent_width = video_width // 32
|
| 2237 |
generator.video_frame_seqlen = generator.latent_height * generator.latent_width
|
| 2238 |
+
# render fps drives the video RoPE clock (see multishot Generate note)
|
| 2239 |
+
generator.VIDEO_FPS = float(video_fps)
|
| 2240 |
|
| 2241 |
# Compute latent shapes
|
| 2242 |
video_shape, audio_shape = compute_latent_shapes(
|