joeygambino commited on
Commit
8d2024c
·
verified ·
1 Parent(s): 23d5b79

fix: video RoPE clock uses the render fps (hardcoded 24 at 25fps renders = 4%/s AV rope skew = the ~10s lip-sync drift)

Browse files
libs/ltx_distillation/models/ltx_wrapper.py CHANGED
@@ -231,8 +231,13 @@ class LTX2DiffusionWrapper(nn.Module):
231
  causal_fix=True,
232
  ).float()
233
 
234
- # Convert temporal dimension from frames to seconds (divide by fps=24)
235
- # This matches VideoLatentTools.create_initial_state
 
 
 
 
 
236
  pixel_coords[:, 0, ...] = pixel_coords[:, 0, ...] / self.VIDEO_FPS
237
 
238
  if downscale_factor != 1:
 
231
  causal_fix=True,
232
  ).float()
233
 
234
+ # Convert temporal dimension from frames to seconds. self.VIDEO_FPS
235
+ # MUST be the actual render fps - the JoyEcho nodes stamp it as an
236
+ # instance attribute before generation. The class default of 24.0 with
237
+ # 25fps renders skewed video rope-time 4% fast against audio's
238
+ # true-seconds rope: a growing mouth-ahead-of-audio drift that crossed
239
+ # visibility at ~9.6s into every shot (the "10s lip-sync cliff",
240
+ # diagnosed 2026-07-23).
241
  pixel_coords[:, 0, ...] = pixel_coords[:, 0, ...] / self.VIDEO_FPS
242
 
243
  if downscale_factor != 1:
nodes.py CHANGED
@@ -1287,6 +1287,12 @@ class JoyEcho_Generate:
1287
  generator.latent_height = video_height // 32
1288
  generator.latent_width = video_width // 32
1289
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
 
 
 
 
 
 
1290
 
1291
  # Compute latent shapes
1292
  video_shape, audio_shape = compute_latent_shapes(
@@ -1890,6 +1896,7 @@ class JoyEcho_Generate:
1890
  generator.latent_height = th // 32
1891
  generator.latent_width = tw // 32
1892
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
 
1893
 
1894
  offl = None
1895
  if sequential_offload:
@@ -2228,6 +2235,8 @@ class JoyEcho_SingleShotGenerate:
2228
  generator.latent_height = video_height // 32
2229
  generator.latent_width = video_width // 32
2230
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
 
 
2231
 
2232
  # Compute latent shapes
2233
  video_shape, audio_shape = compute_latent_shapes(
 
1287
  generator.latent_height = video_height // 32
1288
  generator.latent_width = video_width // 32
1289
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
1290
+ # RENDER fps must drive the video RoPE clock (wrapper class default is
1291
+ # 24.0). At 25fps the hardcoded 24 skewed video rope-time 4% fast vs
1292
+ # audio's true-seconds rope -> ~40ms/s growing mouth-ahead-of-audio
1293
+ # drift, crossing visibility at ~9.6s into every shot (the "10s lip
1294
+ # sync cliff", 2026-07-23).
1295
+ generator.VIDEO_FPS = float(video_fps)
1296
 
1297
  # Compute latent shapes
1298
  video_shape, audio_shape = compute_latent_shapes(
 
1896
  generator.latent_height = th // 32
1897
  generator.latent_width = tw // 32
1898
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
1899
+ generator.VIDEO_FPS = float(fps) # refine RoPE clock = render fps too
1900
 
1901
  offl = None
1902
  if sequential_offload:
 
2235
  generator.latent_height = video_height // 32
2236
  generator.latent_width = video_width // 32
2237
  generator.video_frame_seqlen = generator.latent_height * generator.latent_width
2238
+ # render fps drives the video RoPE clock (see multishot Generate note)
2239
+ generator.VIDEO_FPS = float(video_fps)
2240
 
2241
  # Compute latent shapes
2242
  video_shape, audio_shape = compute_latent_shapes(