deforum_engine.py

import torch
import numpy as np
from diffusers import AutoPipelineForImage2Image, LCMScheduler, EulerAncestralDiscreteScheduler
from PIL import Image
import utils
import os
import uuid
import zipfile
import cv2
import gc

class DeforumRunner:
    def __init__(self, device="cpu"):
        self.device = device
        self.pipe = None
        self.stop_requested = False
        self.current_model_config = (None, None, None)

    def load_model(self, model_id, lora_id, scheduler_name):
        """Loads model, LoRA, and scheduler dynamically."""
        new_config = (model_id, lora_id, scheduler_name)
        if new_config == self.current_model_config and self.pipe is not None:
            return

        print(f"Loading Model: {model_id}, LoRA: {lora_id}, Scheduler: {scheduler_name}")
        if self.pipe:
            del self.pipe
            gc.collect()

        try:
            pipe = AutoPipelineForImage2Image.from_pretrained(
                model_id, safety_checker=None, torch_dtype=torch.float32
            )
        except Exception as e:
            print(f"Error loading model {model_id}: {e}. Falling back.")
            pipe = AutoPipelineForImage2Image.from_pretrained(
                "runwayml/stable-diffusion-v1-5", safety_checker=None, torch_dtype=torch.float32
            )

        if lora_id and lora_id != "None":
            try:
                pipe.load_lora_weights(lora_id)
                pipe.fuse_lora()
                print("LoRA loaded and fused.")
            except Exception as e: print(f"Error loading LoRA: {e}")

        if scheduler_name == "LCM":
            pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
        elif scheduler_name == "Euler A":
            pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)

        pipe.to(self.device)
        pipe.set_progress_bar_config(disable=True)
        pipe.enable_attention_slicing() # Crucial for CPU memory
        self.pipe = pipe
        self.current_model_config = new_config
        print("Pipeline ready.")

    def stop(self):
        self.stop_requested = True

    def render(self, prompts, neg_prompt, max_frames, width, height,
               zoom_s, angle_s, tx_s, ty_s, strength_s, noise_s, 
               fps, steps, cadence, color_mode, border_mode, init_image_upload,
               model_id, lora_id, scheduler_name):
        
        self.stop_requested = False
        self.load_model(model_id, lora_id, scheduler_name)
        
        # 1. Parse Schedules
        keys = ['z', 'a', 'tx', 'ty', 'str', 'noi']
        inputs = [zoom_s, angle_s, tx_s, ty_s, strength_s, noise_s]
        sched = {k: utils.parse_weight_string(v, max_frames) for k, v in zip(keys, inputs)}
        
        # 2. Setup
        run_id = uuid.uuid4().hex[:6]
        output_dir = f"output_{run_id}"
        os.makedirs(output_dir, exist_ok=True)
        
        if init_image_upload:
            prev_img = init_image_upload.resize((width, height), Image.LANCZOS)
            color_anchor = prev_img
        else:
            prev_img = None
            color_anchor = None
            
        generated_frames = []
        print(f"Starting run {run_id}...")

        # 3. Loop
        for i in range(max_frames):
            if self.stop_requested:
                print("Generation stopped.")
                break
            
            # Get Params
            z, a, tx, ty = sched['z'][i], sched['a'][i], sched['tx'][i], sched['ty'][i]
            strength, noise = sched['str'][i], sched['noi'][i]
            current_prompt = prompts[max(k for k in prompts.keys() if k <= i)]
            
            # --- Authentic Deforum Loop ---
            # 1. Warp Previous Frame
            if prev_img is not None:
                warped_img = utils.anim_frame_warp_2d(prev_img, {'angle': a, 'zoom': z, 'translation_x': tx, 'translation_y': ty}, border_mode)
            else:
                warped_img = Image.new("RGB", (width, height), (0,0,0))

            # Decide: Diffusion or Just Warp (Cadence)
            if i % cadence == 0:
                # 2. Color Match & 3. Add Noise (Only before diffusion)
                init_for_diff = utils.maintain_colors(warped_img, color_anchor, color_mode)
                init_for_diff = utils.add_noise(init_for_diff, noise)
                
                # 4. Diffusion (Img2Img)
                # Use high strength for first frame if no init image provided
                curr_strength = strength if prev_img is not None else 0.95
                
                gen_image = self.pipe(
                    prompt=current_prompt, negative_prompt=neg_prompt,
                    image=init_for_diff, num_inference_steps=steps,
                    strength=curr_strength, guidance_scale=1.2, # Low CFG for LCM
                    width=width, height=height
                ).images[0]
            else:
                # Cadence step: Just show the warped image (faster)
                gen_image = warped_img

            # Update state
            prev_img = gen_image
            if color_anchor is None: color_anchor = gen_image
            generated_frames.append(gen_image)
            yield gen_image, None, None

        # 4. Finalize
        vid_path = f"{output_dir}/video.mp4"
        self.save_video(generated_frames, vid_path, fps)
        zip_path = f"{output_dir}/frames.zip"
        self.save_zip(generated_frames, zip_path)
        yield generated_frames[-1], vid_path, zip_path

    # (save_video and save_zip are the same as before, omitted for brevity)
    def save_video(self, frames, path, fps):
        if not frames: return
        w, h = frames[0].size
        out = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
        for f in frames: out.write(cv2.cvtColor(np.array(f), cv2.COLOR_RGB2BGR))
        out.release()
    def save_zip(self, frames, path):
        import io
        with zipfile.ZipFile(path, 'w') as zf:
            for i, f in enumerate(frames):
                buf = io.BytesIO()
                f.save(buf, format="PNG")
                zf.writestr(f"{i:05d}.png", buf.getvalue())