from __future__ import annotations import io import shutil import sys import threading import urllib.request import zipfile from pathlib import Path import cv2 import numpy as np import onnxruntime as ort import torch import torch.nn as nn from huggingface_hub import hf_hub_download from insightface.app import FaceAnalysis from torchvision.transforms.functional import rgb_to_grayscale GHOST_CACHE = Path.home() / ".cache" / "dream-ghost2" SOURCE_DIR = GHOST_CACHE / "source" MODEL_REPO = "hacksider/deep-live-cam" MODEL_REVISION = "e1c6a60039351a68150db2b50b0ef936b9ba259a" SOURCE_ZIP = "https://github.com/ai-forever/ghost-2.0/archive/refs/heads/main.zip" STYLEMATTE_REPO = "yc4ny/SVAD-models" STYLEMATTE_FILE = "submodules/GAGAvatar/assets/matting/stylematte_synth.pt" _LOCK = threading.Lock() _ENGINE: "Ghost2Engine | None" = None def _prepare_source() -> Path: marker = SOURCE_DIR / ".ready" if marker.exists(): return SOURCE_DIR shutil.rmtree(SOURCE_DIR, ignore_errors=True) SOURCE_DIR.mkdir(parents=True, exist_ok=True) with urllib.request.urlopen(SOURCE_ZIP, timeout=120) as response: payload = response.read() with zipfile.ZipFile(io.BytesIO(payload)) as archive: root = archive.namelist()[0].split("/")[0] for member in archive.infolist(): if not member.filename.startswith(root + "/") or member.is_dir(): continue relative = Path(member.filename).relative_to(root) destination = SOURCE_DIR / relative destination.parent.mkdir(parents=True, exist_ok=True) with archive.open(member) as source, destination.open("wb") as target: shutil.copyfileobj(source, target) # emoca_crop is unused during inference; removing this optional training import # avoids pulling DECA/EMOCA/PyTorch3D into the ZeroGPU image. crops = SOURCE_DIR / "src" / "utils" / "crops.py" text = crops.read_text() text = text.replace( "from repos.emoca.gdl.datasets.ImageDatasetHelpers import bbox2point\n", "" ) crops.write_text(text) embedder = SOURCE_DIR / "src" / "aligner" / "embedder.py" text = embedder.read_text() text = text.replace("weights='DEFAULT'", "weights=None") embedder.write_text(text) marker.write_text("ready") return SOURCE_DIR def _model_file(filename: str) -> str: return hf_hub_download( repo_id=MODEL_REPO, filename=f"ghost2/{filename}", revision=MODEL_REVISION, ) class Ghost2Engine: def __init__(self) -> None: root = _prepare_source() if str(root) not in sys.path: sys.path.insert(0, str(root)) from src.aligner.embedder import Embedder from src.aligner.generator import Generator from src.blender.generator import BlenderGenerator from src.utils.crops import norm_crop, wide_crop_face from src.utils.inference import copy_head_back, normalize_and_torch from src.utils.inpainter import LamaInpainter from src.utils.preblending import calc_pseudo_target_bg self.norm_crop = norm_crop self.wide_crop_face = wide_crop_face self.copy_head_back = copy_head_back self.normalize_and_torch = normalize_and_torch self.calc_pseudo_target_bg = calc_pseudo_target_bg backbone = _model_file("backbone50_1.pth") weights_dir = root / "weights" weights_dir.mkdir(exist_ok=True) backbone_link = weights_dir / "backbone50_1.pth" if not backbone_link.exists(): backbone_link.symlink_to(backbone) # Upstream modules use relative asset paths. self._previous_cwd = Path.cwd() import os os.chdir(root) try: class AlignerInference(nn.Module): def __init__(inner_self) -> None: super().__init__() inner_self.embedder = Embedder(d_por=512, d_id=512, d_pose=256, d_exp=0) inner_self.gen = Generator( d_por=512, d_id=512, d_pose=256, d_exp=0, padding="zero", in_channels=3, out_channels=3, num_channels=64, max_num_channels=512, norm_layer="in", gen_constant_input_size=4, gen_num_residual_blocks=2, output_image_size=512, ) def forward(inner_self, batch): return inner_self.gen(inner_self.embedder(batch)) self.aligner = AlignerInference() aligner_state = torch.load(_model_file("aligner_1020_gaze_final.ckpt"), map_location="cpu") if "state_dict" in aligner_state: aligner_state = aligner_state["state_dict"] self.aligner.load_state_dict( {k: v for k, v in aligner_state.items() if k.startswith(("embedder.", "gen."))}, strict=False, ) self.blender = BlenderGenerator() blender_state = torch.load(_model_file("blender_lama.ckpt"), map_location="cpu") if "state_dict" in blender_state: blender_state = blender_state["state_dict"] self.blender.load_state_dict( {k.removeprefix("gen."): v for k, v in blender_state.items() if k.startswith("gen.")}, strict=False, ) self.inpainter = LamaInpainter() finally: os.chdir(self._previous_cwd) self.aligner = self.aligner.cuda().eval() self.blender = self.blender.cuda().eval() self.detector = FaceAnalysis( root=str(GHOST_CACHE / "insightface"), providers=["CUDAExecutionProvider", "CPUExecutionProvider"], allowed_modules=["detection"], ) self.detector.prepare(ctx_id=0, det_size=(640, 640)) self.parsing = ort.InferenceSession( _model_file("segformer_B5_ce.onnx"), providers=["CUDAExecutionProvider", "CPUExecutionProvider"], ) self.parsing_input = self.parsing.get_inputs()[0].name self.parsing_outputs = [item.name for item in self.parsing.get_outputs()] self.mean = np.array([0.51315393, 0.48064056, 0.46301059])[None, :, None, None] self.std = np.array([0.21438347, 0.20799829, 0.20304542])[None, :, None, None] def _parsing(self, image: torch.Tensor) -> torch.Tensor: prepared = (((image[:, [2, 1, 0]] / 2 + 0.5).detach().cpu().numpy() - self.mean) / self.std) result = self.parsing.run( self.parsing_outputs, {self.parsing_input: prepared.astype(np.float32)} )[0] return torch.tensor(result, device="cuda", dtype=torch.float32) @staticmethod def _head_mask(parsing: torch.Tensor) -> torch.Tensor: mask = torch.zeros_like(parsing, dtype=torch.float32) for index in range(1, 21): mask[parsing == index] = 1.0 return mask[0, 0] if mask.ndim == 4 else mask[0] def _process(self, frame: np.ndarray, target: bool = False): faces = self.detector.get(frame) if not faces: raise ValueError("No head was detected in the image/frame.") face = max( faces, key=lambda item: float((item.bbox[2] - item.bbox[0]) * (item.bbox[3] - item.bbox[1])), ) keypoints = face.kps wide = self.wide_crop_face(frame, keypoints, return_M=target) if target: wide, matrix = wide arc = self.norm_crop(frame, keypoints) arc_tensor = self.normalize_and_torch(arc) wide_tensor = self.normalize_and_torch(wide) mask = self._head_mask(self._parsing(wide_tensor)) if target: return wide_tensor, arc_tensor, mask, frame, matrix return wide_tensor, arc_tensor, mask def prepare_source(self, source_bgr: np.ndarray) -> dict[str, torch.Tensor]: wide, arc, mask = self._process(source_bgr) return { "wide": wide.unsqueeze(1), "arc": arc.unsqueeze(1), "mask": mask, } def swap_frame(self, source: dict[str, torch.Tensor], target_bgr: np.ndarray) -> np.ndarray: wide_target, arc_target, target_mask, full_frame, matrix = self._process( target_bgr, target=True ) source_mask = source["mask"] batch = { "source": { "face_arc": source["arc"], "face_wide": source["wide"] * source_mask, "face_wide_mask": source_mask, }, "target": { "face_arc": arc_target, "face_wide": wide_target * target_mask, "face_wide_mask": target_mask, }, } with torch.inference_mode(): aligned = self.aligner(batch) target_parsing = self._parsing(wide_target) pseudo_background = self.calc_pseudo_target_bg(wide_target, target_parsing) aligned_parsing = self._parsing(aligned["fake_rgbs"] * aligned["fake_segm"]) soft_mask = self._head_mask(aligned_parsing).unsqueeze(0) new_source = ( aligned["fake_rgbs"] * soft_mask[:, None] + pseudo_background * (1 - soft_mask[:, None]) ) output = self.blender( new_source, rgb_to_grayscale(new_source[0][[2, 1, 0]]).unsqueeze(0), wide_target, aligned_parsing, target_parsing, gt=wide_target, M_a_noise=None, M_t_noise=None, cycle=False, train=False, return_inputs=True, inpainter=self.inpainter, )[0] crop = np.uint8( (output[0].detach().cpu().numpy().transpose(1, 2, 0)[:, :, ::-1] / 2 + 0.5) * 255 ) rgb = self.copy_head_back(crop, full_frame[..., ::-1], matrix) return rgb[..., ::-1].copy() def get_ghost_engine() -> Ghost2Engine: global _ENGINE with _LOCK: if _ENGINE is None: _ENGINE = Ghost2Engine() return _ENGINE