#!/usr/bin/env python3 """SOTAKA Song Convert engine — Seed-VC (unified Space).""" from __future__ import annotations import os import sys import warnings from pathlib import Path from typing import Optional, Tuple os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") ROOT = Path(__file__).resolve().parent SEED_VC = ROOT / "seed-vc" sys.path.insert(0, str(SEED_VC)) os.environ.setdefault("HF_HUB_CACHE", str(SEED_VC / "checkpoints" / "hf_cache")) def _seed_cwd(): """Seed-VC uses relative checkpoint paths — run under its folder.""" os.chdir(SEED_VC) import librosa import numpy as np import soundfile as sf import torch import torchaudio import yaml try: import spaces except ImportError: class spaces: # type: ignore @staticmethod def GPU(*_a, **_k): def deco(fn): return fn return deco warnings.simplefilter("ignore") OUT_DIR = ROOT / "outputs" OUT_DIR.mkdir(exist_ok=True) # Up to ~2:15; long clips are split into chunks (ZeroGPU ~120s per chunk). # Shorter chunks + higher diffusion steps → clearer singing (Seed-VC recommends 30–100). MAX_SOURCE_SEC = int(os.environ.get("SEEDVC_MAX_SOURCE_SEC", "135")) CHUNK_SEC = int(os.environ.get("SEEDVC_CHUNK_SEC", "30")) CHUNK_OVERLAP_SEC = float(os.environ.get("SEEDVC_CHUNK_OVERLAP", "1.5")) # Latest singing checkpoint (Oct 2024+ ft_ema_v2). SEEDVC_CKPT = os.environ.get( "SEEDVC_CKPT", "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema_v2.pth", ) SEEDVC_CONFIG = os.environ.get( "SEEDVC_CONFIG", "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml", ) _bundle = None def _device() -> torch.device: if torch.cuda.is_available(): return torch.device("cuda") return torch.device("cpu") def _crossfade(chunk1, chunk2, overlap): fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2 fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2 chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out return chunk2 def _adjust_f0_semitones(f0_sequence, n_semitones): return f0_sequence * (2 ** (n_semitones / 12)) def load_bundle(): """Load Seed-VC singing (F0) model + helpers once.""" global _bundle if _bundle is not None: return _bundle _seed_cwd() from modules.commons import build_model, load_checkpoint, recursive_munch from modules.audio import mel_spectrogram from modules.campplus.DTDNN import CAMPPlus from modules.rmvpe import RMVPE from hf_utils import load_custom_model_from_hf from transformers import AutoFeatureExtractor, WhisperModel device = _device() dit_checkpoint_path, dit_config_path = load_custom_model_from_hf( "Plachta/Seed-VC", SEEDVC_CKPT, SEEDVC_CONFIG, ) config = yaml.safe_load(open(dit_config_path, "r")) model_params = recursive_munch(config["model_params"]) model_params.dit_type = "DiT" model = build_model(model_params, stage="DiT") hop_length = config["preprocess_params"]["spect_params"]["hop_length"] sr = config["preprocess_params"]["sr"] model, _, _, _ = load_checkpoint( model, None, dit_checkpoint_path, load_only_params=True, ignore_modules=[], is_distributed=False, ) for key in model: model[key].eval() model[key].to(device) model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192) campplus_ckpt_path = load_custom_model_from_hf( "funasr/campplus", "campplus_cn_common.bin", config_filename=None ) campplus_model = CAMPPlus(feat_dim=80, embedding_size=192) campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu")) campplus_model.eval() campplus_model.to(device) from modules.bigvgan import bigvgan bigvgan_model = bigvgan.BigVGAN.from_pretrained( model_params.vocoder.name, use_cuda_kernel=False ) bigvgan_model.remove_weight_norm() vocoder_fn = bigvgan_model.eval().to(device) whisper_name = model_params.speech_tokenizer.name whisper_model = WhisperModel.from_pretrained( whisper_name, torch_dtype=torch.float16 ).to(device) del whisper_model.decoder whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name) def semantic_fn(waves_16k): ori_inputs = whisper_feature_extractor( [waves_16k.squeeze(0).cpu().numpy()], sampling_rate=16000, return_tensors="pt", return_attention_mask=True, ) ori_input_features = whisper_model._mask_input_features( ori_inputs.input_features, attention_mask=ori_inputs.attention_mask ).to(device) with torch.no_grad(): ori_outputs = whisper_model.encoder( ori_input_features.to(whisper_model.encoder.dtype), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True, ) S_ori = ori_outputs.last_hidden_state.to(torch.float32) S_ori = S_ori[:, : waves_16k.size(-1) // 320 + 1] return S_ori mel_fn_args = { "n_fft": config["preprocess_params"]["spect_params"]["n_fft"], "win_size": config["preprocess_params"]["spect_params"]["win_length"], "hop_size": config["preprocess_params"]["spect_params"]["hop_length"], "num_mels": config["preprocess_params"]["spect_params"]["n_mels"], "sampling_rate": sr, "fmin": config["preprocess_params"]["spect_params"].get("fmin", 0), "fmax": None if config["preprocess_params"]["spect_params"].get("fmax", "None") == "None" else 8000, "center": False, } to_mel = lambda x: mel_spectrogram(x, **mel_fn_args) rmvpe_path = load_custom_model_from_hf( "lj1995/VoiceConversionWebUI", "rmvpe.pt", None ) rmvpe = RMVPE(rmvpe_path, is_half=False, device=device) _bundle = { "device": device, "model": model, "semantic_fn": semantic_fn, "vocoder_fn": vocoder_fn, "campplus_model": campplus_model, "to_mel": to_mel, "f0_fn": rmvpe.infer_from_audio, "sr": sr, "hop_length": hop_length, "overlap_frame_len": 16, } return _bundle def _seed_vc_infer( source_np: np.ndarray, ref_np: np.ndarray, b: dict, steps: int, pitch: int, length_adjust: float, cfg_rate: float, auto_f0_adjust: bool = False, ) -> np.ndarray: """Run Seed-VC on in-memory waveforms (same sample rate as bundle).""" device = b["device"] model = b["model"] semantic_fn = b["semantic_fn"] vocoder_fn = b["vocoder_fn"] campplus_model = b["campplus_model"] to_mel = b["to_mel"] f0_fn = b["f0_fn"] sr = b["sr"] hop_length = b["hop_length"] overlap_frame_len = b["overlap_frame_len"] overlap_wave_len = overlap_frame_len * hop_length max_context_window = sr // hop_length * 30 source_audio_t = torch.tensor(source_np).unsqueeze(0).float().to(device) ref_audio_t = torch.tensor(ref_np).unsqueeze(0).float().to(device) ref_waves_16k = torchaudio.functional.resample(ref_audio_t, sr, 16000) converted_waves_16k = torchaudio.functional.resample(source_audio_t, sr, 16000) if converted_waves_16k.size(-1) <= 16000 * 30: S_alt = semantic_fn(converted_waves_16k) else: overlapping_time = 5 S_alt_list = [] buffer = None traversed_time = 0 while traversed_time < converted_waves_16k.size(-1): if buffer is None: chunk = converted_waves_16k[ :, traversed_time : traversed_time + 16000 * 30 ] else: chunk = torch.cat( [ buffer, converted_waves_16k[ :, traversed_time : traversed_time + 16000 * (30 - overlapping_time), ], ], dim=-1, ) S_alt = semantic_fn(chunk) if traversed_time == 0: S_alt_list.append(S_alt) else: S_alt_list.append(S_alt[:, 50 * overlapping_time :]) buffer = chunk[:, -16000 * overlapping_time :] traversed_time += ( 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time ) S_alt = torch.cat(S_alt_list, dim=1) ori_waves_16k = torchaudio.functional.resample(ref_audio_t, sr, 16000) S_ori = semantic_fn(ori_waves_16k) mel = to_mel(source_audio_t.float()) mel2 = to_mel(ref_audio_t.float()) target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to( mel.device ) target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device) feat2 = torchaudio.compliance.kaldi.fbank( ori_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000 ) feat2 = feat2 - feat2.mean(dim=0, keepdim=True) style2 = campplus_model(feat2.unsqueeze(0)) F0_ori = f0_fn(ori_waves_16k[0], thred=0.03) F0_alt = f0_fn(converted_waves_16k[0], thred=0.03) F0_ori = torch.from_numpy(F0_ori).float().to(device)[None] F0_alt = torch.from_numpy(F0_alt).float().to(device)[None] # Match official Seed-VC app_svc F0 handling (melody keep + optional level shift). log_f0_alt = torch.log(F0_alt + 1e-5) shifted_log_f0_alt = log_f0_alt.clone() if auto_f0_adjust: voiced_ori = F0_ori[F0_ori > 1] voiced_alt = F0_alt[F0_alt > 1] if voiced_ori.numel() > 0 and voiced_alt.numel() > 0: median_log_ori = torch.median(torch.log(voiced_ori + 1e-5)) median_log_alt = torch.median(torch.log(voiced_alt + 1e-5)) shifted_log_f0_alt[F0_alt > 1] = ( log_f0_alt[F0_alt > 1] - median_log_alt + median_log_ori ) shifted_f0_alt = torch.exp(shifted_log_f0_alt) if pitch != 0: shifted_f0_alt[F0_alt > 1] = _adjust_f0_semitones( shifted_f0_alt[F0_alt > 1], pitch ) cond, _, _, _, _ = model.length_regulator( S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt ) prompt_condition, _, _, _, _ = model.length_regulator( S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori ) max_source_window = max_context_window - mel2.size(2) processed_frames = 0 generated_wave_chunks = [] previous_chunk = None with torch.no_grad(): while processed_frames < cond.size(1): chunk_cond = cond[ :, processed_frames : processed_frames + max_source_window ] is_last_chunk = processed_frames + max_source_window >= cond.size(1) cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1) with torch.autocast( device_type=device.type, dtype=torch.float16, enabled=device.type == "cuda", ): vc_target = model.cfm.inference( cat_condition, torch.LongTensor([cat_condition.size(1)]).to(mel2.device), mel2, style2, None, steps, inference_cfg_rate=cfg_rate, ) vc_target = vc_target[:, :, mel2.size(-1) :] vc_wave = vocoder_fn(vc_target.float()).squeeze().cpu() if vc_wave.ndim == 1: vc_wave = vc_wave.unsqueeze(0) if processed_frames == 0: if is_last_chunk: generated_wave_chunks.append(vc_wave[0].numpy()) break generated_wave_chunks.append(vc_wave[0, :-overlap_wave_len].numpy()) previous_chunk = vc_wave[0, -overlap_wave_len:] processed_frames += vc_target.size(2) - overlap_frame_len elif is_last_chunk: generated_wave_chunks.append( _crossfade( previous_chunk.numpy(), vc_wave[0].numpy(), overlap_wave_len ) ) break else: generated_wave_chunks.append( _crossfade( previous_chunk.numpy(), vc_wave[0, :-overlap_wave_len].numpy(), overlap_wave_len, ) ) previous_chunk = vc_wave[0, -overlap_wave_len:] processed_frames += vc_target.size(2) - overlap_frame_len return np.concatenate(generated_wave_chunks).astype(np.float32) # ~30s audio + model load fits ZeroGPU 120s; higher steps = clearer voice. @spaces.GPU(duration=120) def _convert_one_chunk( source_audio: str, voice_ref: str, diffusion_steps: int = 50, pitch_shift: int = 0, length_adjust: float = 1.0, cfg_rate: float = 0.7, auto_f0_adjust: bool = False, ) -> Tuple[Optional[str], str]: global _bundle _bundle = None # ZeroGPU releases CUDA between calls _seed_cwd() try: b = load_bundle() except Exception as e: return None, f"Failed to load Seed-VC: {e}" steps = int(max(10, min(100, diffusion_steps))) pitch = int(max(-12, min(12, pitch_shift))) length_adjust = float(max(0.5, min(2.0, length_adjust))) cfg_rate = float(max(0.0, min(1.0, cfg_rate))) sr = b["sr"] try: source_np = librosa.load(source_audio, sr=sr)[0] # Clearer clone with 8–20s; hard-cap 25s like official Seed-VC. ref_np = librosa.load(voice_ref, sr=sr)[0][: sr * 25] wav = _seed_vc_infer( source_np, ref_np, b, steps, pitch, length_adjust, cfg_rate, auto_f0_adjust=bool(auto_f0_adjust), ) out = OUT_DIR / f"chunk_{os.getpid()}_{len(wav)}.wav" sf.write(out, wav, sr) return ( str(out), f"chunk_ok len={len(wav)/sr:.1f}s steps={steps} ckpt={SEEDVC_CKPT}", ) except Exception as e: return None, f"Chunk error: {e}" def _stitch_chunks(wavs: list, sr: int, overlap_sec: float) -> np.ndarray: if len(wavs) == 1: return wavs[0] overlap = int(sr * overlap_sec) out = wavs[0] for nxt in wavs[1:]: if overlap > 0 and len(out) > overlap and len(nxt) > overlap: merged = _crossfade(out[-overlap:].copy(), nxt[:overlap].copy(), overlap) out = np.concatenate([out[:-overlap], merged, nxt[overlap:]]) else: out = np.concatenate([out, nxt]) return out.astype(np.float32) def convert_song( source_audio: Optional[str], voice_ref: Optional[str], diffusion_steps: int = 50, pitch_shift: int = 0, length_adjust: float = 1.0, cfg_rate: float = 0.7, auto_f0_adjust: bool = False, ) -> Tuple[Optional[str], str]: """Convert up to ~2:15. Long audio is split into ~30s ZeroGPU chunks.""" if not source_audio: return None, "Upload source vocal / song." if not voice_ref: return None, "Upload your voice reference (8–20s clear audio)." steps = int(max(10, min(100, diffusion_steps))) pitch = int(max(-12, min(12, pitch_shift))) length_adjust = float(max(0.5, min(2.0, length_adjust))) cfg_rate = float(max(0.0, min(1.0, cfg_rate))) auto_f0_adjust = bool(auto_f0_adjust) try: # Probe SR from config without full model load when possible source_np, file_sr = librosa.load(source_audio, sr=None) if file_sr != 44100: source_np = librosa.resample(source_np, orig_sr=file_sr, target_sr=44100) sr = 44100 trim_note = "" if len(source_np) > sr * MAX_SOURCE_SEC: source_np = source_np[: sr * MAX_SOURCE_SEC] trim_note = f"Source auto-trimmed to {MAX_SOURCE_SEC // 60}:{MAX_SOURCE_SEC % 60:02d}.\n" chunk_len = sr * CHUNK_SEC hop = max(1, int(sr * (CHUNK_SEC - CHUNK_OVERLAP_SEC))) starts = list(range(0, len(source_np), hop)) # Drop tiny trailing sliver if previous chunk already covers it if len(starts) > 1 and len(source_np) - starts[-1] < sr * 5: starts = starts[:-1] chunk_paths = [] for i, start in enumerate(starts): end = min(len(source_np), start + chunk_len) part = source_np[start:end] p = OUT_DIR / f"src_chunk_{i}.wav" sf.write(p, part, sr) chunk_paths.append(p) converted = [] logs = [] for i, p in enumerate(chunk_paths): out_path, log = _convert_one_chunk( str(p), voice_ref, steps, pitch, length_adjust, cfg_rate, auto_f0_adjust, ) logs.append(f"[{i+1}/{len(chunk_paths)}] {log}") if not out_path: return None, "\n".join(logs) converted.append(librosa.load(out_path, sr=sr)[0]) wav = _stitch_chunks(converted, sr, CHUNK_OVERLAP_SEC) import time as _time out = OUT_DIR / f"SOTAKA_Song_{int(_time.time())}.wav" sf.write(out, wav, sr) return str(out), ( trim_note + f"Engine: Seed-VC singing ({SEEDVC_CKPT})\n" f"chunks={len(chunk_paths)} | steps={steps} | pitch={pitch} | " f"auto_f0={auto_f0_adjust}\n" f"duration={len(wav)/sr:.1f}s | max={MAX_SOURCE_SEC}s | saved {out.name}\n" + "\n".join(logs) ) except Exception as e: return None, f"Conversion error: {e}"