#!/usr/bin/env python3 """SOTAKA Song Convert engine — Seed-VC (unified Space).""" from __future__ import annotations import os import sys import warnings from pathlib import Path from typing import Optional, Tuple os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") ROOT = Path(__file__).resolve().parent SEED_VC = ROOT / "seed-vc" sys.path.insert(0, str(SEED_VC)) os.environ.setdefault("HF_HUB_CACHE", str(SEED_VC / "checkpoints" / "hf_cache")) def _seed_cwd(): """Seed-VC uses relative checkpoint paths — run under its folder.""" os.chdir(SEED_VC) import librosa import numpy as np import soundfile as sf import torch import torchaudio import yaml try: import spaces except ImportError: class spaces: # type: ignore @staticmethod def GPU(*_a, **_k): def deco(fn): return fn return deco warnings.simplefilter("ignore") OUT_DIR = ROOT / "outputs" OUT_DIR.mkdir(exist_ok=True) MAX_SOURCE_SEC = int(os.environ.get("SEEDVC_MAX_SOURCE_SEC", "60")) _bundle = None def _device() -> torch.device: if torch.cuda.is_available(): return torch.device("cuda") return torch.device("cpu") def _crossfade(chunk1, chunk2, overlap): fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2 fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2 chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out return chunk2 def _adjust_f0_semitones(f0_sequence, n_semitones): return f0_sequence * (2 ** (n_semitones / 12)) def load_bundle(): """Load Seed-VC singing (F0) model + helpers once.""" global _bundle if _bundle is not None: return _bundle _seed_cwd() from modules.commons import build_model, load_checkpoint, recursive_munch from modules.audio import mel_spectrogram from modules.campplus.DTDNN import CAMPPlus from modules.rmvpe import RMVPE from hf_utils import load_custom_model_from_hf from transformers import AutoFeatureExtractor, WhisperModel device = _device() dit_checkpoint_path, dit_config_path = load_custom_model_from_hf( "Plachta/Seed-VC", "DiT_seed_v2_uvit_whisper_base_f0_44k_bigvgan_pruned_ft_ema_v2.pth", "config_dit_mel_seed_uvit_whisper_base_f0_44k.yml", ) config = yaml.safe_load(open(dit_config_path, "r")) model_params = recursive_munch(config["model_params"]) model_params.dit_type = "DiT" model = build_model(model_params, stage="DiT") hop_length = config["preprocess_params"]["spect_params"]["hop_length"] sr = config["preprocess_params"]["sr"] model, _, _, _ = load_checkpoint( model, None, dit_checkpoint_path, load_only_params=True, ignore_modules=[], is_distributed=False, ) for key in model: model[key].eval() model[key].to(device) model.cfm.estimator.setup_caches(max_batch_size=1, max_seq_length=8192) campplus_ckpt_path = load_custom_model_from_hf( "funasr/campplus", "campplus_cn_common.bin", config_filename=None ) campplus_model = CAMPPlus(feat_dim=80, embedding_size=192) campplus_model.load_state_dict(torch.load(campplus_ckpt_path, map_location="cpu")) campplus_model.eval() campplus_model.to(device) from modules.bigvgan import bigvgan bigvgan_model = bigvgan.BigVGAN.from_pretrained( model_params.vocoder.name, use_cuda_kernel=False ) bigvgan_model.remove_weight_norm() vocoder_fn = bigvgan_model.eval().to(device) whisper_name = model_params.speech_tokenizer.name whisper_model = WhisperModel.from_pretrained( whisper_name, torch_dtype=torch.float16 ).to(device) del whisper_model.decoder whisper_feature_extractor = AutoFeatureExtractor.from_pretrained(whisper_name) def semantic_fn(waves_16k): ori_inputs = whisper_feature_extractor( [waves_16k.squeeze(0).cpu().numpy()], sampling_rate=16000, return_tensors="pt", return_attention_mask=True, ) ori_input_features = whisper_model._mask_input_features( ori_inputs.input_features, attention_mask=ori_inputs.attention_mask ).to(device) with torch.no_grad(): ori_outputs = whisper_model.encoder( ori_input_features.to(whisper_model.encoder.dtype), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True, ) S_ori = ori_outputs.last_hidden_state.to(torch.float32) S_ori = S_ori[:, : waves_16k.size(-1) // 320 + 1] return S_ori mel_fn_args = { "n_fft": config["preprocess_params"]["spect_params"]["n_fft"], "win_size": config["preprocess_params"]["spect_params"]["win_length"], "hop_size": config["preprocess_params"]["spect_params"]["hop_length"], "num_mels": config["preprocess_params"]["spect_params"]["n_mels"], "sampling_rate": sr, "fmin": config["preprocess_params"]["spect_params"].get("fmin", 0), "fmax": None if config["preprocess_params"]["spect_params"].get("fmax", "None") == "None" else 8000, "center": False, } to_mel = lambda x: mel_spectrogram(x, **mel_fn_args) rmvpe_path = load_custom_model_from_hf( "lj1995/VoiceConversionWebUI", "rmvpe.pt", None ) rmvpe = RMVPE(rmvpe_path, is_half=False, device=device) _bundle = { "device": device, "model": model, "semantic_fn": semantic_fn, "vocoder_fn": vocoder_fn, "campplus_model": campplus_model, "to_mel": to_mel, "f0_fn": rmvpe.infer_from_audio, "sr": sr, "hop_length": hop_length, "overlap_frame_len": 16, } return _bundle @spaces.GPU(duration=150) def convert_song( source_audio: Optional[str], voice_ref: Optional[str], diffusion_steps: int = 30, pitch_shift: int = 0, length_adjust: float = 1.0, cfg_rate: float = 0.7, ) -> Tuple[Optional[str], str]: if not source_audio: return None, "Upload source vocal / song." if not voice_ref: return None, "Upload your voice reference (8–20s clear audio)." _seed_cwd() try: b = load_bundle() except Exception as e: return None, f"Failed to load Seed-VC: {e}" device = b["device"] model = b["model"] semantic_fn = b["semantic_fn"] vocoder_fn = b["vocoder_fn"] campplus_model = b["campplus_model"] to_mel = b["to_mel"] f0_fn = b["f0_fn"] sr = b["sr"] hop_length = b["hop_length"] overlap_frame_len = b["overlap_frame_len"] overlap_wave_len = overlap_frame_len * hop_length max_context_window = sr // hop_length * 30 steps = int(max(10, min(80, diffusion_steps))) pitch = int(max(-12, min(12, pitch_shift))) length_adjust = float(max(0.5, min(2.0, length_adjust))) cfg_rate = float(max(0.0, min(1.0, cfg_rate))) try: source_np = librosa.load(source_audio, sr=sr)[0] ref_np = librosa.load(voice_ref, sr=sr)[0] if len(source_np) > sr * MAX_SOURCE_SEC: source_np = source_np[: sr * MAX_SOURCE_SEC] trim_note = f"Source auto-trimmed to {MAX_SOURCE_SEC}s.\n" else: trim_note = "" ref_np = ref_np[: sr * 25] source_audio_t = torch.tensor(source_np).unsqueeze(0).float().to(device) ref_audio_t = torch.tensor(ref_np).unsqueeze(0).float().to(device) ref_waves_16k = torchaudio.functional.resample(ref_audio_t, sr, 16000) converted_waves_16k = torchaudio.functional.resample(source_audio_t, sr, 16000) if converted_waves_16k.size(-1) <= 16000 * 30: S_alt = semantic_fn(converted_waves_16k) else: overlapping_time = 5 S_alt_list = [] buffer = None traversed_time = 0 while traversed_time < converted_waves_16k.size(-1): if buffer is None: chunk = converted_waves_16k[ :, traversed_time : traversed_time + 16000 * 30 ] else: chunk = torch.cat( [ buffer, converted_waves_16k[ :, traversed_time : traversed_time + 16000 * (30 - overlapping_time), ], ], dim=-1, ) S_alt = semantic_fn(chunk) if traversed_time == 0: S_alt_list.append(S_alt) else: S_alt_list.append(S_alt[:, 50 * overlapping_time :]) buffer = chunk[:, -16000 * overlapping_time :] traversed_time += ( 30 * 16000 if traversed_time == 0 else chunk.size(-1) - 16000 * overlapping_time ) S_alt = torch.cat(S_alt_list, dim=1) ori_waves_16k = torchaudio.functional.resample(ref_audio_t, sr, 16000) S_ori = semantic_fn(ori_waves_16k) mel = to_mel(source_audio_t.float()) mel2 = to_mel(ref_audio_t.float()) target_lengths = torch.LongTensor([int(mel.size(2) * length_adjust)]).to( mel.device ) target2_lengths = torch.LongTensor([mel2.size(2)]).to(mel2.device) feat2 = torchaudio.compliance.kaldi.fbank( ori_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000 ) feat2 = feat2 - feat2.mean(dim=0, keepdim=True) style2 = campplus_model(feat2.unsqueeze(0)) F0_ori = f0_fn(ori_waves_16k[0], thred=0.03) F0_alt = f0_fn(converted_waves_16k[0], thred=0.03) F0_ori = torch.from_numpy(F0_ori).float().to(device)[None] F0_alt = torch.from_numpy(F0_alt).float().to(device)[None] log_f0_alt = torch.log(F0_alt + 1e-5) shifted_log_f0_alt = log_f0_alt.clone() shifted_f0_alt = torch.exp(shifted_log_f0_alt) if pitch != 0: shifted_f0_alt[F0_alt > 1] = _adjust_f0_semitones( shifted_f0_alt[F0_alt > 1], pitch ) cond, _, _, _, _ = model.length_regulator( S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt ) prompt_condition, _, _, _, _ = model.length_regulator( S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori ) interpolated_shifted_f0_alt = torch.nn.functional.interpolate( shifted_f0_alt.unsqueeze(1), size=cond.size(1), mode="nearest" ).squeeze(1) max_source_window = max_context_window - mel2.size(2) processed_frames = 0 generated_wave_chunks = [] previous_chunk = None with torch.no_grad(): while processed_frames < cond.size(1): chunk_cond = cond[ :, processed_frames : processed_frames + max_source_window ] is_last_chunk = processed_frames + max_source_window >= cond.size(1) cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1) with torch.autocast( device_type=device.type, dtype=torch.float16, enabled=device.type == "cuda" ): vc_target = model.cfm.inference( cat_condition, torch.LongTensor([cat_condition.size(1)]).to(mel2.device), mel2, style2, None, steps, inference_cfg_rate=cfg_rate, ) vc_target = vc_target[:, :, mel2.size(-1) :] vc_wave = vocoder_fn(vc_target.float()).squeeze().cpu() if vc_wave.ndim == 1: vc_wave = vc_wave.unsqueeze(0) if processed_frames == 0: if is_last_chunk: generated_wave_chunks.append(vc_wave[0].numpy()) break generated_wave_chunks.append(vc_wave[0, :-overlap_wave_len].numpy()) previous_chunk = vc_wave[0, -overlap_wave_len:] processed_frames += vc_target.size(2) - overlap_frame_len elif is_last_chunk: generated_wave_chunks.append( _crossfade( previous_chunk.numpy(), vc_wave[0].numpy(), overlap_wave_len ) ) break else: generated_wave_chunks.append( _crossfade( previous_chunk.numpy(), vc_wave[0, :-overlap_wave_len].numpy(), overlap_wave_len, ) ) previous_chunk = vc_wave[0, -overlap_wave_len:] processed_frames += vc_target.size(2) - overlap_frame_len wav = np.concatenate(generated_wave_chunks).astype(np.float32) out = OUT_DIR / "SOTAKA_Song.wav" sf.write(out, wav, sr) return str(out), ( trim_note + f"Engine: Seed-VC (ZeroGPU)\n" f"device={device} | steps={steps} | pitch={pitch}\n" f"duration={len(wav)/sr:.1f}s | saved SOTAKA_Song.wav" ) except Exception as e: return None, f"Conversion error: {e}"