Upload DeepSafe services: model weights and inference code

.gitattributes

@@ -33,3 +33,97 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/ASVSpoof-results.png filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/Fig1.jpg filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/SafeEar_logo.jpg filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/exp1.png filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/overall.gif filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/overall.mp4 filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/assert/overall.png filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/fairseq_ours/docs/fairseq.gif filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/fairseq_ours/examples/MMPT/videoclip.png filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/fairseq_ours/examples/MMPT/vlm.png filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/fairseq_ours/examples/hubert/tests/6313-76958-0021.flac filter=lfs diff=lfs merge=lfs -text
+audio/safeear/safeear_repo/fairseq_ours/examples/textless_nlp/speech-resynth/img/fig.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/Images/logo-png.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/Images/logo-transparent-png.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/Images/shiftyspeech_logo.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/tts/Grad-TTS/out/sample_1.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/tts/Grad-TTS/out/sample_2.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/tts/Grad-TTS/resources/reverse-diffusion.gif filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/FreeV/figure/compare_table.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/FreeV/figure/overall.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/dance_24k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/hifitts_44k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/jensen_24k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/libritts_24k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/megalovania_24k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/musdbhq_44k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/musiccaps1_44k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/musiccaps2_44k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/demo/examples/queen_24k.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvgan/BigVGAN/filelists/LibriTTS/train-full.txt filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/bigvsan/LibriTTS/train-full.txt filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/iSTFTNet-pytorch/iSTFTnet.PNG filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/datasets/metadata/libritts_train_clean_360_audiopath_text_sid_train.txt filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/loss.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/model_architecture.png filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/2004_147967_000029_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/337_126286_000008_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/3537_5704_000008_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/5319_84357_000005_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/6294_86679_000035_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c16/949_134657_000002_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/2004_147967_000029_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/337_126286_000008_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/3537_5704_000008_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/5319_84357_000005_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/6294_86679_000035_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/c32/949_134657_000002_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/2004_147967_000029_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/337_126286_000008_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/3537_5704_000008_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/5319_84357_000005_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/6294_86679_000035_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/ground_truth/949_134657_000002_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c16/2004_147967_000029_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c16/337_126286_000008_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c16/3537_5704_000008_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c16/5319_84357_000005_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c16/6294_86679_000035_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c32/2004_147967_000029_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c32/337_126286_000008_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c32/3537_5704_000008_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c32/5319_84357_000005_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/seen/official_c32/6294_86679_000035_000004.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/1089_134686_000007_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/3575_170457_000037_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/4507_16021_000029_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/7021_85628_000037_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/7176_92135_000006_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c16/8224_274384_000016_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/1089_134686_000007_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/3575_170457_000037_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/4507_16021_000029_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/7021_85628_000037_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/7176_92135_000006_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/c32/8224_274384_000016_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/1089_134686_000007_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/3575_170457_000037_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/4507_16021_000029_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/7021_85628_000037_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/7176_92135_000006_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/ground_truth/8224_274384_000016_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c16/1089_134686_000007_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c16/3575_170457_000037_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c16/7021_85628_000037_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c16/7176_92135_000006_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c16/8224_274384_000016_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c32/1089_134686_000007_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c32/3575_170457_000037_000002.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c32/7021_85628_000037_000000.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c32/7176_92135_000006_000005.wav filter=lfs diff=lfs merge=lfs -text
+audio/shiftyspeech/speech_synthesis/vocoders/univnet/docs/samples/unseen/official_c32/8224_274384_000016_000000.wav filter=lfs diff=lfs merge=lfs -text
+image/aide/model_code/docs/Chameleon.jpg filter=lfs diff=lfs merge=lfs -text
+image/aide/model_code/docs/network.png filter=lfs diff=lfs merge=lfs -text
+video/fake-stormer/model_code/demo/method.png filter=lfs diff=lfs merge=lfs -text

audio/aasist3/.gitignore

@@ -0,0 +1 @@
+weights/

audio/aasist3/Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.10-slim
+
+# Install system dependencies (no git/build-essential needed)
+RUN apt-get update && apt-get install -y \
+    ffmpeg \
+    libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Pre-cache wav2vec2 config (not weights, just the config JSON ~1KB)
+RUN python -c "from transformers import Wav2Vec2Config; Wav2Vec2Config.from_pretrained('facebook/wav2vec2-large-xlsr-53', cache_dir='/app/w2v_cache')"
+
+# Copy model code and API
+COPY model/ /app/model/
+COPY api.py .
+
+# Expected weight files in /app/weights/:
+#   - model.safetensors  (AASIST3 checkpoint)
+#   - config.json        (model configuration)
+# These are mounted from the host via docker-compose volumes.
+
+EXPOSE 8005
+
+# Drop root privileges
+RUN adduser --disabled-password --gecos '' appuser
+USER appuser
+
+CMD ["python", "api.py"]

audio/aasist3/api.py

@@ -0,0 +1,314 @@
+"""AASIST3 Audio Deepfake Detection API.
+
+Detects synthetic speech using the AASIST3 model architecture:
+- Frontend: XLSR wav2vec 2.0 (HuggingFace Transformers)
+- Backend: AASIST with KAN (Kolmogorov-Arnold Network) linear
+  layers and Graph Attention Networks
+
+Reference: https://github.com/AI4Bharat/AASIST3
+"""
+
+import base64
+import io
+import logging
+import os
+import platform
+import time
+from typing import Optional
+
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+# Point transformers cache to pre-cached wav2vec2 config
+# (must be set before importing model code)
+os.environ["TRANSFORMERS_CACHE"] = "/app/w2v_cache"
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger("aasist3_api")
+
+# Import model class
+try:
+    from model import aasist3 as AASIST3Model
+except ImportError as e:
+    logger.error(f"Failed to import AASIST3 model: {e}")
+    AASIST3Model = None
+
+# Constants
+MODEL_NAME = "aasist3"
+MODEL_ID = "aasist3_kan_mlaad"
+WEIGHTS_DIR = "/app/weights"
+
+
+def _get_device():
+    """Select optimal device: MPS (Apple) > CUDA (NVIDIA) > CPU."""
+    override = os.environ.get("DEEPSAFE_DEVICE", "").strip().lower()
+    if override == "cpu":
+        return torch.device("cpu")
+    if override == "cuda" and torch.cuda.is_available():
+        return torch.device("cuda")
+    if (
+        override == "mps"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if override:
+        pass  # Invalid override, fall through to auto-detect
+    if (
+        platform.system() == "Darwin"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    return torch.device("cpu")
+
+
+DEVICE = _get_device()
+SAMPLE_RATE = 16000
+TARGET_SAMPLES = 64600  # ~4.04 seconds at 16kHz
+
+# Global model instance
+model = None
+
+
+class AudioInput(BaseModel):
+    """Request schema for audio deepfake detection."""
+
+    audio_data: str = Field(
+        ..., description="Base64 encoded audio string (WAV/MP3/etc)"
+    )
+    threshold: Optional[float] = Field(
+        0.5, ge=0.0, le=1.0, description="Classification threshold"
+    )
+
+
+app = FastAPI(
+    title="AASIST3 Audio Deepfake Detection API",
+    description=(
+        "Service for detecting synthetic speech using the "
+        "AASIST3 model (HuggingFace wav2vec 2.0 + AASIST "
+        "with KAN layers)."
+    ),
+    version="1.0.0",
+)
+
+
+def load_model():
+    """Load the AASIST3 model from pretrained weights.
+
+    Returns:
+        The loaded model, or None if loading fails.
+    """
+    global model
+    if model is not None:
+        return model
+
+    logger.info(f"Loading AASIST3 model onto {DEVICE}...")
+
+    if AASIST3Model is None:
+        logger.error("AASIST3 model class not available.")
+        return None
+
+    weights_safetensors = os.path.join(WEIGHTS_DIR, "model.safetensors")
+    weights_config = os.path.join(WEIGHTS_DIR, "config.json")
+
+    if not os.path.exists(weights_safetensors):
+        logger.error(f"Model weights not found at {weights_safetensors}")
+        return None
+
+    if not os.path.exists(weights_config):
+        logger.error(f"Model config not found at {weights_config}")
+        return None
+
+    try:
+        model = AASIST3Model.from_pretrained(WEIGHTS_DIR)
+        model.to(DEVICE)
+        # Set model to inference mode (disables dropout, batchnorm)
+        model.train(False)
+
+        logger.info("AASIST3 model loaded successfully.")
+        return model
+    except Exception as e:
+        logger.exception(f"Failed to load AASIST3 model: {e}")
+        model = None
+        return None
+
+
+@app.on_event("startup")
+async def startup_event():
+    """Load model on service startup."""
+    load_model()
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint."""
+    return {
+        "status": "healthy" if model is not None else "degraded",
+        "model": MODEL_NAME,
+        "model_id": MODEL_ID,
+        "device": str(DEVICE),
+        "weights_found": os.path.exists(os.path.join(WEIGHTS_DIR, "model.safetensors")),
+    }
+
+
+def _load_audio_bytes(audio_bytes: bytes) -> tuple:
+    """Load audio from raw bytes using soundfile with torchaudio fallback.
+
+    Args:
+        audio_bytes: Raw audio file bytes.
+
+    Returns:
+        Tuple of (audio_numpy_array, sample_rate).
+
+    Raises:
+        ValueError: If audio cannot be loaded by any backend.
+    """
+    # Try soundfile first (handles WAV, FLAC natively)
+    sf_error = None
+    try:
+        audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32")
+        if audio.ndim > 1:
+            audio = audio.mean(axis=1)  # Convert to mono
+        return audio, sr
+    except Exception as sf_err:
+        sf_error = sf_err
+        logger.debug(f"soundfile failed, trying torchaudio: {sf_err}")
+
+    # Fallback to torchaudio (handles MP3, compressed formats)
+    try:
+        import torchaudio
+
+        buf = io.BytesIO(audio_bytes)
+        waveform, sr = torchaudio.load(buf)
+        if waveform.shape[0] > 1:
+            waveform = waveform.mean(dim=0, keepdim=True)
+        return waveform.squeeze(0).numpy(), sr
+    except Exception as ta_err:
+        raise ValueError(
+            f"Failed to load audio with soundfile and torchaudio: "
+            f"sf={sf_error}, ta={ta_err}"
+        )
+
+
+def preprocess_audio(audio_bytes: bytes) -> torch.Tensor:
+    """Preprocess audio for AASIST3 inference.
+
+    Loads audio, resamples to 16kHz mono, and zero-pads or
+    truncates to TARGET_SAMPLES.
+
+    Args:
+        audio_bytes: Raw audio file bytes.
+
+    Returns:
+        Audio tensor of shape (1, TARGET_SAMPLES).
+
+    Raises:
+        ValueError: If audio preprocessing fails.
+    """
+    try:
+        logger.info("Starting audio preprocessing...")
+        audio, sr = _load_audio_bytes(audio_bytes)
+        logger.info(f"Audio loaded. Length: {len(audio)} samples at {sr}Hz")
+
+        # Resample to 16kHz if needed
+        if sr != SAMPLE_RATE:
+            import torchaudio
+
+            resampler = torchaudio.transforms.Resample(
+                orig_freq=sr, new_freq=SAMPLE_RATE
+            )
+            audio_tensor = torch.FloatTensor(audio).unsqueeze(0)
+            audio_tensor = resampler(audio_tensor).squeeze(0)
+            audio = audio_tensor.numpy()
+            logger.info(
+                f"Resampled from {sr}Hz to {SAMPLE_RATE}Hz. "
+                f"New length: {len(audio)} samples"
+            )
+
+        # Zero-pad or truncate to TARGET_SAMPLES
+        if len(audio) >= TARGET_SAMPLES:
+            audio = audio[:TARGET_SAMPLES]
+        else:
+            pad_length = TARGET_SAMPLES - len(audio)
+            audio = np.pad(audio, (0, pad_length), mode="constant")
+
+        logger.info(f"Audio padded/trimmed to {TARGET_SAMPLES} samples")
+
+        audio_tensor = torch.FloatTensor(audio).unsqueeze(0).to(DEVICE)
+        return audio_tensor
+    except Exception as e:
+        logger.error(f"Error preprocessing audio: {e}")
+        raise ValueError(f"Audio preprocessing failed: {str(e)}")
+
+
+@app.post("/predict")
+async def predict(input_data: AudioInput):
+    """Run deepfake detection on base64-encoded audio.
+
+    The model outputs 2 logits: [bonafide_score, spoof_score].
+    Class 0 = bonafide (real), Class 1 = spoof (fake).
+    The returned probability is the spoof/fake probability.
+    """
+    if model is None:
+        if load_model() is None:
+            raise HTTPException(status_code=503, detail="Model not loaded")
+
+    try:
+        start_time = time.time()
+        logger.info(
+            f"Prediction request. Data size: " f"{len(input_data.audio_data)} chars"
+        )
+
+        # Decode base64 audio
+        audio_bytes = base64.b64decode(input_data.audio_data)
+
+        # Preprocess
+        audio_tensor = preprocess_audio(audio_bytes)
+
+        # Inference
+        logger.info("Starting model inference...")
+        with torch.no_grad():
+            output = model(audio_tensor)
+
+            # output shape: [batch, 2]
+            # Index 0 = bonafide logit, Index 1 = spoof logit
+            probs = torch.softmax(output, dim=1)
+            prob_fake = probs[0, 1].item()
+
+        prediction = 1 if prob_fake >= input_data.threshold else 0
+        verdict = "fake" if prediction == 1 else "real"
+        inference_time = time.time() - start_time
+
+        logger.info(
+            f"Prediction: {verdict} (prob_fake={prob_fake:.4f}, "
+            f"time={inference_time:.3f}s)"
+        )
+
+        return {
+            "model": MODEL_NAME,
+            "probability": float(prob_fake),
+            "prediction": int(prediction),
+            "class": verdict,
+            "inference_time": float(inference_time),
+        }
+
+    except Exception as e:
+        logger.exception(f"Error during prediction: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("MODEL_PORT", 8005))
+    uvicorn.run(app, host="0.0.0.0", port=port)

audio/aasist3/model/__init__.py

@@ -0,0 +1 @@
+from .full_model import aasist3

audio/aasist3/model/branch.py

@@ -0,0 +1,34 @@
+import torch.nn as nn
+
+from .hs_gal import HtrgGraphAttentionLayer
+from .pool import GraphPool
+
+class InferenceBranch(nn.Module):
+    def __init__(self, gat_dims, temperature, pool_ratio, size):
+        super().__init__()
+        self.htrg_gat1 = HtrgGraphAttentionLayer(
+            gat_dims[0], gat_dims[1], temperature=temperature, size=size
+        )
+        self.htrg_gat2 = HtrgGraphAttentionLayer(
+            gat_dims[1], gat_dims[1], temperature=temperature, size=size
+        )
+
+        self.pool_hS = GraphPool(pool_ratio, gat_dims[1], 0.3, size=size)
+        self.pool_hT = GraphPool(pool_ratio, gat_dims[1], 0.3, size=size)
+
+    def forward(self, out_T, out_S, master):
+        # Первая стадия
+        out_T_res, out_S_res, master_res = self.htrg_gat1(out_T, out_S, master=master)
+
+        # Пулинг
+        out_S_res = self.pool_hS(out_S_res)
+        out_T_res = self.pool_hT(out_T_res)
+
+        # Вторая стадия с residual connection
+        out_T_aug, out_S_aug, master_aug = self.htrg_gat2(out_T_res, out_S_res, master=master_res)
+        
+        out_T_final = out_T_res + out_T_aug
+        out_S_final = out_S_res + out_S_aug
+        master_final = master_res + master_aug
+
+        return out_T_final, out_S_final, master_final

audio/aasist3/model/full_model.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+import torch
+import torch.nn.functional as F
+import torch.nn as nn
+from huggingface_hub import PyTorchModelHubMixin
+
+from .kan import KANLinear
+from .gat import GraphAttentionLayer
+from .pool import GraphPool
+from .branch import InferenceBranch
+from .residual import Residual_block
+from .wav2vec import Wav2Vec2Encoder
+
+
+class aasist3(nn.Module, PyTorchModelHubMixin):
+    def __init__(self, d_args={
+        "architecture": "AASIST",
+        "nb_samp": 64600,
+        "first_conv": 128,
+        "filts": [70, [1, 32], [32, 32], [32, 64], [64, 64]],
+        "gat_dims": [64, 32],
+        "pool_ratios": [0.5, 0.7, 0.5, 0.5],
+        "temperatures": [2.0, 2.0, 100.0, 100.0],
+    }, size=200, w2v_cache_dir="weights/", load_pretrained=True):
+        super().__init__()
+
+        self.w2v_encoder = Wav2Vec2Encoder(cache_dir=w2v_cache_dir, load_pretrained=load_pretrained)
+        self.bridge = KANLinear(1024, 128)
+
+        self.d_args = d_args
+        filts = d_args["filts"]
+        gat_dims = d_args["gat_dims"]
+        pool_ratios = d_args["pool_ratios"]
+        temperatures = d_args["temperatures"]
+
+        self.first_bn = nn.BatchNorm2d(num_features=1)
+        self.selu = nn.SELU(inplace=True)
+        self.drop = nn.Dropout(0.5, inplace=True)
+        self.drop_way = nn.Dropout(0.2, inplace=True)
+
+        self.encoder = nn.Sequential(
+            nn.Sequential(Residual_block(nb_filts=filts[1], first=True)),
+            nn.Sequential(Residual_block(nb_filts=filts[2])),
+            nn.Sequential(Residual_block(nb_filts=filts[3])),
+            nn.Sequential(Residual_block(nb_filts=filts[4])),
+            nn.Sequential(Residual_block(nb_filts=filts[4])),
+            nn.Sequential(Residual_block(nb_filts=filts[4])))
+
+        self.pos_S = nn.Parameter(torch.randn(1, 42, filts[-1][-1]))
+        self.pos_T = nn.Parameter(torch.randn(1, 67, filts[-1][-1]))
+
+        self.GAT_layer_S = GraphAttentionLayer(filts[-1][-1], gat_dims[0], temperature=temperatures[0], size=size)
+        self.GAT_layer_T = GraphAttentionLayer(filts[-1][-1], gat_dims[0], temperature=temperatures[1], size=size)
+
+        self.pool_S = GraphPool(pool_ratios[0], gat_dims[0], 0.3, size=size)
+        self.pool_T = GraphPool(pool_ratios[1], gat_dims[0], 0.3, size=size)
+
+        self.master1 = nn.Parameter(torch.randn(1, 1, gat_dims[0]))
+        self.master2 = nn.Parameter(torch.randn(1, 1, gat_dims[0]))
+        self.master3 = nn.Parameter(torch.randn(1, 1, gat_dims[0]))
+        self.master4 = nn.Parameter(torch.randn(1, 1, gat_dims[0]))
+
+        self.inference_branch1 = InferenceBranch(
+            gat_dims=gat_dims,
+            temperature=temperatures[2],
+            pool_ratio=pool_ratios[2],
+            size=size
+        )
+        self.inference_branch2 = InferenceBranch(
+            gat_dims=gat_dims,
+            temperature=temperatures[2],
+            pool_ratio=pool_ratios[2],
+            size=size
+        )
+        self.inference_branch3 = InferenceBranch(
+            gat_dims=gat_dims,
+            temperature=temperatures[2],
+            pool_ratio=pool_ratios[2],
+            size=size
+        )
+        self.inference_branch4 = InferenceBranch(
+            gat_dims=gat_dims,
+            temperature=temperatures[2],
+            pool_ratio=pool_ratios[2],
+            size=size
+        )
+
+        self.out_layer = KANLinear(5 * gat_dims[1], 2)
+
+    def forward(self, x, Freq_aug=False):
+        x = self.w2v_encoder(x)
+        x = self.bridge(x)
+        x = x.transpose(1, 2)
+        x = x.unsqueeze(dim=1)
+        x = F.max_pool2d(torch.abs(x), (3, 3))
+        x = self.first_bn(x)
+        x = self.selu(x)
+
+        e = self.encoder(x)
+
+        # GAT-S
+        e_S, _ = torch.max(torch.abs(e), dim=3)
+        e_S = e_S.transpose(1, 2) + self.pos_S
+        gat_S = self.GAT_layer_S(e_S)
+        out_S = self.pool_S(gat_S)
+
+        # GAT-T
+        e_T, _ = torch.max(torch.abs(e), dim=2)
+        e_T = e_T.transpose(1, 2) + self.pos_T
+        gat_T = self.GAT_layer_T(e_T)
+        out_T = self.pool_T(gat_T)
+
+        out_T1, out_S1, master1 = self.inference_branch1(out_T, out_S, self.master1)
+        out_T2, out_S2, master2 = self.inference_branch2(out_T, out_S, self.master2)
+        out_T3, out_S3, master3 = self.inference_branch3(out_T, out_S, self.master3)
+        out_T4, out_S4, master4 = self.inference_branch4(out_T, out_S, self.master4)
+
+        out_T1, out_T2 = self.drop_way(out_T1), self.drop_way(out_T2)
+        out_T3, out_T4 = self.drop_way(out_T3), self.drop_way(out_T4)
+        out_S1, out_S2 = self.drop_way(out_S1), self.drop_way(out_S2)
+        out_S3, out_S4 = self.drop_way(out_S3), self.drop_way(out_S4)
+        master1, master2 = self.drop_way(master1), self.drop_way(master2)
+        master3, master4 = self.drop_way(master3), self.drop_way(master4)
+
+        out_T = torch.stack([out_T1, out_T2, out_T3, out_T4]).max(dim=0)[0]
+        out_S = torch.stack([out_S1, out_S2, out_S3, out_S4]).max(dim=0)[0]
+        master = torch.stack([master1, master2, master3, master4]).max(dim=0)[0]
+
+        T_max, _ = torch.max(torch.abs(out_T), dim=1)
+        T_avg = torch.mean(out_T, dim=1)
+        S_max, _ = torch.max(torch.abs(out_S), dim=1)
+        S_avg = torch.mean(out_S, dim=1)
+
+        last_hidden = torch.cat([T_max, T_avg, S_max, S_avg, master.squeeze(1)], dim=1)
+        last_hidden = self.drop(last_hidden)
+        output = self.out_layer(last_hidden)
+
+        return output

audio/aasist3/model/gat.py

@@ -0,0 +1,99 @@
+import torch, torch.nn as nn, torch.nn.functional as F
+
+from .kan import KANLinear
+
+
+class GraphAttentionLayer(nn.Module):
+    def __init__(self, in_dim, out_dim, size, **kwargs):
+        super().__init__()
+
+        # attention map
+        self.att_proj = KANLinear(in_dim, out_dim)
+        self.att_weight = self._init_new_params(out_dim, 1)
+
+        # project
+        self.proj_with_att = KANLinear(in_dim, out_dim)
+        self.proj_without_att = KANLinear(in_dim, out_dim)
+
+        # batch norm
+        self.bn = nn.BatchNorm1d(out_dim)
+
+        # dropout for inputs
+        self.input_drop = nn.Dropout(p=0.2)
+
+        # activate
+        self.act = nn.SELU(inplace=True)
+
+        # temperature
+        self.temp = 1.
+        if "temperature" in kwargs:
+            self.temp = kwargs["temperature"]
+
+    def forward(self, x):
+        '''
+        x   :(#bs, #node, #dim)
+        '''
+        # apply input dropout
+        x = self.input_drop(x)
+
+        # derive attention map
+        att_map = self._derive_att_map(x)
+
+        # projection
+        x = self._project(x, att_map)
+
+        # apply batch norm
+        x = self._apply_BN(x)
+        x = self.act(x)
+        return x
+
+    def _pairwise_mul_nodes(self, x):
+        '''
+        Calculates pairwise multiplication of nodes.
+        - for attention map
+        x           :(#bs, #node, #dim)
+        out_shape   :(#bs, #node, #node, #dim)
+        '''
+
+        nb_nodes = x.size(1)
+        x = x.unsqueeze(2).expand(-1, -1, nb_nodes, -1)
+        x_mirror = x.transpose(1, 2)
+
+        return x * x_mirror
+
+    def _derive_att_map(self, x):
+        '''
+        x           :(#bs, #node, #dim)
+        out_shape   :(#bs, #node, #node, 1)
+        '''
+        att_map = self._pairwise_mul_nodes(x)
+        # size: (#bs, #node, #node, #dim_out)
+        att_map = torch.tanh(self.att_proj(att_map))
+        # size: (#bs, #node, #node, 1)
+        att_map = torch.matmul(att_map, self.att_weight)
+
+        # apply temperature
+        att_map = att_map / self.temp
+
+        att_map = F.softmax(att_map, dim=-2)
+
+        return att_map
+
+    def _project(self, x, att_map):
+        x1 = self.proj_with_att(torch.matmul(att_map.squeeze(-1), x))
+        x2 = self.proj_without_att(x)
+
+        return x1 + x2
+
+    def _apply_BN(self, x):
+        org_size = x.size()
+        x = x.view(-1, org_size[-1])
+        x = self.bn(x)
+        x = x.view(org_size)
+
+        return x
+
+    def _init_new_params(self, *size):
+        out = nn.Parameter(torch.FloatTensor(*size))
+        nn.init.xavier_normal_(out)
+        return out

audio/aasist3/model/hs_gal.py

@@ -0,0 +1,176 @@
+import torch, torch.nn as nn, torch.nn.functional as F
+
+from .kan import KANLinear
+
+
+class HtrgGraphAttentionLayer(nn.Module):
+    def __init__(self, in_dim, out_dim, size, **kwargs):
+        super().__init__()
+
+        self.proj_type1 = KANLinear(in_dim, in_dim)
+        self.proj_type2 = KANLinear(in_dim, in_dim)
+
+        # attention map
+        self.att_proj = KANLinear(in_dim, out_dim)
+        self.att_projM = KANLinear(in_dim, out_dim)
+
+        self.att_weight11 = self._init_new_params(out_dim, 1)
+        self.att_weight22 = self._init_new_params(out_dim, 1)
+        self.att_weight12 = self._init_new_params(out_dim, 1)
+        self.att_weightM = self._init_new_params(out_dim, 1)
+
+        # project
+        self.proj_with_att = KANLinear(in_dim, out_dim)
+        self.proj_without_att = KANLinear(in_dim, out_dim)
+
+        self.proj_with_attM = KANLinear(in_dim, out_dim)
+        self.proj_without_attM = KANLinear(in_dim, out_dim)
+
+        # batch norm
+        self.bn = nn.BatchNorm1d(out_dim)
+
+        # dropout for inputs
+        self.input_drop = nn.Dropout(p=0.2)
+
+        # activate
+        self.act = nn.SELU(inplace=True)
+
+        # temperature
+        self.temp = 1.
+        if "temperature" in kwargs:
+            self.temp = kwargs["temperature"]
+
+    def forward(self, x1, x2, master=None):
+        '''
+        x1  :(#bs, #node, #dim)
+        x2  :(#bs, #node, #dim)
+        '''
+        num_type1 = x1.size(1)
+        num_type2 = x2.size(1)
+
+        x1 = self.proj_type1(x1)
+        x2 = self.proj_type2(x2)
+
+        x = torch.cat([x1, x2], dim=1)
+
+        if master is None:
+            master = torch.mean(x, dim=1, keepdim=True)
+
+        # apply input dropout
+        x = self.input_drop(x)
+
+        # derive attention map
+        att_map = self._derive_att_map(x, num_type1, num_type2)
+
+        # directional edge for master node
+        master = self._update_master(x, master)
+
+        # projection
+        x = self._project(x, att_map)
+
+        # apply batch norm
+        x = self._apply_BN(x)
+        # x = self.act(x)
+
+        x1 = x.narrow(1, 0, num_type1)
+        x2 = x.narrow(1, num_type1, num_type2)
+
+        return x1, x2, master
+
+    def _update_master(self, x, master):
+
+        att_map = self._derive_att_map_master(x, master)
+        master = self._project_master(x, master, att_map)
+
+        return master
+
+    def _pairwise_mul_nodes(self, x):
+        '''
+        Calculates pairwise multiplication of nodes.
+        - for attention map
+        x           :(#bs, #node, #dim)
+        out_shape   :(#bs, #node, #node, #dim)
+        '''
+
+        nb_nodes = x.size(1)
+        x = x.unsqueeze(2).expand(-1, -1, nb_nodes, -1)
+        x_mirror = x.transpose(1, 2)
+
+        return x * x_mirror
+
+    def _derive_att_map_master(self, x, master):
+        '''
+        x           :(#bs, #node, #dim)
+        out_shape   :(#bs, #node, #node, 1)
+        '''
+        att_map = x * master
+        att_map = torch.tanh(self.att_projM(att_map))
+
+        att_map = torch.matmul(att_map, self.att_weightM)
+
+        # apply temperature
+        att_map = att_map / self.temp
+
+        att_map = F.softmax(att_map, dim=-2)
+
+        return att_map
+
+    def _derive_att_map(self, x, num_type1, num_type2):
+        '''
+        x           :(#bs, #node, #dim)
+        out_shape   :(#bs, #node, #node, 1)
+        '''
+        att_map = self._pairwise_mul_nodes(x)
+        # size: (#bs, #node, #node, #dim_out)
+        att_map = torch.tanh(self.att_proj(att_map))
+        # size: (#bs, #node, #node, 1)
+
+        att_board = torch.zeros_like(att_map[:, :, :, 0]).unsqueeze(-1)
+
+        att_board[:, :num_type1, :num_type1, :] = torch.matmul(
+            att_map[:, :num_type1, :num_type1, :], self.att_weight11)
+        att_board[:, num_type1:, num_type1:, :] = torch.matmul(
+            att_map[:, num_type1:, num_type1:, :], self.att_weight22)
+        att_board[:, :num_type1, num_type1:, :] = torch.matmul(
+            att_map[:, :num_type1, num_type1:, :], self.att_weight12)
+        att_board[:, num_type1:, :num_type1, :] = torch.matmul(
+            att_map[:, num_type1:, :num_type1, :], self.att_weight12)
+
+        att_map = att_board
+
+        # att_map = torch.matmul(att_map, self.att_weight12)
+
+        # apply temperature
+        att_map = att_map / self.temp
+
+        att_map = F.softmax(att_map, dim=-2)
+
+        return att_map
+
+    def _project(self, x, att_map):
+        x1 = self.proj_with_att(torch.matmul(att_map.squeeze(-1), x))
+        x2 = self.proj_without_att(x)
+
+        return x1 + x2
+
+    def _project_master(self, x, master, att_map):
+
+        x1 = self.proj_with_attM(torch.matmul(
+            att_map.squeeze(-1).unsqueeze(1), x))
+        x2 = self.proj_without_attM(master)
+
+        return x1 + x2
+
+    def _apply_BN(self, x):
+        org_size = x.size()
+        x = x.view(-1, org_size[-1])
+        x = self.bn(x)
+        x = x.view(org_size)
+
+        return x
+
+    def _init_new_params(self, *size):
+        out = nn.Parameter(torch.FloatTensor(*size))
+        nn.init.xavier_normal_(out)
+        return out
+

audio/aasist3/model/kan.py

@@ -0,0 +1,213 @@
+import torch, math, torch.nn.functional as F
+
+
+class KANLinear(torch.nn.Module):
+    def __init__(
+        self,
+        in_features,
+        out_features,
+        grid_size=16,
+        spline_order=4,
+        scale_noise=0.1,
+        scale_base=1.0,
+        scale_spline=1.0,
+        enable_standalone_scale_spline=True,
+        base_activation=torch.nn.PReLU,
+        grid_eps=0.02,
+        grid_range=[-1, 1],
+    ):
+        super(KANLinear, self).__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+        self.grid_size = grid_size
+        self.spline_order = spline_order
+
+        h = (grid_range[1] - grid_range[0]) / grid_size
+        grid = (
+            (
+                torch.arange(-spline_order, grid_size + spline_order + 1) * h
+                + grid_range[0]
+            )
+            .expand(in_features, -1)
+            .contiguous()
+        )
+        self.register_buffer("grid", grid)
+
+        self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))
+        self.spline_weight = torch.nn.Parameter(
+            torch.Tensor(out_features, in_features, grid_size + spline_order)
+        )
+        if enable_standalone_scale_spline:
+            self.spline_scaler = torch.nn.Parameter(
+                torch.Tensor(out_features, in_features)
+            )
+
+        self.scale_noise = scale_noise
+        self.scale_base = scale_base
+        self.scale_spline = scale_spline
+        self.enable_standalone_scale_spline = enable_standalone_scale_spline
+        self.base_activation = base_activation()
+        self.grid_eps = grid_eps
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)
+        with torch.no_grad():
+            noise = (
+                (
+                    torch.rand(self.grid_size + 1, self.in_features, self.out_features)
+                    - 1 / 2
+                )
+                * self.scale_noise
+                / self.grid_size
+            )
+            self.spline_weight.data.copy_(
+                (self.scale_spline if not self.enable_standalone_scale_spline else 1.0)
+                * self.curve2coeff(
+                    self.grid.T[self.spline_order : -self.spline_order],
+                    noise,
+                )
+            )
+            if self.enable_standalone_scale_spline:
+                # torch.nn.init.constant_(self.spline_scaler, self.scale_spline)
+                torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)
+
+    def b_splines(self, x: torch.Tensor):
+        """
+        Compute the B-spline bases for the given input tensor.
+
+        Args:
+            x (torch.Tensor): Input tensor of shape (batch_size, in_features).
+
+        Returns:
+            torch.Tensor: B-spline bases tensor of shape (batch_size, in_features, grid_size + spline_order).
+        """
+        assert x.dim() == 2 and x.size(1) == self.in_features
+
+        grid: torch.Tensor = (
+            self.grid
+        )  # (in_features, grid_size + 2 * spline_order + 1)
+        x = x.unsqueeze(-1)
+        bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)
+        for k in range(1, self.spline_order + 1):
+            bases = (
+                (x - grid[:, : -(k + 1)])
+                / (grid[:, k:-1] - grid[:, : -(k + 1)])
+                * bases[:, :, :-1]
+            ) + (
+                (grid[:, k + 1 :] - x)
+                / (grid[:, k + 1 :] - grid[:, 1:(-k)])
+                * bases[:, :, 1:]
+            )
+
+        assert bases.size() == (
+            x.size(0),
+            self.in_features,
+            self.grid_size + self.spline_order,
+        )
+        return bases.contiguous()
+
+    def curve2coeff(self, x: torch.Tensor, y: torch.Tensor):
+        """
+        Compute the coefficients of the curve that interpolates the given points.
+
+        Args:
+            x (torch.Tensor): Input tensor of shape (batch_size, in_features).
+            y (torch.Tensor): Output tensor of shape (batch_size, in_features, out_features).
+
+        Returns:
+            torch.Tensor: Coefficients tensor of shape (out_features, in_features, grid_size + spline_order).
+        """
+        assert x.dim() == 2 and x.size(1) == self.in_features
+        assert y.size() == (x.size(0), self.in_features, self.out_features)
+
+        A = self.b_splines(x).transpose(
+            0, 1
+        )  # (in_features, batch_size, grid_size + spline_order)
+        B = y.transpose(0, 1)  # (in_features, batch_size, out_features)
+        solution = torch.linalg.lstsq(
+            A, B
+        ).solution  # (in_features, grid_size + spline_order, out_features)
+        result = solution.permute(
+            2, 0, 1
+        )  # (out_features, in_features, grid_size + spline_order)
+
+        assert result.size() == (
+            self.out_features,
+            self.in_features,
+            self.grid_size + self.spline_order,
+        )
+        return result.contiguous()
+
+    @property
+    def scaled_spline_weight(self):
+        return self.spline_weight * (
+            self.spline_scaler.unsqueeze(-1)
+            if self.enable_standalone_scale_spline
+            else 1.0
+        )
+
+    def forward(self, x: torch.Tensor):
+        assert x.size(-1) == self.in_features
+        original_shape = x.shape
+        x = x.reshape(-1, self.in_features)
+
+        base_output = F.linear(self.base_activation(x), self.base_weight)
+        spline_output = F.linear(
+            self.b_splines(x).view(x.size(0), -1),
+            self.scaled_spline_weight.reshape(self.out_features, -1),
+        )
+        output = base_output + spline_output
+        # print(*original_shape[:-1], output.shape)
+        output = output.view(*original_shape[:-1], self.out_features)
+        return output
+
+    @torch.no_grad()
+    def update_grid(self, x: torch.Tensor, margin=0.01):
+        assert x.dim() == 2 and x.size(1) == self.in_features
+        batch = x.size(0)
+
+        splines = self.b_splines(x)  # (batch, in, coeff)
+        splines = splines.permute(1, 0, 2)  # (in, batch, coeff)
+        orig_coeff = self.scaled_spline_weight  # (out, in, coeff)
+        orig_coeff = orig_coeff.permute(1, 2, 0)  # (in, coeff, out)
+        unreduced_spline_output = torch.bmm(splines, orig_coeff)  # (in, batch, out)
+        unreduced_spline_output = unreduced_spline_output.permute(
+            1, 0, 2
+        )  # (batch, in, out)
+
+        # sort each channel individually to collect data distribution
+        x_sorted = torch.sort(x, dim=0)[0]
+        grid_adaptive = x_sorted[
+            torch.linspace(
+                0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device
+            )
+        ]
+
+        uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size
+        grid_uniform = (
+            torch.arange(
+                self.grid_size + 1, dtype=torch.float32, device=x.device
+            ).unsqueeze(1)
+            * uniform_step
+            + x_sorted[0]
+            - margin
+        )
+
+        grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive
+        grid = torch.concatenate(
+            [
+                grid[:1]
+                - uniform_step
+                * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1),
+                grid,
+                grid[-1:]
+                + uniform_step
+                * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),
+            ],
+            dim=0,
+        )
+
+        self.grid.copy_(grid.T)
+        self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))

audio/aasist3/model/pool.py

@@ -0,0 +1,45 @@
+import torch, torch.nn as nn
+from typing import Union
+
+from .kan import KANLinear
+
+
+class GraphPool(nn.Module):
+    def __init__(self, k: float, in_dim: int, p: Union[float, int], size):
+        super().__init__()
+        self.k = k
+        self.sigmoid = nn.Sigmoid()
+        self.proj = KANLinear(in_dim, 1)
+        self.drop = nn.Dropout(p=p) if p > 0 else nn.Identity()
+        self.in_dim = in_dim
+
+    def forward(self, h):
+        Z = self.drop(h)
+        weights = self.proj(Z)
+        scores = self.sigmoid(weights)
+        new_h = self.top_k_graph(scores, h, self.k)
+
+        return new_h
+
+    def top_k_graph(self, scores, h, k):
+        """
+        args
+        =====
+        scores: attention-based weights (#bs, #node, 1)
+        h: graph data (#bs, #node, #dim)
+        k: ratio of remaining nodes, (float)
+
+        returns
+        =====
+        h: graph pool applied data (#bs, #node', #dim)
+        """
+        _, n_nodes, n_feat = h.size()
+        n_nodes = max(int(n_nodes * k), 1)
+        _, idx = torch.topk(scores, n_nodes, dim=1)
+        idx = idx.expand(-1, -1, n_feat)
+
+        h = h * scores
+        h = torch.gather(h, 1, idx)
+
+        return h
+

audio/aasist3/model/residual.py

@@ -0,0 +1,56 @@
+import torch.nn as nn
+
+class Residual_block(nn.Module):
+    def __init__(self, nb_filts, first=False):
+        super().__init__()
+        self.first = first
+
+        if not self.first:
+            self.bn1 = nn.BatchNorm2d(num_features=nb_filts[0])
+        self.conv1 = nn.Conv2d(in_channels=nb_filts[0],
+                               out_channels=nb_filts[1],
+                               kernel_size=(2, 3),
+                               padding=(1, 1),
+                               stride=1)
+        self.selu = nn.SELU()
+
+        self.bn2 = nn.BatchNorm2d(num_features=nb_filts[1])
+        self.conv2 = nn.Conv2d(in_channels=nb_filts[1],
+                               out_channels=nb_filts[1],
+                               kernel_size=(2, 3),
+                               padding=(0, 1),
+                               stride=1)
+
+        if nb_filts[0] != nb_filts[1]:
+            self.downsample = True
+            self.conv_downsample = nn.Conv2d(in_channels=nb_filts[0],
+                                             out_channels=nb_filts[1],
+                                             padding=(0, 1),
+                                             kernel_size=(1, 3),
+                                             stride=1)
+
+        else:
+            self.downsample = False
+        # self.mp = nn.MaxPool2d((1, 3))  # self.mp = nn.MaxPool2d((1,4))
+
+    def forward(self, x):
+        identity = x
+        if not self.first:
+            out = self.bn1(x)
+            out = self.selu(out)
+        else:
+            out = x
+        out = self.conv1(x)
+
+        # print('out',out.shape)
+        out = self.bn2(out)
+        out = self.selu(out)
+        # print('out',out.shape)
+        out = self.conv2(out)
+        #print('conv2 out',out.shape)
+        if self.downsample:
+            identity = self.conv_downsample(identity)
+
+        out += identity
+        # out = self.mp(out)
+        return out

audio/aasist3/model/wav2vec.py

@@ -0,0 +1,82 @@
+import torch, torch.nn as nn
+from transformers import Wav2Vec2Model, Wav2Vec2Config
+
+
+class Wav2Vec2Encoder(nn.Module):
+    """SSL encoder based on Hugging Face's Wav2Vec2 model."""
+
+    def __init__(self,
+                 model_name_or_path: str = "facebook/wav2vec2-large-xlsr-53",
+                 ssl_out_dim: int = 768,
+                 use_ssl_n_layers: int = None,
+                 freeze_ssl_n_layers: int = 0,
+                 output_attentions: bool = False,
+                 output_hidden_states: bool = False,
+                 normalize_waveform: bool = True,
+                 cache_dir: str = "weights",
+                 load_pretrained: bool = True):
+        """Initialize the Wav2Vec2 encoder.
+
+        Args:
+            model_name_or_path: HuggingFace model name or path to local model.
+            ssl_out_dim: Output dimension of the Wav2Vec2 encoder.
+            use_ssl_n_layers: Number of Wav2Vec2 layers to use. If None, use all layers.
+            freeze_ssl_n_layers: Number of Wav2Vec2 layers to freeze during training.
+            output_attentions: Whether to output attentions.
+            output_hidden_states: Whether to output hidden states.
+            normalize_waveform: Whether to normalize the waveform input.
+            cache_dir: Directory to cache pretrained models.
+            load_pretrained: Whether to load pretrained weights. If False, initializes with random weights.
+        """
+        super().__init__()
+
+        self.model_name_or_path = model_name_or_path
+        self.ssl_out_dim = ssl_out_dim
+        self.use_ssl_n_layers = use_ssl_n_layers
+        self.freeze_ssl_n_layers = freeze_ssl_n_layers
+        self.output_attentions = output_attentions
+        self.output_hidden_states = output_hidden_states
+        self.normalize_waveform = normalize_waveform
+
+        if load_pretrained:
+            self.model = Wav2Vec2Model.from_pretrained(model_name_or_path, cache_dir=cache_dir)
+        else:
+            config = Wav2Vec2Config.from_pretrained(
+                model_name_or_path, 
+                cache_dir=cache_dir,
+                local_files_only=False
+            )
+            self.model = Wav2Vec2Model(config)
+            self.model.init_weights()
+
+    def forward(self, x):
+        """Forward pass through the Wav2Vec2 encoder.
+
+        Args:
+            x: Input tensor of shape (batch_size, sequence_length, channels)
+
+        Returns:
+            Extracted features of shape (batch_size, sequence_length, ssl_out_dim)
+        """
+        # Handle shape: convert (batch_size, sequence_length, channels) to (batch_size, sequence_length)
+        if x.ndim == 3:
+            x = x.squeeze(-1)  # Remove channel dimension if present
+
+        if self.normalize_waveform:
+            x = x / (torch.max(torch.abs(x), dim=1, keepdim=True)[0] + 1e-8)
+
+        outputs = self.model(
+            x,
+            output_attentions=self.output_attentions,
+            output_hidden_states=self.output_hidden_states,
+            return_dict=True
+        )
+
+        last_hidden_state = outputs.last_hidden_state
+
+        if self.use_ssl_n_layers is not None and self.output_hidden_states and outputs.hidden_states is not None:
+            selected = outputs.hidden_states[-self.use_ssl_n_layers:]
+            last_hidden_state = torch.mean(torch.stack(selected, dim=0), dim=0)
+        del outputs
+
+        return last_hidden_state

audio/aasist3/requirements.txt

@@ -0,0 +1,11 @@
+torch==2.5.0
+torchaudio==2.5.0
+transformers>=4.40.0
+huggingface-hub>=0.20.0
+safetensors>=0.4.0
+numpy<2.0
+soundfile
+fastapi
+uvicorn[standard]
+pydantic
+python-multipart

audio/aasist3/weights/config.json

@@ -0,0 +1,45 @@
+{
+  "d_args": {
+    "architecture": "AASIST",
+    "filts": [
+      70,
+      [
+        1,
+        32
+      ],
+      [
+        32,
+        32
+      ],
+      [
+        32,
+        64
+      ],
+      [
+        64,
+        64
+      ]
+    ],
+    "first_conv": 128,
+    "gat_dims": [
+      64,
+      32
+    ],
+    "nb_samp": 64600,
+    "pool_ratios": [
+      0.5,
+      0.7,
+      0.5,
+      0.5
+    ],
+    "temperatures": [
+      2.0,
+      2.0,
+      100.0,
+      100.0
+    ]
+  },
+  "load_pretrained": false,
+  "size": 200,
+  "w2v_cache_dir": "/app/w2v_cache"
+}

audio/aasist3/weights/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a06d43d8d4a7e11b62fb4fa833f13a598fee23be7c368c416df4bec2e44e664f
+size 1287120456

audio/nes2net/.gitignore

@@ -0,0 +1 @@
+weights/

audio/nes2net/Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.10-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    ffmpeg \
+    libsndfile1 \
+    wget \
+    build-essential \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Clone fairseq at the specific commit required by Nes2Net,
+# then patch setup.py to skip C extensions (not needed for
+# checkpoint_utils), and install as editable without deps.
+RUN git clone https://github.com/facebookresearch/fairseq.git /app/fairseq_repo && \
+    cd /app/fairseq_repo && \
+    git checkout a54021305d6b3c4c5959ac9395135f63202db8f1 && \
+    sed -i 's/ext_modules=extensions/ext_modules=[]/' setup.py && \
+    pip install --no-cache-dir --no-deps -e .
+
+# Copy model scripts
+COPY model_scripts /app/model_scripts
+
+# Copy API code
+COPY api.py .
+
+# Model weights are mounted from the host via docker-compose volumes.
+# Expected weight files in /app/weights/:
+#   - nes2net_itw_valaug.pt  (fine-tuned Nes2Net checkpoint)
+#   - xlsr2_300m.pt          (XLSR wav2vec 2.0 for architecture init)
+
+EXPOSE 8004
+
+# Drop root privileges
+RUN adduser --disabled-password --gecos '' appuser
+USER appuser
+
+CMD ["python", "api.py"]

audio/nes2net/api.py

@@ -0,0 +1,284 @@
+"""Nes2Net (XLSR + Nested Res2Net TDNN) Audio Deepfake Detection API.
+
+Detects synthetic speech using the Nes2Net model architecture:
+- Frontend: XLSR wav2vec 2.0 (Self-Supervised Learning)
+- Backend: Nested Res2Net TDNN with SE modules
+
+Reference: https://github.com/TianchiLiu/Nes2Net
+"""
+
+import argparse
+import base64
+import io
+import logging
+import os
+import platform
+import sys
+import time
+import warnings
+from typing import Optional
+
+import librosa
+import numpy as np
+import torch
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+# Suppress deprecation warnings from fairseq/omegaconf compatibility
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+# Monkey-patch omegaconf for fairseq compatibility (older fairseq
+# expects is_primitive_type which was removed in newer omegaconf).
+import omegaconf._utils as _omegaconf_utils
+
+if not hasattr(_omegaconf_utils, "is_primitive_type"):
+    _omegaconf_utils.is_primitive_type = lambda t: t in (int, float, bool, str, bytes)
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger("nes2net_api")
+
+# Add the model code to the path
+if "/app" not in sys.path:
+    sys.path.insert(0, "/app")
+
+# Import model class (deferred to allow path setup)
+try:
+    from model_scripts.wav2vec2_Nes2Net_X import (
+        wav2vec2_Nes2Net_no_Res_w_allT as Nes2NetModel,
+    )
+except ImportError as e:
+    logger.error(f"Failed to import Nes2Net model: {e}")
+    Nes2NetModel = None
+
+# Constants
+MODEL_NAME = "nes2net"
+MODEL_ID = "nes2net_xlsr_itw_valaug"
+WEIGHTS_PATH = "/app/weights/nes2net_itw_valaug.pt"
+
+
+def _get_device():
+    """Select optimal device: MPS (Apple) > CUDA (NVIDIA) > CPU."""
+    override = os.environ.get("DEEPSAFE_DEVICE", "").strip().lower()
+    if override == "cpu":
+        return torch.device("cpu")
+    if override == "cuda" and torch.cuda.is_available():
+        return torch.device("cuda")
+    if (
+        override == "mps"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if override:
+        pass  # Invalid override, fall through to auto-detect
+    if (
+        platform.system() == "Darwin"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    return torch.device("cpu")
+
+
+DEVICE = _get_device()
+SAMPLE_RATE = 16000
+TARGET_SAMPLES = 64600  # ~4.04 seconds at 16kHz
+
+# Global model instance
+model = None
+
+
+class AudioInput(BaseModel):
+    """Request schema for audio deepfake detection."""
+
+    audio_data: str = Field(
+        ..., description="Base64 encoded audio string (WAV/MP3/etc)"
+    )
+    threshold: Optional[float] = Field(
+        0.5, ge=0.0, le=1.0, description="Classification threshold"
+    )
+
+
+app = FastAPI(
+    title="Nes2Net Audio Deepfake Detection API",
+    description=(
+        "Service for detecting synthetic speech using the "
+        "Nes2Net model (XLSR wav2vec 2.0 + Nested Res2Net TDNN)."
+    ),
+    version="1.0.0",
+)
+
+
+def load_model():
+    """Load the Nes2Net model with fine-tuned weights.
+
+    Returns:
+        The loaded model, or None if loading fails.
+    """
+    global model
+    if model is not None:
+        return model
+
+    logger.info(f"Loading Nes2Net model onto {DEVICE}...")
+
+    if Nes2NetModel is None:
+        logger.error("Nes2Net model class not available.")
+        return None
+
+    if not os.path.exists(WEIGHTS_PATH):
+        logger.error(f"Model weights not found at {WEIGHTS_PATH}")
+        return None
+
+    try:
+        args = argparse.Namespace(
+            n_output_logits=2,
+            dilation=2,
+            pool_func="mean",
+            SE_ratio=[1],
+            Nes_ratio=[8, 8],
+        )
+        model = Nes2NetModel(args, str(DEVICE))
+
+        # Load fine-tuned weights
+        try:
+            state_dict = torch.load(
+                WEIGHTS_PATH,
+                map_location=DEVICE,
+                weights_only=False,
+            )
+        except TypeError:
+            state_dict = torch.load(WEIGHTS_PATH, map_location=DEVICE)
+
+        model.load_state_dict(state_dict)
+        model.to(DEVICE)
+        model.eval()
+
+        logger.info("Nes2Net model loaded successfully.")
+        return model
+    except Exception as e:
+        logger.exception(f"Failed to load Nes2Net model: {e}")
+        model = None
+        return None
+
+
+@app.on_event("startup")
+async def startup_event():
+    """Load model on service startup."""
+    load_model()
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint."""
+    return {
+        "status": "healthy" if model is not None else "degraded",
+        "model": MODEL_NAME,
+        "model_id": MODEL_ID,
+        "device": str(DEVICE),
+        "weights_found": os.path.exists(WEIGHTS_PATH),
+    }
+
+
+def preprocess_audio(audio_bytes: bytes) -> torch.Tensor:
+    """Preprocess audio for Nes2Net inference.
+
+    Loads audio, resamples to 16kHz mono, and pads/trims
+    to TARGET_SAMPLES using tiling (matching original training
+    preprocessing).
+
+    Args:
+        audio_bytes: Raw audio file bytes.
+
+    Returns:
+        Audio tensor of shape (1, TARGET_SAMPLES).
+
+    Raises:
+        ValueError: If audio preprocessing fails.
+    """
+    try:
+        logger.info("Starting audio preprocessing...")
+        audio, sr = librosa.load(io.BytesIO(audio_bytes), sr=SAMPLE_RATE, mono=True)
+        logger.info(f"Audio loaded. Length: {len(audio)} samples at {sr}Hz")
+
+        # Pad/trim to TARGET_SAMPLES using tiling
+        if len(audio) >= TARGET_SAMPLES:
+            audio = audio[:TARGET_SAMPLES]
+        else:
+            num_repeats = TARGET_SAMPLES // len(audio) + 1
+            audio = np.tile(audio, num_repeats)[:TARGET_SAMPLES]
+
+        logger.info(f"Audio padded/trimmed to {TARGET_SAMPLES} samples")
+
+        audio_tensor = torch.FloatTensor(audio).unsqueeze(0).to(DEVICE)
+        return audio_tensor
+    except Exception as e:
+        logger.error(f"Error preprocessing audio: {e}")
+        raise ValueError(f"Audio preprocessing failed: {str(e)}")
+
+
+@app.post("/predict")
+async def predict(input_data: AudioInput):
+    """Run deepfake detection on base64-encoded audio.
+
+    The model outputs 2 logits: [spoof_score, bonafide_score].
+    Class 0 = spoof (fake), Class 1 = bonafide (real).
+    The returned probability is the spoof/fake probability.
+    """
+    if model is None:
+        if load_model() is None:
+            raise HTTPException(status_code=503, detail="Model not loaded")
+
+    try:
+        start_time = time.time()
+        logger.info(
+            f"Prediction request. Data size: " f"{len(input_data.audio_data)} chars"
+        )
+
+        # Decode base64 audio
+        audio_bytes = base64.b64decode(input_data.audio_data)
+
+        # Preprocess
+        audio_tensor = preprocess_audio(audio_bytes)
+
+        # Inference
+        logger.info("Starting model inference...")
+        with torch.no_grad():
+            output = model(audio_tensor)
+
+            # output shape: [batch, 2]
+            # Index 0 = spoof logit, Index 1 = bonafide logit
+            probs = torch.softmax(output, dim=1)
+            prob_fake = probs[0, 0].item()
+
+        prediction = 1 if prob_fake >= input_data.threshold else 0
+        verdict = "fake" if prediction == 1 else "real"
+        inference_time = time.time() - start_time
+
+        logger.info(
+            f"Prediction: {verdict} (prob_fake={prob_fake:.4f}, "
+            f"time={inference_time:.3f}s)"
+        )
+
+        return {
+            "model": MODEL_NAME,
+            "probability": float(prob_fake),
+            "prediction": int(prediction),
+            "class": verdict,
+            "inference_time": float(inference_time),
+        }
+
+    except Exception as e:
+        logger.exception(f"Error during prediction: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("MODEL_PORT", 8004))
+    uvicorn.run(app, host="0.0.0.0", port=port)

audio/nes2net/model_scripts/wav2vec2_Nes2Net_X.py

@@ -0,0 +1,317 @@
+import math
+
+import fairseq
+import torch
+import torch.nn as nn
+
+___author__ = "Tianchi Liu"
+__email__ = "tianchi_liu@u.nus.edu"
+# modified from the model script from Hemlata Tak
+
+
+class SSLModel(nn.Module):
+    def __init__(self, device):
+        super(SSLModel, self).__init__()
+        cp_path = (
+            "/app/weights/xlsr2_300m.pt"  # Change the pre-trained XLSR model path.
+        )
+        model, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task(
+            [cp_path]
+        )
+        self.model = model[0]
+        self.device = device
+        self.out_dim = 1024
+        return
+
+    def extract_feat(self, input_data):
+        # put the model to GPU if it not there
+        if (
+            next(self.model.parameters()).device != input_data.device
+            or next(self.model.parameters()).dtype != input_data.dtype
+        ):
+            self.model.to(input_data.device, dtype=input_data.dtype)
+            self.model.train()
+        if True:
+            # input should be in shape (batch, length)
+            if input_data.ndim == 3:
+                input_tmp = input_data[:, :, 0]
+            else:
+                input_tmp = input_data
+            # [batch, length, dim]
+            emb = self.model(input_tmp, mask=False, features_only=True)["x"]
+        return emb
+
+
+class SEModule(nn.Module):
+    def __init__(self, channels, SE_ratio=8):
+        super(SEModule, self).__init__()
+        self.se = nn.Sequential(
+            nn.AdaptiveAvgPool1d(1),
+            nn.Conv1d(channels, channels // SE_ratio, kernel_size=1, padding=0),
+            nn.ReLU(),
+            nn.Conv1d(channels // SE_ratio, channels, kernel_size=1, padding=0),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, input):
+        x = self.se(input)
+        return input * x
+
+
+class Bottle2neck(nn.Module):
+
+    def __init__(
+        self, inplanes, planes, kernel_size=None, dilation=None, scale=8, SE_ratio=8
+    ):
+        super(Bottle2neck, self).__init__()
+        width = int(math.floor(planes / scale))
+        self.conv1 = nn.Conv1d(inplanes, width * scale, kernel_size=1)
+        self.bn1 = nn.BatchNorm1d(width * scale)
+        self.nums = scale - 1
+        convs = []
+        bns = []
+        weighted_sum = []
+        num_pad = math.floor(kernel_size / 2) * dilation
+        for i in range(self.nums):
+            convs.append(
+                nn.Conv2d(
+                    width,
+                    width,
+                    kernel_size=(kernel_size, 1),
+                    dilation=(dilation, 1),
+                    padding=(num_pad, 0),
+                )
+            )
+            bns.append(nn.BatchNorm2d(width))
+            initial_value = torch.ones(1, 1, 1, i + 2) * (1 / (i + 2))
+            weighted_sum.append(nn.Parameter(initial_value, requires_grad=True))
+        self.weighted_sum = nn.ParameterList(weighted_sum)
+        self.convs = nn.ModuleList(convs)
+        self.bns = nn.ModuleList(bns)
+        self.conv3 = nn.Conv1d(width * scale, planes, kernel_size=1)
+        self.bn3 = nn.BatchNorm1d(planes)
+        self.relu = nn.ReLU()
+        self.width = width
+        self.se = SEModule(planes, SE_ratio)
+
+    def forward(self, x):
+        residual = x
+        out = self.conv1(x)
+        out = self.relu(out)
+        out = self.bn1(out).unsqueeze(-1)  # bz c T 1
+
+        spx = torch.split(out, self.width, 1)
+        sp = spx[self.nums]
+        for i in range(self.nums):
+            sp = torch.cat((sp, spx[i]), -1)
+
+            sp = self.bns[i](self.relu(self.convs[i](sp)))
+            sp_s = sp * self.weighted_sum[i]
+            sp_s = torch.sum(sp_s, dim=-1, keepdim=False)
+
+            if i == 0:
+                out = sp_s
+            else:
+                out = torch.cat((out, sp_s), 1)
+        out = torch.cat((out, spx[self.nums].squeeze(-1)), 1)
+        out = self.conv3(out)
+        out = self.relu(out)
+        out = self.bn3(out)
+        out = self.se(out)
+        out += residual
+        return out
+
+
+class ASTP(nn.Module):
+    """Attentive statistics pooling: Channel- and context-dependent
+    statistics pooling, first used in ECAPA_TDNN.
+    """
+
+    def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False):
+        super(ASTP, self).__init__()
+        self.global_context_att = global_context_att
+
+        # Use Conv1d with stride == 1 rather than Linear, then we don't
+        # need to transpose inputs.
+        if global_context_att:
+            self.linear1 = nn.Conv1d(
+                in_dim * 3, bottleneck_dim, kernel_size=1
+            )  # equals W and b in the paper
+        else:
+            self.linear1 = nn.Conv1d(
+                in_dim, bottleneck_dim, kernel_size=1
+            )  # equals W and b in the paper
+        self.linear2 = nn.Conv1d(
+            bottleneck_dim, in_dim, kernel_size=1
+        )  # equals V and k in the paper
+
+    def forward(self, x):
+        """
+        x: a 3-dimensional tensor in tdnn-based architecture (B,F,T)
+            or a 4-dimensional tensor in resnet architecture (B,C,F,T)
+            0-dim: batch-dimension, last-dim: time-dimension (frame-dimension)
+        """
+        if len(x.shape) == 4:
+            x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3])
+        assert len(x.shape) == 3
+
+        if self.global_context_att:
+            context_mean = torch.mean(x, dim=-1, keepdim=True).expand_as(x)
+            context_std = torch.sqrt(
+                torch.var(x, dim=-1, keepdim=True) + 1e-10
+            ).expand_as(x)
+            x_in = torch.cat((x, context_mean, context_std), dim=1)
+        else:
+            x_in = x
+
+        # DON'T use ReLU here! ReLU may be hard to converge.
+        alpha = torch.tanh(self.linear1(x_in))  # alpha = F.relu(self.linear1(x_in))
+        alpha = torch.softmax(self.linear2(alpha), dim=2)
+        mean = torch.sum(alpha * x, dim=2)
+        var = torch.sum(alpha * (x**2), dim=2) - mean**2
+        std = torch.sqrt(var.clamp(min=1e-10))
+        return torch.cat([mean, std], dim=1)
+
+
+class Nested_Res2Net_TDNN(nn.Module):
+
+    def __init__(
+        self,
+        Nes_ratio=[8, 8],
+        input_channel=1024,
+        n_output_logits=2,
+        dilation=2,
+        pool_func="mean",
+        SE_ratio=[8],
+    ):
+
+        super(Nested_Res2Net_TDNN, self).__init__()
+        self.Nes_ratio = Nes_ratio[0]
+        assert input_channel % Nes_ratio[0] == 0
+        C = input_channel // Nes_ratio[0]
+        self.C = C
+        Build_in_Res2Nets = []
+        bns = []
+        for i in range(Nes_ratio[0] - 1):
+            Build_in_Res2Nets.append(
+                Bottle2neck(
+                    C,
+                    C,
+                    kernel_size=3,
+                    dilation=dilation,
+                    scale=Nes_ratio[1],
+                    SE_ratio=SE_ratio[0],
+                )
+            )
+            bns.append(nn.BatchNorm1d(C))
+        self.Build_in_Res2Nets = nn.ModuleList(Build_in_Res2Nets)
+        self.bns = nn.ModuleList(bns)
+        self.bn = nn.BatchNorm1d(1024)
+        self.relu = nn.ReLU()
+        self.pool_func = pool_func
+        if pool_func == "mean":
+            self.fc = nn.Linear(1024, n_output_logits)
+        elif pool_func == "ASTP":
+            self.pooling = ASTP(
+                in_dim=input_channel, bottleneck_dim=128, global_context_att=False
+            )
+            self.fc = nn.Linear(2048, n_output_logits)
+
+    def forward(self, x):
+        spx = torch.split(x, self.C, 1)
+        for i in range(self.Nes_ratio - 1):
+            if i == 0:
+                sp = spx[i]
+            else:
+                sp = sp + spx[i]
+            sp = self.Build_in_Res2Nets[i](sp)
+            sp = self.relu(sp)
+            sp = self.bns[i](sp)
+            if i == 0:
+                out = sp
+            else:
+                out = torch.cat((out, sp), 1)
+        out = torch.cat((out, spx[-1]), 1)
+        out = self.bn(out)
+        out = self.relu(out)
+        if self.pool_func == "mean":
+            out = torch.mean(out, dim=-1)
+        elif self.pool_func == "ASTP":
+            out = self.pooling(out)
+        out = self.fc(out)
+        return out
+
+
+class wav2vec2_Nes2Net_no_Res_w_allT(nn.Module):
+    def __init__(self, args, device):
+        super().__init__()
+        self.device = device
+
+        self.n_output_logits = args.n_output_logits
+
+        ####
+        # create network wav2vec 2.0
+        ####
+        self.ssl_model = SSLModel(self.device)
+        self.Nested_Res2Net_TDNN = Nested_Res2Net_TDNN(
+            Nes_ratio=args.Nes_ratio,
+            input_channel=1024,
+            n_output_logits=self.n_output_logits,
+            dilation=args.dilation,
+            pool_func=args.pool_func,
+            SE_ratio=args.SE_ratio,
+        )
+
+    def forward(self, x):
+        # -------pre-trained Wav2vec model fine tunning ------------------------##
+        x_ssl_feat = self.ssl_model.extract_feat(x.squeeze(-1))
+        x_ssl_feat = x_ssl_feat.permute(0, 2, 1)
+        output = self.Nested_Res2Net_TDNN(x_ssl_feat)
+
+        return output
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--n_output_logits", type=int, default=2)
+    parser.add_argument("--dilation", type=int, default=2)  # not important
+    parser.add_argument(
+        "--pool_func",
+        type=str,
+        default="mean",
+        choices=["mean", "ASTP"],
+        help="pooling function, choose from mean and ASTP",
+    )
+    parser.add_argument(
+        "--Nes_ratio",
+        type=int,
+        nargs="+",
+        default=[8, 8],
+        help="Nes_ratio, from outer to inner",
+    )
+    parser.add_argument(
+        "--SE_ratio",
+        type=int,
+        nargs="+",
+        default=[1],
+        help="SE downsampling ratio in the bottleneck",
+    )
+    args = parser.parse_args()
+
+    model = wav2vec2_Nes2Net_no_Res_w_allT(args=args, device="cpu")
+    x = torch.rand((4, 32000)).to("cpu")
+    model = model.to("cpu")
+    y = model(x)
+    print(y)
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print("all:", trainable_params)
+    trainable_params = sum(
+        p.numel() for p in model.ssl_model.parameters() if p.requires_grad
+    )
+    print("SSL:", trainable_params)
+    trainable_params = sum(
+        p.numel() for p in model.Nested_Res2Net_TDNN.parameters() if p.requires_grad
+    )
+    print("Backend:", trainable_params)

audio/nes2net/requirements.txt

@@ -0,0 +1,13 @@
+torch==2.5.0
+torchaudio==2.5.0
+numpy==1.23.5
+librosa==0.9.1
+soundfile
+scipy
+omegaconf
+hydra-core
+bitarray
+fastapi
+uvicorn[standard]
+pydantic
+python-multipart

audio/nes2net/weights/nes2net_itw_valaug.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:891de3f03cd846a5d6b69a193889440e26a5dde4028c7c4ff3bc80273987a024
+size 1272037256

audio/nes2net/weights/xlsr2_300m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b08927597f2c9eb2ebd7dcc3ac78ee4b5f6021cbac4b3a6c5a9deec445d80ed9
+size 3808868242

audio/safeear/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.9-slim
+
+RUN apt-get update && apt-get install -y \
+    git \
+    ffmpeg \
+    libsndfile1 \
+    wget \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Clone SafeEar repository (for model code imports)
+RUN git clone --depth 1 https://github.com/LetterLiGo/SafeEar.git /app/safeear_repo
+
+# Install the fairseq fork required by SafeEar's SpeechTokenizer
+WORKDIR /app/safeear_repo/fairseq_ours
+RUN pip install --no-cache-dir -e .
+WORKDIR /app
+
+# Download model weights from HuggingFace
+RUN mkdir -p /app/weights && \
+    wget -q -O /app/weights/SpeechTokenizer.pt \
+      "https://huggingface.co/TEC2004/SafeEar-ASV19-spoof-detection/resolve/main/SpeechTokenizer.pt" && \
+    wget -q -O /app/weights/model.ckpt \
+      "https://huggingface.co/TEC2004/SafeEar-ASV19-spoof-detection/resolve/main/model.ckpt"
+
+COPY api.py .
+
+EXPOSE 8002
+
+# Drop root privileges
+RUN adduser --disabled-password --gecos '' appuser
+USER appuser
+
+CMD ["python", "api.py"]

audio/safeear/api.py

@@ -0,0 +1,290 @@
+"""SafeEar audio deepfake detection API service.
+
+Uses the SafeEar content privacy-preserving model (CCS 2024) to detect
+synthetic speech. Two-stage pipeline:
+  1. SpeechTokenizer (neural audio codec) decouples acoustic features
+  2. SafeEar1s (transformer classifier) detects spoofing from acoustic tokens
+
+Weights: HuggingFace TEC2004/SafeEar-ASV19-spoof-detection
+"""
+
+import base64
+import logging
+import os
+import sys
+import tempfile
+import time
+from typing import Optional
+
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger("safeear_api")
+
+import platform
+
+import librosa
+import numpy as np
+import torch
+
+# Add SafeEar repo to path for model imports
+SAFEEAR_REPO_PATH = os.environ.get(
+    "SAFEEAR_REPO_PATH",
+    os.path.join(os.path.dirname(__file__), "safeear_repo"),
+)
+if SAFEEAR_REPO_PATH not in sys.path:
+    sys.path.insert(0, SAFEEAR_REPO_PATH)
+
+
+def _get_device():
+    """Select optimal device: MPS (Apple) > CUDA (NVIDIA) > CPU."""
+    override = os.environ.get("DEEPSAFE_DEVICE", "").strip().lower()
+    if override == "cpu":
+        return torch.device("cpu")
+    if override == "cuda" and torch.cuda.is_available():
+        return torch.device("cuda")
+    if (
+        override == "mps"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if (
+        platform.system() == "Darwin"
+        and hasattr(torch.backends, "mps")
+        and torch.backends.mps.is_available()
+    ):
+        return torch.device("mps")
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    return torch.device("cpu")
+
+
+# Constants
+MODEL_NAME = "safeear"
+WEIGHTS_DIR = os.environ.get(
+    "WEIGHTS_DIR",
+    os.path.join(os.path.dirname(__file__), "weights"),
+)
+DEVICE = _get_device()
+SAMPLE_RATE = 16000
+MAX_AUDIO_LENGTH = 64600  # ~4 seconds at 16kHz (ASVspoof standard)
+SOFTMAX_TEMPERATURE = 5.0  # Calibration temperature for out-of-distribution data
+NUM_INFERENCE_PASSES = 5  # Monte Carlo passes for stable predictions
+
+# Global model instances
+decouple_model = None
+detect_model = None
+
+
+class AudioInput(BaseModel):
+    """Schema for audio prediction requests."""
+
+    audio_data: str = Field(
+        ..., description="Base64 encoded audio string (WAV/MP3/etc)"
+    )
+    threshold: Optional[float] = Field(
+        0.5, ge=0.0, le=1.0, description="Classification threshold"
+    )
+
+
+app = FastAPI(
+    title="SafeEar Audio Deepfake Detection API",
+    description="Content privacy-preserving deepfake detection using SafeEar.",
+    version="1.0.0",
+)
+
+
+def load_models():
+    """Load both the decouple model (SpeechTokenizer) and detect model."""
+    global decouple_model, detect_model
+
+    if decouple_model is not None and detect_model is not None:
+        return True
+
+    speech_tokenizer_path = os.path.join(WEIGHTS_DIR, "SpeechTokenizer.pt")
+    checkpoint_path = os.path.join(WEIGHTS_DIR, "model.ckpt")
+
+    if not os.path.exists(speech_tokenizer_path):
+        logger.error(f"SpeechTokenizer weights not found: {speech_tokenizer_path}")
+        return False
+    if not os.path.exists(checkpoint_path):
+        logger.error(f"Model checkpoint not found: {checkpoint_path}")
+        return False
+
+    try:
+        # --- Load SpeechTokenizer (decouple model) ---
+        from safeear.models.decouple import SpeechTokenizer
+
+        logger.info("Loading SpeechTokenizer...")
+        decouple_model = SpeechTokenizer(
+            n_filters=64,
+            strides=[8, 5, 4, 2],
+            dimension=1024,
+            semantic_dimension=768,
+            bidirectional=True,
+            dilation_base=2,
+            residual_kernel_size=3,
+            n_residual_layers=1,
+            lstm_layers=2,
+            activation="ELU",
+            codebook_size=1024,
+            n_q=8,
+            sample_rate=16000,
+        )
+        st_state = torch.load(speech_tokenizer_path, map_location="cpu")
+        decouple_model.load_state_dict(st_state)
+        decouple_model.to(DEVICE)
+        decouple_model.eval()
+        logger.info("SpeechTokenizer loaded.")
+
+        # --- Load SafeEar1s (detect model) from Lightning checkpoint ---
+        from safeear.models.safeear import SafeEar1s, SE_Rawformer_front
+
+        logger.info("Loading SafeEar1s detect model...")
+        detect_model = SafeEar1s(
+            front=SE_Rawformer_front(),
+            embedding_dim=1024,
+            dropout_rate=0.1,
+            attention_dropout=0.1,
+            stochastic_depth=0.1,
+            num_layers=2,
+            num_heads=8,
+            num_classes=2,
+            positional_embedding="sine",
+            mlp_ratio=1.0,
+        )
+
+        # The .ckpt is a PyTorch Lightning checkpoint
+        ckpt = torch.load(checkpoint_path, map_location="cpu")
+        state_dict = ckpt.get("state_dict", ckpt)
+
+        # Lightning prefixes keys with "detect_model."
+        detect_state = {}
+        for k, v in state_dict.items():
+            if k.startswith("detect_model."):
+                detect_state[k.replace("detect_model.", "", 1)] = v
+
+        detect_model.load_state_dict(detect_state)
+        detect_model.to(DEVICE)
+        detect_model.eval()
+        logger.info("SafeEar1s detect model loaded.")
+        return True
+
+    except Exception as e:
+        logger.exception(f"Failed to load SafeEar models: {e}")
+        decouple_model = None
+        detect_model = None
+        return False
+
+
+def preprocess_audio(audio_bytes: bytes) -> torch.Tensor:
+    """Load audio bytes, resample to 16kHz mono, pad/trim."""
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+        tmp.write(audio_bytes)
+        tmp_path = tmp.name
+
+    try:
+        waveform, _ = librosa.load(tmp_path, sr=SAMPLE_RATE, mono=True)
+    finally:
+        os.unlink(tmp_path)
+
+    if len(waveform) < MAX_AUDIO_LENGTH:
+        waveform = np.pad(waveform, (0, MAX_AUDIO_LENGTH - len(waveform)))
+    else:
+        waveform = waveform[:MAX_AUDIO_LENGTH]
+
+    # Shape: (1, 1, samples) -- batch=1, channels=1, time
+    tensor = torch.FloatTensor(waveform).unsqueeze(0).unsqueeze(0).to(DEVICE)
+    return tensor
+
+
+@app.on_event("startup")
+async def startup_event():
+    """Attempt to load models at startup."""
+    load_models()
+
+
+@app.get("/health")
+async def health():
+    """Return service health status and model availability."""
+    models_loaded = decouple_model is not None and detect_model is not None
+    return {
+        "status": "healthy" if models_loaded else "degraded",
+        "model": MODEL_NAME,
+        "device": str(DEVICE),
+        "weights_found": (
+            os.path.exists(os.path.join(WEIGHTS_DIR, "SpeechTokenizer.pt"))
+            and os.path.exists(os.path.join(WEIGHTS_DIR, "model.ckpt"))
+        ),
+    }
+
+
+@app.post("/predict")
+async def predict(input_data: AudioInput):
+    """Run SafeEar inference on base64-encoded audio data."""
+    if decouple_model is None or detect_model is None:
+        if not load_models():
+            raise HTTPException(status_code=503, detail="Models not loaded")
+
+    try:
+        start_time = time.time()
+        logger.info(
+            "Received prediction request. "
+            f"Data size: {len(input_data.audio_data)} chars"
+        )
+
+        audio_bytes = base64.b64decode(input_data.audio_data)
+        x_wav = preprocess_audio(audio_bytes)
+
+        with torch.no_grad():
+            # Step 1: Extract acoustic tokens via SpeechTokenizer
+            # forward() returns:
+            #   (reconstructed, commit_loss, semantic_feature, acoustic_tokens)
+            # layers=[0,1,2,3,4,5,6,7] means layer 0 goes to
+            # semantic_feature; layers 1-7 go to acoustic_tokens list
+            _, _, _, acoustic_tokens = decouple_model(
+                x_wav, layers=[0, 1, 2, 3, 4, 5, 6, 7]
+            )
+
+            # Step 2: Run detection model with Monte Carlo averaging
+            # SafeEar1s uses torch.randperm() in forward, so we average
+            # multiple passes for stable predictions
+            logit_sum = torch.zeros(1, 2, device=DEVICE)
+            for _ in range(NUM_INFERENCE_PASSES):
+                raw_logits, _ = detect_model(acoustic_tokens)
+                logit_sum += raw_logits
+            avg_logits = logit_sum / NUM_INFERENCE_PASSES
+
+            # Step 3: Get fake probability with temperature-scaled softmax
+            # The model produces extreme logits that saturate standard
+            # softmax. Temperature scaling preserves discrimination while
+            # giving more interpretable probabilities.
+            probs = torch.softmax(avg_logits / SOFTMAX_TEMPERATURE, dim=-1)
+            prob_fake = probs[0, 1].item()
+
+        prediction = 1 if prob_fake >= input_data.threshold else 0
+        verdict = "fake" if prediction == 1 else "real"
+        inference_time = time.time() - start_time
+
+        return {
+            "model": MODEL_NAME,
+            "probability": float(prob_fake),
+            "prediction": int(prediction),
+            "class": verdict,
+            "inference_time": float(inference_time),
+        }
+
+    except Exception as e:
+        logger.exception(f"Error during prediction: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("MODEL_PORT", 8002))
+    uvicorn.run(app, host="0.0.0.0", port=port)

audio/safeear/download_weights.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+WEIGHTS_DIR="${WEIGHTS_DIR:-/app/weights}"
+REPO_DIR="${REPO_DIR:-/app/safeear_repo}"
+
+mkdir -p "$WEIGHTS_DIR"
+
+echo "==> Cloning SafeEar source repository..."
+if [ ! -d "$REPO_DIR/.git" ]; then
+    git clone --depth 1 https://github.com/LetterLiGo/SafeEar.git "$REPO_DIR"
+fi
+
+echo "==> Downloading SpeechTokenizer.pt from HuggingFace..."
+wget -q --show-progress -O "$WEIGHTS_DIR/SpeechTokenizer.pt" \
+  "https://huggingface.co/TEC2004/SafeEar-ASV19-spoof-detection/resolve/main/SpeechTokenizer.pt"
+
+echo "==> Downloading model.ckpt from HuggingFace..."
+wget -q --show-progress -O "$WEIGHTS_DIR/model.ckpt" \
+  "https://huggingface.co/TEC2004/SafeEar-ASV19-spoof-detection/resolve/main/model.ckpt"
+
+echo "==> Weights downloaded to $WEIGHTS_DIR"
+ls -lh "$WEIGHTS_DIR"

audio/safeear/requirements.txt

@@ -0,0 +1,15 @@
+torch>=1.13.0
+torchaudio>=0.13.0
+librosa>=0.10.0
+soundfile>=0.11.0
+numpy>=1.23.0
+einops>=0.7.0
+timm>=0.9.0
+hydra-core>=1.0.7
+omegaconf>=2.1.0
+pytorch-lightning>=1.6.0
+scipy>=1.11.0
+fastapi>=0.100.0
+uvicorn[standard]>=0.20.0
+python-multipart>=0.0.5
+pydantic>=2.0.0

audio/safeear/safeear_repo/.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+model_zoos/*
+Exps/*
+datas/datasets
+datas/ASVSpoof2019/LA
+datas/ASVSpoof2021/ASVspoof2021_LA_eval
+datas/ASVSpoof2021/keys
+create_tsv.py

audio/safeear/safeear_repo/LICENSE

@@ -0,0 +1,23 @@
+# Creative Commons Attribution 4.0 International License
+
+## License
+
+You are free to:
+
+- Share — copy and redistribute the material in any medium or format
+- Adapt — remix, transform, and build upon the material for any purpose, even commercially.
+
+Under the following terms:
+
+1. **Attribution** — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+
+2. **No additional restrictions** — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+
+## Other Terms
+
+- This license applies to all types of works, including but not limited to text, images, audio, video, etc.
+- This license does not apply to any third-party materials included in the work, for which you must obtain permission separately.
+
+## Disclaimer
+
+This work is provided on an "as is" basis, without any warranties or conditions of any kind, either express or implied, including but not limited to implied warranties of merchantability, fitness for a particular purpose, or non-infringement.

audio/safeear/safeear_repo/README.md

@@ -0,0 +1,134 @@
+# <font color=E7595C>Safe</font><font color=F6C446>Ear</font><img src="assert/SafeEar_logo.jpg" alt="icon" style="width: 2em; height: 1.5em; vertical-align: middle;">: <font color=E7595C>Content Privacy-Preserving</font> <font color=F6C446>Audio Deepfake Detection</font>
+
+[![arXiv](https://img.shields.io/badge/arXiv-2409.09272-b31b1b.svg)](https://arxiv.org/abs/2409.09272)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://makeapullrequest.com) 
+[![CC BY 4.0](https://img.shields.io/badge/license-CC%20BY%204.0-blue.svg)](https://creativecommons.org/licenses/by/4.0/)
+![GitHub stars](https://img.shields.io/github/stars/LetterLiGo/SafeEar)
+![GitHub forks](https://img.shields.io/github/forks/LetterLiGo/SafeEar)
+![Website](https://img.shields.io/website?url=https://safeearweb.github.io/Project/)
+
+
+By [1] Zhejiang University, [2] Tsinghua University.
+* [Xinfeng Li](https://letterligo.github.io)* [1], [Kai Li](https://cslikai.cn)* [2], Yifan Zheng [1], Chen Yan† [1], Xiaoyu Ji [1], Wenyuan Xu [1].
+
+This repository is an official implementation of the SafeEar accepted to **ACM CCS 2024** (Core-A*, CCF-A, Big4) .
+
+Please also visit our <a href="https://safeearweb.github.io/Project/">(1) Project Website</a>, <a href="https://zenodo.org/records/14062964">(2) Full CVoiceFake Dataset</a>, and <a href="https://zenodo.org/records/11124319">(3) Sampled CVoiceFake Dataset</a>.
+
+## 🔥News
+
+[2025-03-18]: Supported the batch testing for ASVspoof 2019 and 2021, fixed some bugs for datasets and trainer.
+
+[2024-12-10]: Fixed all the bugs for training and test, and uploaded the files for data generation `datas/`.
+
+[2024-12-01]: Uploaded the checkpoint for data generation `datas/`.
+
+## ✨Key Highlights:
+
+In this paper, we propose SafeEar, a novel framework that aims to detect deepfake audios without relying on accessing the speech content within. Our key idea is to devise a neural audio codec into a novel decoupling model that well separates the semantic and acoustic information from audio samples, and only use the acoustic information (e.g., prosody and timbre) for deepfake detection. In this way, no semantic content will be exposed to the detector. To overcome the challenge of identifying diverse deepfake audio without semantic clues, we enhance our deepfake detector with multi-head self-attention and codec augmentation. Extensive experiments conducted on four benchmark datasets demonstrate SafeEar’s effectiveness in detecting various deepfake techniques with an equal error rate (EER) down to 2.02%. Simultaneously, it shields five-language speech content from being deciphered by both machine and human auditory analysis, demonstrated by word error rates (WERs) all above 93.93% and our user study. Furthermore, our benchmark constructed for anti-deepfake and anti-content recovery evaluation helps provide a basis for future research in the realms of audio privacy preservation and deepfake detection.
+
+## 🚀Overall Pipeline
+
+![pipeline](assert/overall.gif)
+
+## 🔧Installation
+
+1. Clone the repository:
+
+```shell
+git clone git@github.com:LetterLiGo/SafeEar.git
+cd SafeEar/
+```
+
+2. Create and activate the conda environment:
+
+```shell
+conda create -n safeear python=3.9 
+conda activate safeear
+```
+
+3. Install PyTorch and torchvision following the [official instructions](https://pytorch.org). The code requires `python=3.9`, `pytorch=1.13`, `torchvision=0.14`.
+
+
+```shell
+pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+
+```
+4. Install other dependencies:
+
+```shell 
+pip install pip==24.0
+pip install -r requirements.txt
+```
+
+## 📊Model Performance
+### ASVspoof 2019 & 2021
+![](assert/ASVSpoof-results.png)
+### Speech Recognition Performance
+![](assert/exp1.png)
+
+## Data preparation
+
+### AVSpoof 2019 & 2021
+
+Please download the [ASVspoof 2019](https://datashare.is.ed.ac.uk/handle/10283/3336) and [ASVspoof 2021](https://www.asvspoof.org/index2021.html) datasets and extract them to the `datas/datasets` directory.
+
+```shell
+datas/datasets/ASVspoof2019
+datas/datasets/ASVspoof2021
+```
+
+#### Generate the Hubert L9 feature files
+
+```shell
+mkdir model_zoos
+cd model_zoos
+wget https://dl.fbaipublicfiles.com/hubert/hubert_base_ls960.pt
+wget https://cloud.tsinghua.edu.cn/f/413a0cd2e6f749eea956/?dl=1 -O SpeechTokenizer.pt
+cd ../datas
+# Generate the Hubert L9 feature files for ASVspoof 2019
+python dump_hubert_avg_feature.py datasets/ASVSpoof2019 datasets/ASVSpoof2019_Hubert_L9
+# Generate the Hubert L9 feature files for ASVspoof 2021
+python dump_hubert_avg_feature.py datasets/ASVSpoof2021 datasets/ASVSpoof2021_Hubert_L9
+```
+
+## 📚Training
+
+Before starting training, please modify the parameter configurations in [`configs`](configs).
+
+Use the following commands to start training:
+
+```shell
+python train.py --conf_dir config/train19.yaml
+python train.py --conf_dir config/train21.yaml
+```
+
+## 📈Testing/Inference
+
+To evaluate a model on one or more GPUs, specify the `CUDA_VISIBLE_DEVICES`, `dataset`, `model` and `checkpoint`:
+
+```shell
+python test.py --conf_dir Exps/ASVspoof19/config.yaml
+python test.py --conf_dir Exps/ASVspoof21/config.yaml
+```
+
+## Bugs and Issues
+
+If you meet `RuntimeError: Failed to load audio from <_io.BytesIO object at 0x7f45cb978f90>`, please use the following command to fix it:
+
+```shell
+conda install -c anaconda 'ffmpeg<4.4'
+```
+
+## 📜Citation
+
+If you find our work/code/dataset helpful, please consider citing:
+
+```
+@inproceedings{li2024safeear,
+  author       = {Li, Xinfeng and Li, Kai and Zheng, Yifan and Yan, Chen and Ji, Xiaoyu, and Xu, Wenyuan},
+  title        = {{SafeEar: Content Privacy-Preserving Audio Deepfake Detection}},
+  booktitle    = {Proceedings of the 2024 {ACM} {SIGSAC} Conference on Computer and Communications Security (CCS)}
+  year         = {2024},
+}
+```

audio/safeear/safeear_repo/assert/overall.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d34cf40c912995aeb9070d168bc41663e365b5e64544538e8cb7e36770233afc
+size 2170657

audio/safeear/safeear_repo/config/train19.yaml

@@ -0,0 +1,87 @@
+datamodule:
+  _target_: safeear.datas.asvspoof19.DataModule
+  batch_size: 2
+  num_workers: 8
+  pin_memory: true
+  DataClass_dict:
+    _target_: safeear.datas.asvspoof19.DataClass
+    train_path: ["datas/ASVSpoof2019/train.tsv", "datas/ASVSpoof2019/ASVspoof2019.LA.cm.train.trn.txt", "datas/datasets/ASVSpoof2019_Hubert_L9/ASVspoof2019_LA_train/flac"]
+    val_path: ["datas/ASVSpoof2019/dev.tsv", "datas/ASVSpoof2019/ASVspoof2019.LA.cm.dev.trl.txt", "datas/datasets/ASVSpoof2019_Hubert_L9/ASVspoof2019_LA_dev/flac"]
+    test_path: ["datas/ASVSpoof2019/eval.tsv", "datas/ASVSpoof2019/ASVspoof2019.LA.cm.eval.trl.txt", "datas/datasets/ASVSpoof2019_Hubert_L9/ASVspoof2019_LA_eval/flac"]
+    max_len: 64600
+
+decouple_model:
+  _target_: safeear.models.decouple.SpeechTokenizer
+  n_filters: 64
+  strides: [8,5,4,2]
+  dimension: 1024
+  semantic_dimension: 768
+  bidirectional: true
+  dilation_base: 2
+  residual_kernel_size: 3
+  n_residual_layers: 1
+  lstm_layers: 2
+  activation: ELU
+  codebook_size: 1024
+  n_q: 8
+  sample_rate: 16000
+
+speechtokenizer_path: model_zoos/SpeechTokenizer.pt
+
+detect_model:
+  _target_: safeear.models.safeear.SafeEar1s
+  front:
+    _target_: safeear.models.safeear.SE_Rawformer_front
+  embedding_dim: 1024 
+  dropout_rate: 0.1 
+  attention_dropout: 0.1 
+  stochastic_depth: 0.1 
+  num_layers: 2
+  num_heads: 8 
+  num_classes: 2 
+  positional_embedding: 'sine' 
+  mlp_ratio: 1.0
+
+system:
+  _target_: safeear.trainer.safeear_trainer.SafeEarTrainer
+  lr_raw_former: 3.0e-4
+  save_score_path: ${exp.dir}/${exp.name}
+
+exp: 
+  dir: Exps/ # 修改
+  name: ASVspoof19 # 修改
+
+early_stopping:
+  _target_: pytorch_lightning.callbacks.EarlyStopping
+  monitor: val_eer # 修改
+  mode: min
+  patience: 40
+  verbose: true
+
+checkpoint:
+  _target_: pytorch_lightning.callbacks.ModelCheckpoint
+  dirpath: ${exp.dir}/${exp.name}/checkpoints
+  monitor: val_eer # 修改
+  mode: min
+  verbose: true
+  save_top_k: 1
+  save_last: true
+  filename: '{epoch}-{val_eer:.4f}' # 修改
+
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  name: ${exp.name}
+  save_dir: ${exp.dir}/${exp.name}/logs
+  offline: true
+  project: SafeEar
+
+trainer:
+  _target_: pytorch_lightning.Trainer
+  devices: [0]
+  max_epochs: 500
+  sync_batchnorm: true
+  default_root_dir: ${exp.dir}/${exp.name}/
+  accelerator: gpu
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  fast_dev_run: false

audio/safeear/safeear_repo/config/train21.yaml

@@ -0,0 +1,87 @@
+datamodule:
+  _target_: safeear.datas.asvspoof21.DataModule
+  batch_size: 2
+  num_workers: 8
+  pin_memory: true
+  DataClass_dict:
+    _target_: safeear.datas.asvspoof21.DataClass
+    train_path: ["datas/ASVSpoof2019/train.tsv", "datas/ASVSpoof2019/ASVspoof2019.LA.cm.train.trn.txt", "datas/datasets/ASVSpoof2019_Hubert_L9/ASVspoof2019_LA_train/flac"]
+    val_path: ["datas/ASVSpoof2019/dev.tsv", "datas/ASVSpoof2019/ASVspoof2019.LA.cm.dev.trl.txt", "datas/datasets/ASVSpoof2019_Hubert_L9/ASVspoof2019_LA_dev/flac"]
+    test_path: ["datas/ASVSpoof2021/eval.tsv", "datas/ASVSpoof2021/ASVspoof2021.LA.cm.eval.trl.txt", "datas/datasets/ASVSpoof2021_Hubert_L9"]
+    max_len: 64600
+
+decouple_model:
+  _target_: safeear.models.decouple.SpeechTokenizer
+  n_filters: 64
+  strides: [8,5,4,2]
+  dimension: 1024
+  semantic_dimension: 768
+  bidirectional: true
+  dilation_base: 2
+  residual_kernel_size: 3
+  n_residual_layers: 1
+  lstm_layers: 2
+  activation: ELU
+  codebook_size: 1024
+  n_q: 8
+  sample_rate: 16000
+
+speechtokenizer_path: model_zoos/SpeechTokenizer.pt
+
+detect_model:
+  _target_: safeear.models.safeear.SafeEar1s
+  front:
+    _target_: safeear.models.safeear.SE_Rawformer_front
+  embedding_dim: 1024 
+  dropout_rate: 0.1 
+  attention_dropout: 0.1 
+  stochastic_depth: 0.1 
+  num_layers: 2
+  num_heads: 8 
+  num_classes: 2 
+  positional_embedding: 'sine' 
+  mlp_ratio: 1.0
+
+system:
+  _target_: safeear.trainer.safeear_trainer.SafeEarTrainer
+  lr_raw_former: 3.0e-4
+  save_score_path: ${exp.dir}/${exp.name}
+
+exp: 
+  dir: Exps/ # 修改
+  name: ASVspoof21 # 修改
+
+early_stopping:
+  _target_: pytorch_lightning.callbacks.EarlyStopping
+  monitor: val_eer # 修改
+  mode: min
+  patience: 40
+  verbose: true
+
+checkpoint:
+  _target_: pytorch_lightning.callbacks.ModelCheckpoint
+  dirpath: ${exp.dir}/${exp.name}/checkpoints
+  monitor: val_eer # 修改
+  mode: min
+  verbose: true
+  save_top_k: 1
+  save_last: true
+  filename: '{epoch}-{val_eer:.4f}' # 修改
+
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  name: ${exp.name}
+  save_dir: ${exp.dir}/${exp.name}/logs
+  offline: true
+  project: SafeEar
+
+trainer:
+  _target_: pytorch_lightning.Trainer
+  devices: [0]
+  max_epochs: 40
+  sync_batchnorm: true
+  default_root_dir: ${exp.dir}/${exp.name}/
+  accelerator: gpu
+  limit_train_batches: 1.0
+  limit_val_batches: 1.0
+  fast_dev_run: false

audio/safeear/safeear_repo/datas/dump_hubert_avg_feature.py

@@ -0,0 +1,109 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import os
+import sys
+import warnings
+
+warnings.filterwarnings('ignore')
+from pathlib import Path
+
+import fairseq
+import librosa
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+import tqdm
+from feature_utils import dump_feature, get_path_iterator
+from npy_append_array import NpyAppendArray
+
+logging.basicConfig(
+    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    level=os.environ.get("LOGLEVEL", "INFO").upper(),
+    stream=sys.stdout,
+)
+logger = logging.getLogger("dump_hubert_feature")
+
+
+class HubertFeatureReader(object):
+    def __init__(self, ckpt_path, layer, max_chunk=1600000):
+        (
+            model,
+            cfg,
+            task,
+        ) = fairseq.checkpoint_utils.load_model_ensemble_and_task([ckpt_path])
+        self.model = model[0].eval().cuda()
+        self.task = task
+        self.layer = layer
+        self.max_chunk = max_chunk
+        logger.info(f"TASK CONFIG:\n{self.task.cfg}")
+        logger.info(f" max_chunk = {self.max_chunk}")
+
+    def read_audio(self, path, ref_len=None):
+        wav, sr = librosa.load(path, sr=None)
+        assert sr == self.task.cfg.sample_rate, sr
+        if wav.ndim == 2:
+            wav = wav.mean(-1)
+        assert wav.ndim == 1, wav.ndim
+        if ref_len is not None and abs(ref_len - len(wav)) > 160:
+            logging.warning(f"ref {ref_len} != read {len(wav)} ({path})")
+        return wav
+
+    def get_feats(self, path, ref_len=None):
+        x = self.read_audio(path, ref_len)
+        with torch.no_grad():
+            x = torch.from_numpy(x).float().cuda()
+            if self.task.cfg.normalize:
+                x = F.layer_norm(x, x.shape)
+            x = x.view(1, -1)
+
+            avg_feat = []
+            for start in range(0, x.size(1), self.max_chunk):
+                x_chunk = x[:, start: start + self.max_chunk]
+                feat_chunk, _, avg_feat_chunk = self.model.extract_features(
+                    source=x_chunk,
+                    padding_mask=None,
+                    mask=False,
+                    output_layer=self.layer,
+                )
+                avg_feat.append(avg_feat_chunk)
+        return torch.cat(avg_feat, 1).squeeze(0)
+
+def dump_feature(reader,audio_dir,save_dir):
+    save_dir = Path(save_dir)
+    audio_dir = Path(audio_dir)
+    
+    audio_files = list(audio_dir.glob("**/*.flac"))
+    for audio_file in tqdm.tqdm(audio_files):
+        releative_path = audio_file.relative_to(audio_dir).with_suffix(".npy")
+        save_path = save_dir / releative_path
+        if not save_path.parent.exists():
+            save_path.parent.mkdir(parents=True)
+        
+        feat_f = NpyAppendArray(save_path)
+        feat = reader.get_feats(audio_file)
+        feat_f.append(feat.cpu().numpy())
+    logger.info("finished successfully")
+
+def main(audio_dir, save_dir, ckpt_path, layer, max_chunk):
+    reader = HubertFeatureReader(ckpt_path, layer, max_chunk)
+    dump_feature(reader, audio_dir, save_dir)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("audio_dir", nargs="?", default="datasets/ASVSpoof2019", help="Directory containing audio files")
+    parser.add_argument("save_dir", nargs="?", default="datasets/ASVSpoof2019_Hubert_L9", help="Directory to save extracted features")
+    parser.add_argument("ckpt_path", nargs="?", default="../model_zoos/hubert_base_ls960.pt", help="Path to the checkpoint file")
+    parser.add_argument("layer", nargs="?", type=int, default=9, help="Layer number to extract features from")
+    parser.add_argument("--max_chunk", type=int, default=1600000, help="Maximum chunk size for processing")
+    args = parser.parse_args()
+    logger.info(args)
+
+    main(**vars(args))