Add condition-embedding replay comparison

README.md

@@ -340,6 +340,34 @@ Interpretation:
 
 Machine-readable comparison: [`comparison-metrics.json`](evaluation/v3/comparison-metrics.json).
 
+## Condition-Embedding Replay Comparison
+
+Protocol:
+
+- 130 exact-alignment holdout tracks;
+- each final checkpoint predicts argmax RVQ codes from cached DAV latents;
+- predicted codes are teacher-forced through the official language model and RVQ depth decoder;
+- hidden states pass through the official condition encoder with recorded chunk stitching;
+- reconstructed condition embeddings are compared with stored condition embeddings;
+- metric: per-track mean cosine over stitched condition-latent frames;
+- true sampled codes provide the replay control.
+
+| Model | Parameters | Mean cosine | Standard deviation | 5th-95th percentile |
+|---|---:|---:|---:|---:|
+| Serveurperso v1 | 40,978,944 | 0.663329 | 0.022175 | 0.628052-0.696328 |
+| SimpleTuner v1 | 40,978,944 | 0.762442 | 0.019550 | 0.734519-0.790450 |
+| SimpleTuner v2 | 154,736,064 | 0.769841 | 0.019063 | 0.742991-0.798636 |
+| SimpleTuner v3 | 154,736,064 | **0.770259** | 0.019274 | 0.741585-0.800492 |
+| True-code control | - | 0.999907 | - | - |
+
+![Condition-embedding replay comparison](evaluation/replay-comparison/condition-embedding-replay-comparison.png)
+
+V1 exceeds the independent Serveurperso checkpoint by 0.099114 mean cosine. V2 adds 0.007399 over v1. V3 adds 0.000418 over v2. The MERT gain remains small downstream.
+
+This test stops before diffusion and DAV decode. It is not an STFT, waveform, lyric-identity, or listening score.
+
+Data: [`combined-aggregate.json`](evaluation/replay-comparison/combined-aggregate.json), [`provenance.json`](evaluation/replay-comparison/provenance.json), and [`raw per-record metrics`](evaluation/replay-comparison/raw-metrics-simpletuner-v3.json).
+
 ## Limitations
 
 - 5.12-second context.
@@ -350,7 +378,7 @@ Machine-readable comparison: [`comparison-metrics.json`](evaluation/v3/compariso
 - Exact-token accuracy understates perceptual equivalence.
 - Semantic CE can dominate early because its vocabulary is larger.
 - Acoustic heads are independent despite residual-codebook dependence.
-- End-to-end condition-embedding replay evaluation is pending.
+- Diffusion render and audio-domain evaluation remain pending.
 - No stable packaged loading API.
 - Use is subject to MiniMax Music 3, dataset, and MERT terms.
 

evaluation/replay-comparison/combined-aggregate.json

@@ -0,0 +1,183 @@
+{
+  "dataset": {
+    "exact_alignment_records": 130,
+    "repo_id": "bghira/minimax-music3-rvq-reverse-distillation",
+    "revision": "5029b1e7f1bbfbf028b76b38564fecccda94a111",
+    "split": "holdout"
+  },
+  "format": "simpletuner-minimaxmusic-replay-comparison-v1",
+  "metric": "per-record mean stitched condition-embedding cosine",
+  "models": [
+    {
+      "checkpoint": {
+        "checkpoint_bytes": 163955786,
+        "checkpoint_sha256": "fd7bfefb197cb7b65a6b6d9b1373c1772a15979f1a4b67d820764d610f0fb9a5",
+        "checkpoint_url": "https://www.serveurperso.com/ia/ssd/workspace/git/minimaxmusic.cpp/training/checkpoints/v1/best.pt",
+        "epoch": 17,
+        "reported_validation_loss": 5.292583839459852,
+        "source_commit": "d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1",
+        "source_url": "https://github.com/ServeurpersoCom/minimaxmusic.cpp/commit/d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1"
+      },
+      "model_id": "serveurperso-v1",
+      "parameter_count": 40978944,
+      "predicted_code_condition_cosine": {
+        "count": 130,
+        "max": 0.7290002703666687,
+        "mean": 0.6633286081827604,
+        "min": 0.6026018261909485,
+        "quantiles": {
+          "0.0": 0.6026018261909485,
+          "0.05": 0.6280524551868438,
+          "0.25": 0.6483299732208252,
+          "0.5": 0.6645751297473907,
+          "0.75": 0.6784153282642365,
+          "0.95": 0.6963281780481339,
+          "1.0": 0.7290002703666687
+        },
+        "std": 0.02217457298933777
+      },
+      "true_code_control": {
+        "count": 130,
+        "max": 0.999957263469696,
+        "mean": 0.9999072317893688,
+        "min": 0.9983662366867065,
+        "quantiles": {
+          "0.0": 0.9983662366867065,
+          "0.05": 0.999821075797081,
+          "0.25": 0.9999240636825562,
+          "0.5": 0.9999381899833679,
+          "0.75": 0.9999445825815201,
+          "0.95": 0.9999500513076782,
+          "1.0": 0.999957263469696
+        },
+        "std": 0.00015022638681815362
+      }
+    },
+    {
+      "checkpoint": {
+        "checkpoint": "full/output/final",
+        "checkpoint_bytes": 163931632,
+        "checkpoint_sha256": "1cea88dddf7a48583c13ac661a548b7a3535b933a76356bb158420d24da32f22",
+        "global_step": 17640
+      },
+      "model_id": "simpletuner-v1",
+      "parameter_count": 40978944,
+      "predicted_code_condition_cosine": {
+        "count": 130,
+        "max": 0.8286042809486389,
+        "mean": 0.7624421353523548,
+        "min": 0.7123062014579773,
+        "quantiles": {
+          "0.0": 0.7123062014579773,
+          "0.05": 0.7345191180706024,
+          "0.25": 0.747437372803688,
+          "0.5": 0.7631425857543945,
+          "0.75": 0.7765287756919861,
+          "0.95": 0.7904498308897019,
+          "1.0": 0.8286042809486389
+        },
+        "std": 0.019549602827450108
+      },
+      "true_code_control": {
+        "count": 130,
+        "max": 0.999957263469696,
+        "mean": 0.9999072317893688,
+        "min": 0.9983662366867065,
+        "quantiles": {
+          "0.0": 0.9983662366867065,
+          "0.05": 0.999821075797081,
+          "0.25": 0.9999240636825562,
+          "0.5": 0.9999381899833679,
+          "0.75": 0.9999445825815201,
+          "0.95": 0.9999500513076782,
+          "1.0": 0.999957263469696
+        },
+        "std": 0.00015022638681815362
+      }
+    },
+    {
+      "checkpoint": {
+        "checkpoint": "v2/output/final",
+        "checkpoint_bytes": 618960456,
+        "checkpoint_sha256": "47dfffb7a76d95589752df0c59ff6eaf3496f6fcf5408d324218887e471023c7",
+        "global_step": 17660
+      },
+      "model_id": "simpletuner-v2",
+      "parameter_count": 154736064,
+      "predicted_code_condition_cosine": {
+        "count": 130,
+        "max": 0.8361330032348633,
+        "mean": 0.7698414775041433,
+        "min": 0.7206224203109741,
+        "quantiles": {
+          "0.0": 0.7206224203109741,
+          "0.05": 0.7429912149906158,
+          "0.25": 0.7548101097345352,
+          "0.5": 0.770626574754715,
+          "0.75": 0.7835157215595245,
+          "0.95": 0.7986360996961593,
+          "1.0": 0.8361330032348633
+        },
+        "std": 0.019062989675575924
+      },
+      "true_code_control": {
+        "count": 130,
+        "max": 0.999957263469696,
+        "mean": 0.9999072317893688,
+        "min": 0.9983662366867065,
+        "quantiles": {
+          "0.0": 0.9983662366867065,
+          "0.05": 0.999821075797081,
+          "0.25": 0.9999240636825562,
+          "0.5": 0.9999381899833679,
+          "0.75": 0.9999445825815201,
+          "0.95": 0.9999500513076782,
+          "1.0": 0.999957263469696
+        },
+        "std": 0.00015022638681815362
+      }
+    },
+    {
+      "checkpoint": {
+        "checkpoint": "v3/output/final",
+        "checkpoint_bytes": 618960456,
+        "checkpoint_sha256": "356e97fea65c486a28814dc778b52645807ab8f784c6021d2403f688a2bfe788",
+        "global_step": 17660
+      },
+      "model_id": "simpletuner-v3",
+      "parameter_count": 154736064,
+      "predicted_code_condition_cosine": {
+        "count": 130,
+        "max": 0.8356426954269409,
+        "mean": 0.7702590204202212,
+        "min": 0.7236603498458862,
+        "quantiles": {
+          "0.0": 0.7236603498458862,
+          "0.05": 0.7415853887796402,
+          "0.25": 0.7547859847545624,
+          "0.5": 0.7704743146896362,
+          "0.75": 0.7837727516889572,
+          "0.95": 0.8004916191101075,
+          "1.0": 0.8356426954269409
+        },
+        "std": 0.019273954474224628
+      },
+      "true_code_control": {
+        "count": 130,
+        "max": 0.999957263469696,
+        "mean": 0.9999072317893688,
+        "min": 0.9983662366867065,
+        "quantiles": {
+          "0.0": 0.9983662366867065,
+          "0.05": 0.999821075797081,
+          "0.25": 0.9999240636825562,
+          "0.5": 0.9999381899833679,
+          "0.75": 0.9999445825815201,
+          "0.95": 0.9999500513076782,
+          "1.0": 0.999957263469696
+        },
+        "std": 0.00015022638681815362
+      }
+    }
+  ]
+}

evaluation/replay-comparison/evaluate_replay.py

@@ -0,0 +1,571 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import importlib.util
+import json
+import math
+import os
+import statistics
+import tempfile
+import time
+import zipfile
+from pathlib import Path
+from typing import Any
+
+import torch
+import torch.nn.functional as F
+from safetensors import safe_open
+from transformers import Qwen2Tokenizer, Qwen3ForCausalLM
+
+from scripts.train_minimax_music_rvq_encoder import (
+    EvaluationCheckpoint,
+    _cache_paths,
+    _read_zip_safetensors,
+    _resolve_shard_path,
+    build_parser,
+    build_pool_matrix,
+    frame_latent_starts,
+    load_evaluation_model,
+    load_records,
+    load_trainer_state,
+)
+from simpletuner.helpers.models.minimaxmusic.condition_encoder import MiniMaxMusic3ConditionEncoder
+from simpletuner.helpers.models.minimaxmusic.encoders import _clean_caption, _normalize_lyrics
+from simpletuner.helpers.models.minimaxmusic.rvq_depth_decoder import MiniMaxMusic3RVQDepthDecoder
+
+
+CHECKPOINT_URL = (
+    "https://www.serveurperso.com/ia/ssd/workspace/git/minimaxmusic.cpp/"
+    "training/checkpoints/v1/best.pt"
+)
+CHECKPOINT_SHA256 = "fd7bfefb197cb7b65a6b6d9b1373c1772a15979f1a4b67d820764d610f0fb9a5"
+SOURCE_COMMIT = "d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1"
+SOURCE_URL = f"https://github.com/ServeurpersoCom/minimaxmusic.cpp/commit/{SOURCE_COMMIT}"
+OFFICIAL_MODEL_REVISION = "fbdf52fbaaca799592917417eb05f1899f1255ec"
+DATASET_REVISION = "5029b1e7f1bbfbf028b76b38564fecccda94a111"
+WINDOW_FRAMES = 128
+SEMANTIC_OFFSET = 151675
+LM_BLOCK_FRAMES = 256
+MODEL_IDS = ("serveurperso-v1", "simpletuner-v1", "simpletuner-v2", "simpletuner-v3")
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def atomic_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    encoded = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
+    fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(encoded)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+        directory_fd = os.open(path.parent, os.O_RDONLY)
+        try:
+            os.fsync(directory_fd)
+        finally:
+            os.close(directory_fd)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def append_jsonl(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
+    try:
+        written = os.write(fd, payload)
+        if written != len(payload):
+            raise OSError(f"Short JSONL append: {written} of {len(payload)} bytes.")
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    rows: list[dict[str, Any]] = []
+    valid_bytes = 0
+    with path.open("rb") as handle:
+        lines = handle.readlines()
+    for index, line in enumerate(lines):
+        try:
+            rows.append(json.loads(line))
+            valid_bytes += len(line)
+        except json.JSONDecodeError:
+            if index != len(lines) - 1:
+                raise
+            with path.open("r+b") as handle:
+                handle.truncate(valid_bytes)
+                handle.flush()
+                os.fsync(handle.fileno())
+    ids = [row["sample_id"] for row in rows]
+    if len(ids) != len(set(ids)):
+        raise ValueError(f"Duplicate sample IDs in {path}.")
+    return rows
+
+
+def load_external_encoder(checkpoint: Path, source: Path, device: torch.device):
+    if sha256_file(checkpoint) != CHECKPOINT_SHA256:
+        raise ValueError("Serveurperso checkpoint SHA-256 mismatch.")
+    spec = importlib.util.spec_from_file_location("serveurperso_rvq_v1", source)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load pinned source {source}.")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    model = module.RVQEncoder().to(device).eval()
+    state = torch.load(checkpoint, map_location="cpu", weights_only=True)
+    model.load_state_dict(state["model"])
+    provenance = {
+        "checkpoint_url": CHECKPOINT_URL,
+        "checkpoint_sha256": CHECKPOINT_SHA256,
+        "checkpoint_bytes": checkpoint.stat().st_size,
+        "source_commit": SOURCE_COMMIT,
+        "source_url": SOURCE_URL,
+        "epoch": state["epoch"],
+        "reported_validation_loss": state["val_loss"],
+    }
+    return model, provenance
+
+
+def load_simpletuner_encoder(checkpoint: Path, device: torch.device):
+    state = load_trainer_state(str(checkpoint))
+    exported = checkpoint / "rvq_encoder.safetensors"
+    evaluation_checkpoint = EvaluationCheckpoint(checkpoint.name, checkpoint, int(state["global_step"]))
+    model = load_evaluation_model(evaluation_checkpoint).to(device).eval()
+    provenance = {
+        "checkpoint": f"{checkpoint.parent.parent.name}/output/final",
+        "checkpoint_sha256": sha256_file(exported),
+        "checkpoint_bytes": exported.stat().st_size,
+        "global_step": int(state["global_step"]),
+    }
+    return model, provenance
+
+
+def normalize_logits(output: Any) -> list[torch.Tensor]:
+    if isinstance(output, tuple) and len(output) == 2 and isinstance(output[1], (list, tuple)):
+        return [output[0], *output[1]]
+    if isinstance(output, (list, tuple)) and len(output) == 8:
+        return list(output)
+    raise TypeError(f"Unsupported encoder output type: {type(output)!r}.")
+
+
+def predict_track(model, record, cache_dir: Path, device: torch.device):
+    tensor_path, _ = _cache_paths(cache_dir, record)
+    with safe_open(str(tensor_path), framework="pt", device="cpu") as tensors:
+        latents = tensors.get_tensor("latents").float()
+        codes = tensors.get_tensor("codes").long()
+    frame_count = min(record.emitted_frames, codes.shape[0] - 1)
+    starts, _ = frame_latent_starts(frame_count, record.alignment)
+    while frame_count > 0 and starts[frame_count] > latents.shape[0]:
+        frame_count -= 1
+    predictions = torch.empty((frame_count, 8), dtype=torch.long)
+    assigned = torch.zeros(frame_count, dtype=torch.bool)
+    regular_starts = list(range(0, frame_count - WINDOW_FRAMES + 1, WINDOW_FRAMES))
+    inference_starts = list(regular_starts)
+    tail_start = frame_count - WINDOW_FRAMES
+    if tail_start >= 0 and (not inference_starts or inference_starts[-1] != tail_start):
+        inference_starts.append(tail_start)
+    head_stats = [dict(ce_sum=0.0, top1_correct=0, top5_correct=0, count=0) for _ in range(8)]
+    with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
+        for frame_start in inference_starts:
+            bounds = starts[frame_start : frame_start + WINDOW_FRAMES + 1]
+            latent_start, latent_end = bounds[0], bounds[-1]
+            window_latents = latents[latent_start:latent_end].to(device)
+            pool = build_pool_matrix(bounds).to(device)
+            logits = normalize_logits(model(window_latents.unsqueeze(0), pool.unsqueeze(0)))
+            predicted = torch.stack([head.argmax(-1)[0] for head in logits], dim=-1).cpu()
+            take = ~assigned[frame_start : frame_start + WINDOW_FRAMES]
+            predictions[frame_start : frame_start + WINDOW_FRAMES][take] = predicted[take]
+            assigned[frame_start : frame_start + WINDOW_FRAMES][take] = True
+            if frame_start in regular_starts:
+                target = codes[frame_start + 1 : frame_start + 1 + WINDOW_FRAMES].to(device)
+                for head_index, head_logits in enumerate(logits):
+                    values = head_logits[0].float()
+                    labels = target[:, head_index]
+                    stats = head_stats[head_index]
+                    stats["ce_sum"] += F.cross_entropy(values, labels, reduction="sum").item()
+                    stats["top1_correct"] += (values.argmax(-1) == labels).sum().item()
+                    stats["top5_correct"] += (
+                        (values.topk(5, dim=-1).indices == labels[:, None]).any(-1).sum().item()
+                    )
+                    stats["count"] += labels.numel()
+    if not assigned.all():
+        raise RuntimeError(f"Predictions do not cover every frame for {record.sample_id}.")
+    return codes[: frame_count + 1], predictions, head_stats
+
+
+def prompt_ids(tokenizer, metadata: dict[str, Any], device: torch.device) -> torch.Tensor:
+    caption = _clean_caption(metadata["prompt"])
+    lyrics = _normalize_lyrics(metadata["lyrics"])
+    text = (
+        f"<|im_start|><|caption_start|>{caption}<|caption_end|>"
+        f"<|lyrics_start|>{lyrics}<|lyrics_end|><|im_end|><|audio_start|>"
+    )
+    return tokenizer(text, return_tensors="pt")["input_ids"].to(device)
+
+
+def replay_frame_hiddens(tokenizer, language_model, depth_decoder, metadata, codes, device):
+    codes = codes.to(device)
+    ids = prompt_ids(tokenizer, metadata, device)
+    with torch.inference_mode():
+        semantic = language_model.model.embed_tokens(codes[:-1, :1] + SEMANTIC_OFFSET)
+        offsets = torch.arange(7, device=device).view(1, 7) * 1024
+        acoustic = depth_decoder.audio_embeddings(codes[:-1, 1:] + offsets).sum(1, keepdim=True)
+        feedback = ((semantic + acoustic.to(semantic.dtype)) * (8**-0.5)).squeeze(1).unsqueeze(0)
+        output = language_model.model(inputs_embeds=language_model.model.embed_tokens(ids), use_cache=True)
+        past = output.past_key_values
+        hidden_parts = []
+        for start in range(0, feedback.shape[1], LM_BLOCK_FRAMES):
+            output = language_model.model(
+                inputs_embeds=feedback[:, start : start + LM_BLOCK_FRAMES],
+                past_key_values=past,
+                use_cache=True,
+            )
+            past = output.past_key_values
+            hidden_parts.append(output.last_hidden_state)
+        hidden = torch.cat(hidden_parts, dim=1).squeeze(0)
+        emitted = codes[1:]
+        sequence = [
+            depth_decoder.projection(hidden).unsqueeze(1),
+            depth_decoder.projection(
+                language_model.model.embed_tokens(emitted[:, :1] + SEMANTIC_OFFSET).squeeze(1)
+            ).unsqueeze(1),
+        ]
+        depth_parts = []
+        for codebook in range(7):
+            depth_hidden = depth_decoder(torch.cat(sequence, dim=1))[:, -1]
+            depth_parts.append(depth_hidden)
+            if codebook < 6:
+                embedding = depth_decoder.audio_embeddings(emitted[:, codebook + 1] + codebook * 1024)
+                sequence.append(depth_decoder.projection(embedding).unsqueeze(1))
+        return torch.cat((hidden, torch.cat(depth_parts, dim=-1)), dim=-1).unsqueeze(0)
+
+
+def stitched_condition(condition_encoder, frame_hiddens, chunks):
+    outputs = []
+    previous = None
+    with torch.inference_mode():
+        for chunk in chunks:
+            start = int(chunk["semantic_frame_start"])
+            end = int(chunk["semantic_frame_end_exclusive"])
+            condition = condition_encoder(frame_hiddens[:, start:end])
+            if previous is not None:
+                overlap = min(previous.shape[1], condition.shape[1])
+                condition[:, :overlap] = previous[:, :overlap]
+            overlap_start = max(0, condition.shape[1] - 344)
+            overlap_end = max(overlap_start, condition.shape[1] - 172)
+            previous = condition[:, overlap_start:overlap_end].clone()
+            keep_start = int(chunk["kept_flow_latent_start"])
+            keep_end = int(chunk["kept_flow_latent_end_exclusive"])
+            outputs.append(condition[:, keep_start:keep_end])
+    return torch.cat(outputs, dim=1).squeeze(0).float()
+
+
+def condition_score(predicted: torch.Tensor, reference: torch.Tensor) -> dict[str, float | int]:
+    cosine = F.cosine_similarity(predicted, reference, dim=-1)
+    squared_error = torch.sum((predicted - reference) ** 2).item()
+    reference_squared = torch.sum(reference * reference).item()
+    return {
+        "cosine_mean": cosine.mean().item(),
+        "cosine_std": cosine.std(correction=0).item(),
+        "cosine_min": cosine.min().item(),
+        "cosine_max": cosine.max().item(),
+        "flattened_dot": torch.sum(predicted * reference).item(),
+        "predicted_squared": torch.sum(predicted * predicted).item(),
+        "reference_squared": reference_squared,
+        "squared_error": squared_error,
+        "mse": squared_error / predicted.numel(),
+        "relative_l2": math.sqrt(squared_error / reference_squared),
+        "latent_frames": predicted.shape[0],
+        "values": predicted.numel(),
+    }
+
+
+def distribution(values: list[float]) -> dict[str, Any]:
+    tensor = torch.tensor(values, dtype=torch.float64)
+    quantiles = {str(q): torch.quantile(tensor, q).item() for q in (0.0, 0.05, 0.25, 0.5, 0.75, 0.95, 1.0)}
+    return {
+        "count": len(values),
+        "mean": statistics.fmean(values),
+        "std": statistics.pstdev(values),
+        "min": min(values),
+        "max": max(values),
+        "quantiles": quantiles,
+    }
+
+
+def aggregate_condition(rows: list[dict[str, Any]], key: str) -> dict[str, Any]:
+    scores = [row["condition_embedding_replay"][key] for row in rows]
+    frame_count = sum(score["latent_frames"] for score in scores)
+    dot = sum(score["flattened_dot"] for score in scores)
+    predicted_squared = sum(score["predicted_squared"] for score in scores)
+    reference_squared = sum(score["reference_squared"] for score in scores)
+    squared_error = sum(score["squared_error"] for score in scores)
+    values = sum(score["values"] for score in scores)
+    return {
+        "per_record_cosine": distribution([score["cosine_mean"] for score in scores]),
+        "latent_weighted_mean_cosine": sum(
+            score["cosine_mean"] * score["latent_frames"] for score in scores
+        )
+        / frame_count,
+        "global_flattened_cosine": dot / math.sqrt(predicted_squared * reference_squared),
+        "mse": squared_error / values,
+        "relative_l2": math.sqrt(squared_error / reference_squared),
+        "latent_frames": frame_count,
+    }
+
+
+def aggregate_tokens(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    per_head = []
+    for head in range(8):
+        stats = [row["token_metrics"]["per_head"][head] for row in rows]
+        count = sum(item["count"] for item in stats)
+        per_head.append(
+            {
+                "head": head,
+                "vocab_size": 16384 if head == 0 else 1024,
+                "frames": count,
+                "cross_entropy": sum(item["ce_sum"] for item in stats) / count,
+                "top1": sum(item["top1_correct"] for item in stats) / count,
+                "top5": sum(item["top5_correct"] for item in stats) / count,
+            }
+        )
+    return {
+        "loss": statistics.fmean(item["cross_entropy"] for item in per_head),
+        "semantic_top1": per_head[0]["top1"],
+        "semantic_top5": per_head[0]["top5"],
+        "acoustic_top1": statistics.fmean(item["top1"] for item in per_head[1:]),
+        "acoustic_top5": statistics.fmean(item["top5"] for item in per_head[1:]),
+        "per_head": per_head,
+    }
+
+
+def run(args: argparse.Namespace) -> None:
+    device = torch.device("cuda")
+    if torch.cuda.device_count() != 1:
+        raise RuntimeError("Expose exactly one assigned GPU with CUDA_VISIBLE_DEVICES.")
+    output_dir = Path(args.output_dir)
+    jsonl_path = output_dir / f"records-{args.model_id}.jsonl"
+    rows = load_jsonl(jsonl_path)
+    completed = {row["sample_id"] for row in rows}
+    if args.encoder_kind == "serveurperso":
+        encoder, checkpoint_provenance = load_external_encoder(
+            Path(args.checkpoint), Path(args.external_source), device
+        )
+    else:
+        encoder, checkpoint_provenance = load_simpletuner_encoder(Path(args.checkpoint), device)
+    encoder.requires_grad_(False)
+    tokenizer = Qwen2Tokenizer.from_pretrained(Path(args.official_model_root) / "tokenizer")
+    language_model = Qwen3ForCausalLM.from_pretrained(
+        Path(args.official_model_root) / "language_model",
+        torch_dtype=torch.bfloat16,
+        attn_implementation="sdpa",
+    ).to(device).eval()
+    depth_decoder = MiniMaxMusic3RVQDepthDecoder.from_pretrained(
+        Path(args.official_model_root) / "rvq_depth_decoder", torch_dtype=torch.bfloat16
+    ).to(device).eval()
+    condition_encoder = MiniMaxMusic3ConditionEncoder.from_pretrained(
+        Path(args.official_model_root) / "condition_encoder", torch_dtype=torch.bfloat16
+    ).to(device).eval()
+    for model in (language_model, depth_decoder, condition_encoder):
+        model.requires_grad_(False)
+
+    parser = build_parser()
+    dataset_args = parser.parse_args([])
+    dataset_args.dataset_repo_id = args.dataset_repo_id
+    dataset_args.dataset_revision = DATASET_REVISION
+    dataset_args.hf_cache_dir = args.hf_cache_dir
+    dataset_args.dataset_cache_dir = None
+    dataset_args.validation_fraction = 0.1
+    dataset_args.seed = 42
+    _, validation = load_records(dataset_args)
+    records = [record for record in validation if record.alignment.get("chunk_stitching")]
+    if len(records) != 130:
+        raise RuntimeError(f"Expected 130 exact holdout records, found {len(records)}.")
+
+    started = time.time()
+    for index, record in enumerate(records, start=1):
+        if record.sample_id in completed:
+            continue
+        true_codes, predicted, token_stats = predict_track(encoder, record, Path(args.latent_cache_dir), device)
+        replay_codes = torch.cat((true_codes[:1], predicted), dim=0)
+        shard_path = _resolve_shard_path(dataset_args, record)
+        with zipfile.ZipFile(shard_path) as archive:
+            metadata_name = next(name for name in archive.namelist() if name.endswith("metadata.json"))
+            metadata = json.loads(archive.read(metadata_name))
+            reference = _read_zip_safetensors(
+                archive, record.tensor_file, required_names=("condition_embeddings",)
+            )["condition_embeddings"].float().to(device)
+        chunks = metadata["alignment"]["chunk_stitching"]
+        true_condition = stitched_condition(
+            condition_encoder,
+            replay_frame_hiddens(tokenizer, language_model, depth_decoder, metadata, true_codes, device),
+            chunks,
+        )
+        predicted_condition = stitched_condition(
+            condition_encoder,
+            replay_frame_hiddens(tokenizer, language_model, depth_decoder, metadata, replay_codes, device),
+            chunks,
+        )
+        if true_condition.shape != reference.shape or predicted_condition.shape != reference.shape:
+            raise RuntimeError(f"Condition shape mismatch for {record.sample_id}.")
+        row = {
+            "model_id": args.model_id,
+            "sample_id": record.sample_id,
+            "shard_id": record.shard_id,
+            "emitted_frames": int(predicted.shape[0]),
+            "condition_latent_frames": int(reference.shape[0]),
+            "token_metrics": {"per_head": token_stats},
+            "condition_embedding_replay": {
+                "true_code_control": condition_score(true_condition, reference),
+                "predicted_codes": condition_score(predicted_condition, reference),
+            },
+        }
+        append_jsonl(jsonl_path, row)
+        rows.append(row)
+        completed.add(record.sample_id)
+        progress = {
+            "model_id": args.model_id,
+            "completed": len(completed),
+            "total": len(records),
+            "last_sample_id": record.sample_id,
+        }
+        atomic_json(output_dir / f"progress-{args.model_id}.json", progress)
+        predicted_cosine = row["condition_embedding_replay"]["predicted_codes"]["cosine_mean"]
+        true_cosine = row["condition_embedding_replay"]["true_code_control"]["cosine_mean"]
+        print(
+            f"[{args.model_id} {index}/{len(records)}] true={true_cosine:.6f} "
+            f"predicted={predicted_cosine:.6f} elapsed={time.time() - started:.1f}s",
+            flush=True,
+        )
+        del true_condition, predicted_condition, reference
+        torch.cuda.empty_cache()
+
+    rows = load_jsonl(jsonl_path)
+    if len(rows) != 130:
+        raise RuntimeError(f"Evaluation incomplete: {len(rows)} of 130 records.")
+    result = {
+        "format": "simpletuner-minimaxmusic-external-replay-v1",
+        "model_id": args.model_id,
+        "parameter_count": sum(parameter.numel() for parameter in encoder.parameters()),
+        "checkpoint": checkpoint_provenance,
+        "dataset": {
+            "repo_id": args.dataset_repo_id,
+            "revision": DATASET_REVISION,
+            "split": "holdout",
+            "exact_alignment_records": len(rows),
+        },
+        "protocol": {
+            "window_frames": WINDOW_FRAMES,
+            "code_selection": "argmax",
+            "replay": "teacher-forced official language model and RVQ depth decoder",
+            "condition": "official condition encoder with recorded chunk stitching",
+            "metric": "cosine similarity per stitched condition-embedding latent frame",
+            "official_model_revision": OFFICIAL_MODEL_REVISION,
+        },
+        "condition_embedding_replay": {
+            "true_code_control": aggregate_condition(rows, "true_code_control"),
+            "predicted_codes": aggregate_condition(rows, "predicted_codes"),
+        },
+        "token_metrics": aggregate_tokens(rows),
+        "records": rows,
+        "limitations": [
+            "This is teacher-forced hidden-state replay, not a diffusion render or audio-domain score.",
+            "Only the 130 holdout records with recorded exact chunk alignment are included.",
+            "Complete non-overlapping windows define token metrics; a shifted final window fills replay-only tail frames.",
+            "The Serveurperso checkpoint was trained on a separate corpus and has no checkpoint-specific license declaration.",
+        ],
+    }
+    atomic_json(output_dir / f"raw-metrics-{args.model_id}.json", result)
+    atomic_json(output_dir / f"progress-{args.model_id}.json", {"model_id": args.model_id, "completed": 130, "total": 130, "status": "complete"})
+
+
+def combine(args: argparse.Namespace) -> None:
+    output_dir = Path(args.output_dir)
+    results = [json.loads((output_dir / f"raw-metrics-{model_id}.json").read_text()) for model_id in MODEL_IDS]
+    models = []
+    for result in results:
+        predicted = result["condition_embedding_replay"]["predicted_codes"]["per_record_cosine"]
+        models.append(
+            {
+                "model_id": result["model_id"],
+                "parameter_count": result["parameter_count"],
+                "checkpoint": result["checkpoint"],
+                "predicted_code_condition_cosine": predicted,
+                "true_code_control": result["condition_embedding_replay"]["true_code_control"]["per_record_cosine"],
+            }
+        )
+    combined = {
+        "format": "simpletuner-minimaxmusic-replay-comparison-v1",
+        "metric": "per-record mean stitched condition-embedding cosine",
+        "dataset": results[0]["dataset"],
+        "models": models,
+    }
+    atomic_json(output_dir / "combined-aggregate.json", combined)
+    atomic_json(output_dir / "comparison-chart-data.json", combined)
+    provenance = {
+        "serveurperso": {
+            "checkpoint_url": CHECKPOINT_URL,
+            "checkpoint_sha256": CHECKPOINT_SHA256,
+            "source_commit": SOURCE_COMMIT,
+            "source_url": SOURCE_URL,
+            "code_license": "MIT",
+            "checkpoint_license": "not separately declared",
+        },
+        "official_replay_components": {
+            "repo_id": "MiniMaxAI/MiniMax-Music3",
+            "revision": OFFICIAL_MODEL_REVISION,
+            "license": "MiniMax-Music3 Community License",
+        },
+        "dataset": {"repo_id": results[0]["dataset"]["repo_id"], "revision": DATASET_REVISION},
+        "checkpoints": {result["model_id"]: result["checkpoint"] for result in results},
+    }
+    atomic_json(output_dir / "provenance.json", provenance)
+
+
+def build_cli() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--combine", action="store_true")
+    parser.add_argument("--model-id", choices=MODEL_IDS)
+    parser.add_argument("--encoder-kind", choices=("serveurperso", "simpletuner"))
+    parser.add_argument("--checkpoint")
+    parser.add_argument("--external-source")
+    parser.add_argument("--official-model-root")
+    parser.add_argument("--latent-cache-dir")
+    parser.add_argument("--hf-cache-dir")
+    parser.add_argument("--dataset-repo-id", default="bghira/minimax-music3-rvq-reverse-distillation")
+    parser.add_argument("--output-dir", default="model_cards/external/serveurperso")
+    return parser
+
+
+if __name__ == "__main__":
+    arguments = build_cli().parse_args()
+    if arguments.combine:
+        combine(arguments)
+    else:
+        required = (
+            "model_id",
+            "encoder_kind",
+            "checkpoint",
+            "official_model_root",
+            "latent_cache_dir",
+            "hf_cache_dir",
+        )
+        missing = [name for name in required if getattr(arguments, name) is None]
+        if missing:
+            raise ValueError(f"Missing required arguments: {missing}.")
+        if arguments.encoder_kind == "serveurperso" and arguments.external_source is None:
+            raise ValueError("--external-source is required for Serveurperso.")
+        run(arguments)

evaluation/replay-comparison/provenance.json

@@ -0,0 +1,48 @@
+{
+  "checkpoints": {
+    "serveurperso-v1": {
+      "checkpoint_bytes": 163955786,
+      "checkpoint_sha256": "fd7bfefb197cb7b65a6b6d9b1373c1772a15979f1a4b67d820764d610f0fb9a5",
+      "checkpoint_url": "https://www.serveurperso.com/ia/ssd/workspace/git/minimaxmusic.cpp/training/checkpoints/v1/best.pt",
+      "epoch": 17,
+      "reported_validation_loss": 5.292583839459852,
+      "source_commit": "d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1",
+      "source_url": "https://github.com/ServeurpersoCom/minimaxmusic.cpp/commit/d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1"
+    },
+    "simpletuner-v1": {
+      "checkpoint": "full/output/final",
+      "checkpoint_bytes": 163931632,
+      "checkpoint_sha256": "1cea88dddf7a48583c13ac661a548b7a3535b933a76356bb158420d24da32f22",
+      "global_step": 17640
+    },
+    "simpletuner-v2": {
+      "checkpoint": "v2/output/final",
+      "checkpoint_bytes": 618960456,
+      "checkpoint_sha256": "47dfffb7a76d95589752df0c59ff6eaf3496f6fcf5408d324218887e471023c7",
+      "global_step": 17660
+    },
+    "simpletuner-v3": {
+      "checkpoint": "v3/output/final",
+      "checkpoint_bytes": 618960456,
+      "checkpoint_sha256": "356e97fea65c486a28814dc778b52645807ab8f784c6021d2403f688a2bfe788",
+      "global_step": 17660
+    }
+  },
+  "dataset": {
+    "repo_id": "bghira/minimax-music3-rvq-reverse-distillation",
+    "revision": "5029b1e7f1bbfbf028b76b38564fecccda94a111"
+  },
+  "official_replay_components": {
+    "license": "MiniMax-Music3 Community License",
+    "repo_id": "MiniMaxAI/MiniMax-Music3",
+    "revision": "fbdf52fbaaca799592917417eb05f1899f1255ec"
+  },
+  "serveurperso": {
+    "checkpoint_license": "not separately declared",
+    "checkpoint_sha256": "fd7bfefb197cb7b65a6b6d9b1373c1772a15979f1a4b67d820764d610f0fb9a5",
+    "checkpoint_url": "https://www.serveurperso.com/ia/ssd/workspace/git/minimaxmusic.cpp/training/checkpoints/v1/best.pt",
+    "code_license": "MIT",
+    "source_commit": "d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1",
+    "source_url": "https://github.com/ServeurpersoCom/minimaxmusic.cpp/commit/d19efe9f94e41ac4c900aa30d56fe90c8dac7ef1"
+  }
+}