{
  "candidate_generation": {
    "avg_generation_time_s": 15.521200100076385,
    "avg_peak_memory_mb": 26266.0,
    "generation_time_s": 15.521200100076385,
    "max_peak_memory_mb": 26266.0,
    "peak_memory_mb": 26266.0,
    "per_run_generation_time_s": [
      15.521200100076385
    ],
    "per_run_peak_memory_mb": [
      26266.0
    ]
  },
  "measure_runs": 1,
  "metric_guidance": {
    "descriptions": {
      "avg_generation_time_s": "Lower is better. Mean wall-clock time across measured runs.",
      "cosine_similarity": "Higher is better. Measures vector direction similarity and is less sensitive to scale.",
      "generation_time_s": "Lower is better. Wall-clock time for the last measured generation run.",
      "l2": "Lower is better. Absolute L2 distance; shape- and scale-dependent.",
      "mae": "Lower is better. Mean absolute error in the tensor's native scale; uint8 output MAE is in pixel values.",
      "max_abs": "Lower is better. Worst absolute element error; useful for debugging outliers, not stable as a gate.",
      "max_peak_memory_mb": "Lower is better. Maximum worker-reported peak memory across measured runs.",
      "mse": "Lower is better. Mean squared error; more sensitive to large localized differences than MAE.",
      "output_metrics": "Pixel-space metrics computed on generated RGB images or video frames after decode. These are often the most actionable signal for image/video quality regressions.",
      "peak_memory_mb": "Lower is better. Worker-reported peak device memory for the last measured generation run.",
      "psnr_db": "Higher is better. Pixel-space signal-to-noise ratio in dB for uint8 images/frames.",
      "relative_l2": "Lower is better. L2 distance normalized by the reference L2 norm.",
      "rmse": "Lower is better. Square root of MSE, in the same unit as the compared tensor."
    },
    "recommended_thresholds": {
      "output_images_or_frames_uint8": {
        "cosine_similarity": {
          "note": "Useful as a broad image/frame similarity sanity check.",
          "recommended_min": 0.98,
          "strict_min": 0.995
        },
        "mae": {
          "good_max": 6.0,
          "note": "Measured on uint8 RGB pixels or frames.",
          "recommended_max": 12.0,
          "strict_max": 3.0
        },
        "psnr_db": {
          "good_min": 25.0,
          "note": "For stochastic diffusion outputs, 20 dB is a practical smoke threshold; 25+ dB is usually close.",
          "recommended_min": 20.0,
          "strict_min": 30.0
        },
        "relative_l2": {
          "note": "Pixel-space relative L2 is generally more interpretable than latent-space relative L2.",
          "recommended_max": 0.2,
          "strict_max": 0.08
        }
      },
      "performance": {
        "avg_generation_time_ratio_candidate_over_reference": {
          "note": "A value above 1.10 suggests a latency regression unless expected for the quantization mode.",
          "recommended_max": 1.1
        },
        "max_peak_memory_ratio_candidate_over_reference": {
          "note": "Quantization should usually not increase peak memory; <=0.95 indicates a meaningful saving.",
          "recommended_max": 1.0,
          "target_max": 0.95
        }
      }
    },
    "scope": "Heuristic defaults for same-model comparisons with identical prompt, seed, resolution, scheduler, and inference steps. Tune thresholds per model family, task, and quantization method; visual review is still recommended for release gating."
  },
  "model": "Qwen/Qwen-Image",
  "negative_prompt": null,
  "output_metrics": {
    "all_frames_metrics": {
      "cosine_similarity": 0.997544112200434,
      "l2": 84377.22239443533,
      "mae": 7.506763565607411,
      "max_abs": 220.0,
      "mse": 144.93750282973082,
      "psnr_db": 26.51899586406769,
      "relative_l2": 0.07156675913721222,
      "rmse": 12.038999245358013
    },
    "all_images_metrics": {
      "cosine_similarity": 0.997544112200434,
      "l2": 84377.22239443533,
      "mae": 7.506763565607411,
      "max_abs": 220.0,
      "mse": 144.93750282973082,
      "psnr_db": 26.51899586406769,
      "relative_l2": 0.07156675913721222,
      "rmse": 12.038999245358013
    },
    "frame0_metrics": {
      "cosine_similarity": 0.9973642377448177,
      "l2": 13549.689369133153,
      "mae": 7.316384548611111,
      "max_abs": 197.0,
      "mse": 153.2402527377137,
      "psnr_db": 26.277075012784575,
      "relative_l2": 0.07333721540778097,
      "rmse": 12.379024708663994
    },
    "image0_metrics": {
      "cosine_similarity": 0.9973642377448177,
      "l2": 13549.689369133153,
      "mae": 7.316384548611111,
      "max_abs": 197.0,
      "mse": 153.2402527377137,
      "psnr_db": 26.277075012784575,
      "relative_l2": 0.07333721540778097,
      "rmse": 12.379024708663994
    },
    "mid_frame_index": 20,
    "mid_frame_metrics": {
      "cosine_similarity": 0.9972522794331613,
      "l2": 13854.737493002169,
      "mae": 8.300264590010684,
      "max_abs": 189.0,
      "mse": 160.21780765892095,
      "psnr_db": 26.083695760972308,
      "relative_l2": 0.07601119114218123,
      "rmse": 12.657717316282621
    },
    "mid_image_index": 20,
    "mid_image_metrics": {
      "cosine_similarity": 0.9972522794331613,
      "l2": 13854.737493002169,
      "mae": 8.300264590010684,
      "max_abs": 189.0,
      "mse": 160.21780765892095,
      "psnr_db": 26.083695760972308,
      "relative_l2": 0.07601119114218123,
      "rmse": 12.657717316282621
    },
    "num_frames": 41,
    "num_images": 41
  },
  "prompt": "A serene lakeside sunrise with mist over the water.",
  "reference_generation": {
    "avg_generation_time_s": 6.458570759976283,
    "avg_peak_memory_mb": 32084.0,
    "generation_time_s": 6.458570759976283,
    "max_peak_memory_mb": 32084.0,
    "peak_memory_mb": 32084.0,
    "per_run_generation_time_s": [
      6.458570759976283
    ],
    "per_run_peak_memory_mb": [
      32084.0
    ]
  },
  "sampling_kwargs": {
    "cfg_scale": 4.0,
    "flow_shift": null,
    "guidance_scale": 4.0,
    "guidance_scale_2": null,
    "height": 480,
    "num_frames": 41,
    "num_images_per_prompt": 1,
    "num_inference_steps": 20,
    "step_execution": false,
    "task": "t2v",
    "width": 832
  },
  "saved_outputs": {
    "candidate": [
      "/models/yiliu7/tmp/wan5b_sim_calib_out/candidate/video.mp4",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/candidate/frame_0.png",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/candidate/frame_20.png",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/candidate/frame_40.png"
    ],
    "reference": [
      "/models/yiliu7/tmp/wan5b_sim_calib_out/reference/video.mp4",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/reference/frame_0.png",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/reference/frame_20.png",
      "/models/yiliu7/tmp/wan5b_sim_calib_out/reference/frame_40.png"
    ]
  },
  "seed": 42,
  "variant_configs": {
    "candidate": {
      "diffusion_load_format": "default",
      "label": "candidate",
      "model": "/models/yiliu7/tmp/wan5b-w4a8-svd-calibrated",
      "quantization": null,
      "quantization_config": {
        "method": "quark_svdquant"
      }
    },
    "reference": {
      "diffusion_load_format": "default",
      "label": "reference",
      "model": "/models/yiliu7/Wan-AI/Wan2.2-TI2V-5B-Diffusers",
      "quantization": null,
      "quantization_config": null
    }
  },
  "warmup_runs": 0
}