{ "candidate_generation": { "avg_generation_time_s": 597.4663525009528, "avg_peak_memory_mb": 56444.0, "generation_time_s": 597.4663525009528, "max_peak_memory_mb": 56444.0, "peak_memory_mb": 56444.0, "per_run_generation_time_s": [ 597.4663525009528 ], "per_run_peak_memory_mb": [ 56444.0 ] }, "measure_runs": 1, "metric_guidance": { "descriptions": { "avg_generation_time_s": "Lower is better. Mean wall-clock time across measured runs.", "cosine_similarity": "Higher is better. Measures vector direction similarity and is less sensitive to scale.", "generation_time_s": "Lower is better. Wall-clock time for the last measured generation run.", "l2": "Lower is better. Absolute L2 distance; shape- and scale-dependent.", "mae": "Lower is better. Mean absolute error in the tensor's native scale; uint8 output MAE is in pixel values.", "max_abs": "Lower is better. Worst absolute element error; useful for debugging outliers, not stable as a gate.", "max_peak_memory_mb": "Lower is better. Maximum worker-reported peak memory across measured runs.", "mse": "Lower is better. Mean squared error; more sensitive to large localized differences than MAE.", "output_metrics": "Pixel-space metrics computed on generated RGB images or video frames after decode. These are often the most actionable signal for image/video quality regressions.", "peak_memory_mb": "Lower is better. Worker-reported peak device memory for the last measured generation run.", "psnr_db": "Higher is better. Pixel-space signal-to-noise ratio in dB for uint8 images/frames.", "relative_l2": "Lower is better. L2 distance normalized by the reference L2 norm.", "rmse": "Lower is better. Square root of MSE, in the same unit as the compared tensor." }, "recommended_thresholds": { "output_images_or_frames_uint8": { "cosine_similarity": { "note": "Useful as a broad image/frame similarity sanity check.", "recommended_min": 0.98, "strict_min": 0.995 }, "mae": { "good_max": 6.0, "note": "Measured on uint8 RGB pixels or frames.", "recommended_max": 12.0, "strict_max": 3.0 }, "psnr_db": { "good_min": 25.0, "note": "For stochastic diffusion outputs, 20 dB is a practical smoke threshold; 25+ dB is usually close.", "recommended_min": 20.0, "strict_min": 30.0 }, "relative_l2": { "note": "Pixel-space relative L2 is generally more interpretable than latent-space relative L2.", "recommended_max": 0.2, "strict_max": 0.08 } }, "performance": { "avg_generation_time_ratio_candidate_over_reference": { "note": "A value above 1.10 suggests a latency regression unless expected for the quantization mode.", "recommended_max": 1.1 }, "max_peak_memory_ratio_candidate_over_reference": { "note": "Quantization should usually not increase peak memory; <=0.95 indicates a meaningful saving.", "recommended_max": 1.0, "target_max": 0.95 } } }, "scope": "Heuristic defaults for same-model comparisons with identical prompt, seed, resolution, scheduler, and inference steps. Tune thresholds per model family, task, and quantization method; visual review is still recommended for release gating." }, "model": "Qwen/Qwen-Image", "negative_prompt": "\u8272\u8c03\u8273\u4e3d\uff0c\u8fc7\u66dd\uff0c\u9759\u6001\uff0c\u7ec6\u8282\u6a21\u7cca\u4e0d\u6e05\uff0c\u5b57\u5e55\uff0c\u98ce\u683c\uff0c\u4f5c\u54c1\uff0c\u753b\u4f5c\uff0c\u753b\u9762\uff0c\u9759\u6b62\uff0c\u6574\u4f53\u53d1\u7070\uff0c\u6700\u5dee\u8d28\u91cf\uff0c\u4f4e\u8d28\u91cf\uff0cJPEG\u538b\u7f29\u6b8b\u7559\uff0c\u4e11\u964b\u7684\uff0c\u6b8b\u7f3a\u7684\uff0c\u591a\u4f59\u7684\u624b\u6307\uff0c\u753b\u5f97\u4e0d\u597d\u7684\u624b\u90e8\uff0c\u753b\u5f97\u4e0d\u597d\u7684\u8138\u90e8\uff0c\u7578\u5f62\u7684\uff0c\u6bc1\u5bb9\u7684\uff0c\u5f62\u6001\u7578\u5f62\u7684\u80a2\u4f53\uff0c\u624b\u6307\u878d\u5408\uff0c\u9759\u6b62\u4e0d\u52a8\u7684\u753b\u9762\uff0c\u6742\u4e71\u7684\u80cc\u666f\uff0c\u4e09\u6761\u817f\uff0c\u80cc\u666f\u4eba\u5f88\u591a\uff0c\u5012\u7740\u8d70", "output_metrics": { "all_frames_metrics": { "cosine_similarity": 0.8037929091027018, "l2": 883570.5652385666, "mae": 34.20187691115112, "max_abs": 255.0, "mse": 3486.0510248592536, "psnr_db": 12.707466213375852, "relative_l2": 0.614866124029748, "rmse": 59.04278977876345 }, "all_images_metrics": { "cosine_similarity": 0.8037929091027018, "l2": 883570.5652385666, "mae": 34.20187691115112, "max_abs": 255.0, "mse": 3486.0510248592536, "psnr_db": 12.707466213375852, "relative_l2": 0.614866124029748, "rmse": 59.04278977876345 }, "frame0_metrics": { "cosine_similarity": 0.819452058788871, "l2": 101280.13322957272, "mae": 36.83278826678241, "max_abs": 255.0, "mse": 3710.093094256366, "psnr_db": 12.436955537287162, "relative_l2": 0.5818974379697734, "rmse": 60.9105335246406 }, "image0_metrics": { "cosine_similarity": 0.819452058788871, "l2": 101280.13322957272, "mae": 36.83278826678241, "max_abs": 255.0, "mse": 3710.093094256366, "psnr_db": 12.436955537287162, "relative_l2": 0.5818974379697734, "rmse": 60.9105335246406 }, "mid_frame_index": 40, "mid_frame_metrics": { "cosine_similarity": 0.8339844622702532, "l2": 90708.42115812622, "mae": 32.221018880208334, "max_abs": 255.0, "mse": 2975.9901869936343, "psnr_db": 13.394488660308774, "relative_l2": 0.5598904836985759, "rmse": 54.5526368473022 }, "mid_image_index": 40, "mid_image_metrics": { "cosine_similarity": 0.8339844622702532, "l2": 90708.42115812622, "mae": 32.221018880208334, "max_abs": 255.0, "mse": 2975.9901869936343, "psnr_db": 13.394488660308774, "relative_l2": 0.5598904836985759, "rmse": 54.5526368473022 }, "num_frames": 81, "num_images": 81 }, "prompt": "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage.", "reference_generation": { "avg_generation_time_s": 389.2413684539497, "avg_peak_memory_mb": 75886.0, "generation_time_s": 389.2413684539497, "max_peak_memory_mb": 75886.0, "peak_memory_mb": 75886.0, "per_run_generation_time_s": [ 389.2413684539497 ], "per_run_peak_memory_mb": [ 75886.0 ] }, "sampling_kwargs": { "cfg_scale": 4.0, "flow_shift": null, "guidance_scale": 4.0, "guidance_scale_2": 3.0, "height": 720, "num_frames": 81, "num_images_per_prompt": 1, "num_inference_steps": 40, "step_execution": false, "task": "t2v", "width": 1280 }, "saved_outputs": { "candidate": [ "/models/yiliu7/tmp/wan_a14b_sim_calib_out/candidate/video.mp4", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/candidate/frame_0.png", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/candidate/frame_40.png", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/candidate/frame_80.png" ], "reference": [ "/models/yiliu7/tmp/wan_a14b_sim_calib_out/reference/video.mp4", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/reference/frame_0.png", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/reference/frame_40.png", "/models/yiliu7/tmp/wan_a14b_sim_calib_out/reference/frame_80.png" ] }, "seed": 42, "variant_configs": { "candidate": { "diffusion_load_format": "default", "label": "candidate", "model": "/models/yiliu7/tmp/wana14b-w4a8-svd-calibrated", "quantization": null, "quantization_config": { "method": "quark_svdquant" } }, "reference": { "diffusion_load_format": "default", "label": "reference", "model": "/models/Wan2.2-T2V-A14B-Diffusers", "quantization": null, "quantization_config": null } }, "warmup_runs": 0 }