| { |
| "evaluation_contract": { |
| "sample_size_per_benchmark": 200, |
| "sequence_length": 16384, |
| "timeline_sampling": "full_span_2fps", |
| "visual_token_budget": 8192 |
| }, |
| "results": [ |
| { |
| "benchmark": "lvbench", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/lvbench/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_041030_results.json", |
| "sample_count": 200, |
| "score": 49.5 |
| }, |
| "delta": 1.5, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/lvbench/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_035924_results.json", |
| "sample_count": 200, |
| "score": 48.0 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/lvbench_n200.json", |
| "paired_ids_verified": true |
| }, |
| { |
| "benchmark": "mvbench", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/mvbench/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_043534_results.json", |
| "sample_count": 200, |
| "score": 68.5 |
| }, |
| "delta": -7.0, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/mvbench/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_043132_results.json", |
| "sample_count": 200, |
| "score": 75.5 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/mvbench_n200.json", |
| "paired_ids_verified": true |
| }, |
| { |
| "benchmark": "nextqa", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/nextqa/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_054907_results.json", |
| "sample_count": 200, |
| "score": 83.5 |
| }, |
| "delta": -0.5, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/nextqa/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_054534_results.json", |
| "sample_count": 200, |
| "score": 84.0 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/nextqa_n200.json", |
| "paired_ids_verified": true |
| }, |
| { |
| "benchmark": "perceptiontest", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/perceptiontest/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_045116_results.json", |
| "sample_count": 200, |
| "score": 83.0 |
| }, |
| "delta": 1.0, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/perceptiontest/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_044634_results.json", |
| "sample_count": 200, |
| "score": 82.0 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/perceptiontest_n200.json", |
| "paired_ids_verified": true |
| }, |
| { |
| "benchmark": "tempcompass", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/tempcompass/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_044308_results.json", |
| "sample_count": 200, |
| "score": 68.5 |
| }, |
| "delta": -0.5, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/tempcompass/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_044022_results.json", |
| "sample_count": 200, |
| "score": 69.0 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/tempcompass_n200.json", |
| "paired_ids_verified": true |
| }, |
| { |
| "benchmark": "videomme", |
| "codec": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/videomme/codec/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_035721_results.json", |
| "sample_count": 200, |
| "score": 69.0 |
| }, |
| "delta": -1.0, |
| "dense": { |
| "result_path": "/fsx/home/weikai.huang/molmo2_codec/evals/multibench_step100_fullspan_n200/videomme/dense/codec_stage2_8k_hex_validation__step100_consolidated.pt/20260807_034942_results.json", |
| "sample_count": 200, |
| "score": 70.0 |
| }, |
| "manifest": "/fsx/home/weikai.huang/adacodec_vlm/eval_manifests/videomme_n200.json", |
| "paired_ids_verified": true |
| } |
| ], |
| "schema_version": 1, |
| "status": "complete" |
| } |
|
|
