| { |
| "reference": "runs/bf16-golden/bf16-golden.jsonl", |
| "candidate": "runs/nvfp4-gptq-fixtures.jsonl", |
| "thresholds": { |
| "max_mean_delta": 1.0, |
| "max_coord_delta": 5.0, |
| "require_label_match": true |
| }, |
| "cases": [ |
| { |
| "case_id": "coco-000000037777-banana", |
| "reference_labels": { |
| "banana": 1 |
| }, |
| "candidate_labels": { |
| "banana": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 12.0, |
| "max_abs_delta": 40.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": false |
| }, |
| { |
| "case_id": "coco-000000037777-multi", |
| "reference_labels": { |
| "orange": 1, |
| "": 4, |
| "refrigerator": 1 |
| }, |
| "candidate_labels": { |
| "orange": 1, |
| "refrigerator": 1 |
| }, |
| "label_match": false, |
| "mean_abs_delta": 21.25, |
| "max_abs_delta": 114.0, |
| "missing_objects": 4, |
| "extra_objects": 0, |
| "pass": false |
| }, |
| { |
| "case_id": "coco-000000039769-cat", |
| "reference_labels": { |
| "cat": 1 |
| }, |
| "candidate_labels": { |
| "cat": 1, |
| "": 1 |
| }, |
| "label_match": false, |
| "mean_abs_delta": 325.5, |
| "max_abs_delta": 540.0, |
| "missing_objects": 0, |
| "extra_objects": 1, |
| "pass": false |
| }, |
| { |
| "case_id": "coco-000000039769-referring", |
| "reference_labels": { |
| "the larger cat on the right": 1 |
| }, |
| "candidate_labels": { |
| "the larger cat on the right": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000128372-bus", |
| "reference_labels": { |
| "bus": 1 |
| }, |
| "candidate_labels": { |
| "bus": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000252219-multi", |
| "reference_labels": { |
| "person": 1, |
| "": 1, |
| "shopping cart": 1 |
| }, |
| "candidate_labels": { |
| "person": 1, |
| "": 2, |
| "shopping cart": 1 |
| }, |
| "label_match": false, |
| "mean_abs_delta": 41.0, |
| "max_abs_delta": 432.0, |
| "missing_objects": 0, |
| "extra_objects": 1, |
| "pass": false |
| }, |
| { |
| "case_id": "coco-000000252219-person", |
| "reference_labels": { |
| "person": 1, |
| "": 2 |
| }, |
| "candidate_labels": { |
| "person": 1, |
| "": 2 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 18.166666666666668, |
| "max_abs_delta": 150.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": false |
| }, |
| { |
| "case_id": "coco-000000289343-dog", |
| "reference_labels": { |
| "dog": 1 |
| }, |
| "candidate_labels": { |
| "dog": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000397133-person", |
| "reference_labels": { |
| "person": 1 |
| }, |
| "candidate_labels": { |
| "person": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000397133-referring", |
| "reference_labels": { |
| "the person standing at the stove wearing an apron": 1 |
| }, |
| "candidate_labels": { |
| "the person standing at the stove wearing an apron": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000500270-skateboard", |
| "reference_labels": { |
| "skateboard": 1 |
| }, |
| "candidate_labels": { |
| "skateboard": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.0, |
| "max_abs_delta": 0.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| }, |
| { |
| "case_id": "coco-000000581100-giraffe", |
| "reference_labels": { |
| "giraffe": 1, |
| "": 1 |
| }, |
| "candidate_labels": { |
| "giraffe": 1, |
| "": 1 |
| }, |
| "label_match": true, |
| "mean_abs_delta": 0.375, |
| "max_abs_delta": 2.0, |
| "missing_objects": 0, |
| "extra_objects": 0, |
| "pass": true |
| } |
| ], |
| "overall_mean_abs_delta": 31.01388888888889, |
| "overall_max_abs_delta": 540.0, |
| "verdict": "FAIL" |
| } |
|
|
