{
  "schema_version": 1,
  "evaluation_split": "test",
  "independent_groups": 8,
  "examples": 576,
  "bootstrap_unit": "group_id",
  "bootstrap_iterations": 1000,
  "systems": {
    "learned_hybrid": {
      "examples": 576,
      "positives": 520,
      "true_positives": 493,
      "false_positives": 54,
      "false_negatives": 27,
      "recall": 0.948076923076923,
      "precision": 0.9012797074954296,
      "specificity": 0.03571428571428571,
      "escalation_rate": 0.9496527777777778,
      "brier": 0.25443746488730223,
      "ece": 0.4136310890206015,
      "auroc": 0.8687843406593406,
      "recall_95ci": [
        0.8519230769230769,
        1.0
      ],
      "escalation_rate_95ci": [
        0.8628472222222222,
        1.0
      ],
      "estimated_vlm_gpu_seconds": 191.45,
      "vlm_call_savings_vs_always_vlm": 0.05034722222222221
    },
    "v3_rule_router": {
      "examples": 576,
      "positives": 520,
      "true_positives": 155,
      "false_positives": 3,
      "false_negatives": 365,
      "recall": 0.2980769230769231,
      "precision": 0.9810126582278481,
      "specificity": 0.9464285714285714,
      "escalation_rate": 0.2743055555555556,
      "brier": 0.6388888888888888,
      "ece": 0.6388888888888888,
      "auroc": 0.6222527472527473,
      "recall_95ci": [
        0.03125,
        0.6319444444444444
      ],
      "escalation_rate_95ci": [
        0.03125,
        0.5954861111111112
      ],
      "estimated_vlm_gpu_seconds": 55.3,
      "vlm_call_savings_vs_always_vlm": 0.7256944444444444
    },
    "always_vlm": {
      "examples": 576,
      "positives": 520,
      "true_positives": 520,
      "false_positives": 56,
      "false_negatives": 0,
      "recall": 1.0,
      "precision": 0.9027777777777778,
      "specificity": 0.0,
      "escalation_rate": 1.0,
      "brier": 0.09722222222222222,
      "ece": 0.09722222222222221,
      "auroc": 0.5,
      "recall_95ci": [
        1.0,
        1.0
      ],
      "escalation_rate_95ci": [
        1.0,
        1.0
      ],
      "estimated_vlm_gpu_seconds": 201.6,
      "vlm_call_savings_vs_always_vlm": 0.0
    },
    "never_vlm": {
      "examples": 576,
      "positives": 520,
      "true_positives": 0,
      "false_positives": 0,
      "false_negatives": 520,
      "recall": 0.0,
      "precision": 1.0,
      "specificity": 1.0,
      "escalation_rate": 0.0,
      "brier": 0.9027777777777778,
      "ece": 0.9027777777777778,
      "auroc": 0.5,
      "recall_95ci": [
        0.0,
        0.0
      ],
      "escalation_rate_95ci": [
        0.0,
        0.0
      ],
      "estimated_vlm_gpu_seconds": 0.0,
      "vlm_call_savings_vs_always_vlm": 1.0
    }
  },
  "v4_condition_metrics": {
    "risk": {
      "examples": 576,
      "positives": 144,
      "true_positives": 142,
      "false_positives": 0,
      "false_negatives": 2,
      "recall": 0.9861111111111112,
      "precision": 1.0,
      "specificity": 1.0,
      "escalation_rate": 0.2465277777777778,
      "brier": 0.06036160690906683,
      "ece": 0.22424719773002424,
      "auroc": 1.0
    },
    "uncertainty": {
      "examples": 576,
      "positives": 240,
      "true_positives": 175,
      "false_positives": 179,
      "false_negatives": 65,
      "recall": 0.7291666666666666,
      "precision": 0.4943502824858757,
      "specificity": 0.46726190476190477,
      "escalation_rate": 0.6145833333333334,
      "brier": 0.2293827050503838,
      "ece": 0.07416748310606931,
      "auroc": 0.6601066468253968
    },
    "conflict": {
      "examples": 576,
      "positives": 144,
      "true_positives": 131,
      "false_positives": 173,
      "false_negatives": 13,
      "recall": 0.9097222222222222,
      "precision": 0.4309210526315789,
      "specificity": 0.5995370370370371,
      "escalation_rate": 0.5277777777777778,
      "brier": 0.12098456100215688,
      "ece": 0.21078960513805695,
      "auroc": 0.9134677211934157
    },
    "visual": {
      "examples": 576,
      "positives": 216,
      "true_positives": 169,
      "false_positives": 132,
      "false_negatives": 47,
      "recall": 0.7824074074074074,
      "precision": 0.5614617940199336,
      "specificity": 0.6333333333333333,
      "escalation_rate": 0.5225694444444444,
      "brier": 0.16616077941724472,
      "ece": 0.1971511931456856,
      "auroc": 0.8709362139917696
    }
  },
  "v4_robustness_slices": {
    "background_id": {
      "pink": {
        "examples": 192,
        "positives": 169,
        "true_positives": 157,
        "false_positives": 21,
        "false_negatives": 12,
        "recall": 0.9289940828402367,
        "precision": 0.8820224719101124,
        "specificity": 0.08695652173913043,
        "escalation_rate": 0.9270833333333334,
        "brier": 0.31135914332280196,
        "ece": 0.46788093565946537,
        "auroc": 0.8662207357859532
      },
      "reverb": {
        "examples": 192,
        "positives": 181,
        "true_positives": 177,
        "false_positives": 11,
        "false_negatives": 4,
        "recall": 0.9779005524861878,
        "precision": 0.9414893617021277,
        "specificity": 0.0,
        "escalation_rate": 0.9791666666666666,
        "brier": 0.16562298844441178,
        "ece": 0.3286566249272864,
        "auroc": 0.8679055750878956
      },
      "white": {
        "examples": 192,
        "positives": 170,
        "true_positives": 159,
        "false_positives": 22,
        "false_negatives": 11,
        "recall": 0.9352941176470588,
        "precision": 0.8784530386740331,
        "specificity": 0.0,
        "escalation_rate": 0.9427083333333334,
        "brier": 0.2863302628946927,
        "ece": 0.44435570647505257,
        "auroc": 0.8737967914438503
      }
    },
    "snr_db": {
      "0.0": {
        "examples": 144,
        "positives": 133,
        "true_positives": 133,
        "false_positives": 9,
        "false_negatives": 0,
        "recall": 1.0,
        "precision": 0.9366197183098591,
        "specificity": 0.18181818181818182,
        "escalation_rate": 0.9861111111111112,
        "brier": 0.15135230694589374,
        "ece": 0.33521018341449565,
        "auroc": 0.9958988380041012
      },
      "15.0": {
        "examples": 144,
        "positives": 130,
        "true_positives": 119,
        "false_positives": 14,
        "false_negatives": 11,
        "recall": 0.9153846153846154,
        "precision": 0.8947368421052632,
        "specificity": 0.0,
        "escalation_rate": 0.9236111111111112,
        "brier": 0.2976779995627198,
        "ece": 0.45929345723247444,
        "auroc": 0.7461538461538462
      },
      "30.0": {
        "examples": 144,
        "positives": 126,
        "true_positives": 110,
        "false_positives": 18,
        "false_negatives": 16,
        "recall": 0.873015873015873,
        "precision": 0.859375,
        "specificity": 0.0,
        "escalation_rate": 0.8888888888888888,
        "brier": 0.36197668627847096,
        "ece": 0.5134423018117535,
        "auroc": 0.8276014109347443
      },
      "5.0": {
        "examples": 144,
        "positives": 131,
        "true_positives": 131,
        "false_positives": 13,
        "false_negatives": 0,
        "recall": 1.0,
        "precision": 0.9097222222222222,
        "specificity": 0.0,
        "escalation_rate": 1.0,
        "brier": 0.20674286676212414,
        "ece": 0.3714515093453985,
        "auroc": 0.9301233118027011
      }
    },
    "blur_radius": {
      "0.0": {
        "examples": 192,
        "positives": 174,
        "true_positives": 165,
        "false_positives": 17,
        "false_negatives": 9,
        "recall": 0.9482758620689655,
        "precision": 0.9065934065934066,
        "specificity": 0.05555555555555555,
        "escalation_rate": 0.9479166666666666,
        "brier": 0.26204597095700016,
        "ece": 0.42548523065516897,
        "auroc": 0.8805874840357599
      },
      "1.5": {
        "examples": 192,
        "positives": 174,
        "true_positives": 167,
        "false_positives": 18,
        "false_negatives": 7,
        "recall": 0.9597701149425287,
        "precision": 0.9027027027027027,
        "specificity": 0.0,
        "escalation_rate": 0.9635416666666666,
        "brier": 0.23997474327744386,
        "ece": 0.41915464769013605,
        "auroc": 0.9613665389527458
      },
      "3.0": {
        "examples": 192,
        "positives": 172,
        "true_positives": 161,
        "false_positives": 19,
        "false_negatives": 11,
        "recall": 0.936046511627907,
        "precision": 0.8944444444444445,
        "specificity": 0.05,
        "escalation_rate": 0.9375,
        "brier": 0.26129168042746237,
        "ece": 0.40869253006803635,
        "auroc": 0.7604651162790698
      }
    },
    "occlusion_ratio": {
      "0.0": {
        "examples": 288,
        "positives": 261,
        "true_positives": 248,
        "false_positives": 26,
        "false_negatives": 13,
        "recall": 0.9501915708812261,
        "precision": 0.9051094890510949,
        "specificity": 0.037037037037037035,
        "escalation_rate": 0.9513888888888888,
        "brier": 0.24373840488323129,
        "ece": 0.41284212751138916,
        "auroc": 0.8809422449269193
      },
      "0.25": {
        "examples": 288,
        "positives": 259,
        "true_positives": 245,
        "false_positives": 28,
        "false_negatives": 14,
        "recall": 0.9459459459459459,
        "precision": 0.8974358974358975,
        "specificity": 0.034482758620689655,
        "escalation_rate": 0.9479166666666666,
        "brier": 0.265136524891373,
        "ece": 0.4240392733918322,
        "auroc": 0.8691252829183864
      }
    }
  },
  "limitations": "Routing quality only. Final answer quality must be measured by the separate end-to-end VLM/EvidenceAgent evaluation."
}