{
  "schema_version": 1,
  "physical_cases_per_system": 32,
  "independent_test_groups": 8,
  "latency_comparable_across_systems": false,
  "latency_note": "Observed wall latency is retained for execution audit only. ASR/OCR observations are memoized across systems in requested order, so later systems exclude analyzer work and their latency must not be compared with the first system.",
  "systems": {
    "always_vlm": {
      "cases": 32,
      "quality_pass_rate": 0.6875,
      "state_accuracy": 0.71875,
      "exact_state_accuracy": 0.6875,
      "answer_fact_coverage": 0.96875,
      "citation_validity": 0.875,
      "mean_answer_char_f1": 0.6598538614163614,
      "visual_backend_call_rate": 1.0,
      "text_answer_backend_call_rate": 1.0,
      "backend_calls_per_case": 2.0,
      "mean_observed_wall_latency_ms": 2576.251625374425,
      "backend_call_savings_vs_always": 0.0
    },
    "learned": {
      "cases": 32,
      "quality_pass_rate": 0.65625,
      "state_accuracy": 0.6875,
      "exact_state_accuracy": 0.6875,
      "answer_fact_coverage": 0.96875,
      "citation_validity": 0.84375,
      "mean_answer_char_f1": 0.6769566939286767,
      "visual_backend_call_rate": 0.90625,
      "text_answer_backend_call_rate": 1.0,
      "backend_calls_per_case": 1.90625,
      "mean_observed_wall_latency_ms": 12376.805822888855,
      "backend_call_savings_vs_always": 0.046875
    },
    "never_vlm": {
      "cases": 32,
      "quality_pass_rate": 0.78125,
      "state_accuracy": 0.8125,
      "exact_state_accuracy": 0.8125,
      "answer_fact_coverage": 0.96875,
      "citation_validity": 0.84375,
      "mean_answer_char_f1": 0.6423656732759995,
      "visual_backend_call_rate": 0.0,
      "text_answer_backend_call_rate": 1.0,
      "backend_calls_per_case": 1.0,
      "mean_observed_wall_latency_ms": 1510.641771601513,
      "backend_call_savings_vs_always": 0.5
    }
  }
}