| { |
| "schema_version": 1, |
| "physical_cases_per_system": 32, |
| "independent_test_groups": 8, |
| "latency_comparable_across_systems": false, |
| "latency_note": "Observed wall latency is retained for execution audit only. ASR/OCR observations are memoized across systems in requested order, so later systems exclude analyzer work and their latency must not be compared with the first system.", |
| "systems": { |
| "always_vlm": { |
| "cases": 32, |
| "quality_pass_rate": 0.6875, |
| "state_accuracy": 0.71875, |
| "exact_state_accuracy": 0.6875, |
| "answer_fact_coverage": 0.96875, |
| "citation_validity": 0.875, |
| "mean_answer_char_f1": 0.6598538614163614, |
| "visual_backend_call_rate": 1.0, |
| "text_answer_backend_call_rate": 1.0, |
| "backend_calls_per_case": 2.0, |
| "mean_observed_wall_latency_ms": 2576.251625374425, |
| "backend_call_savings_vs_always": 0.0 |
| }, |
| "learned": { |
| "cases": 32, |
| "quality_pass_rate": 0.65625, |
| "state_accuracy": 0.6875, |
| "exact_state_accuracy": 0.6875, |
| "answer_fact_coverage": 0.96875, |
| "citation_validity": 0.84375, |
| "mean_answer_char_f1": 0.6769566939286767, |
| "visual_backend_call_rate": 0.90625, |
| "text_answer_backend_call_rate": 1.0, |
| "backend_calls_per_case": 1.90625, |
| "mean_observed_wall_latency_ms": 12376.805822888855, |
| "backend_call_savings_vs_always": 0.046875 |
| }, |
| "never_vlm": { |
| "cases": 32, |
| "quality_pass_rate": 0.78125, |
| "state_accuracy": 0.8125, |
| "exact_state_accuracy": 0.8125, |
| "answer_fact_coverage": 0.96875, |
| "citation_validity": 0.84375, |
| "mean_answer_char_f1": 0.6423656732759995, |
| "visual_backend_call_rate": 0.0, |
| "text_answer_backend_call_rate": 1.0, |
| "backend_calls_per_case": 1.0, |
| "mean_observed_wall_latency_ms": 1510.641771601513, |
| "backend_call_savings_vs_always": 0.5 |
| } |
| } |
| } |
