{
  "analysis_sha256": "sha256:a7584f10c35ed6eb60056da52c65d5719de730e7e6568eee184db9f1864567fe",
  "attempted_once": true,
  "author": {
    "name": "Nobuyuki Horiuchi",
    "profile": "https://huggingface.co/horiuchinobuyuki"
  },
  "base": {
    "budget_accuracy": 0.7291666666666666,
    "budget_correct": 280,
    "completion_tokens": {
      "max": 8192,
      "mean": 3903.4895833333335,
      "p50": 2545.0,
      "p90": 8192.0,
      "total": 1498940
    },
    "finish_reason_counts": {
      "length": 98,
      "stop": 286
    },
    "format_ok": 343,
    "format_rate": 0.8932291666666666,
    "n": 384,
    "raw_accuracy": 0.7552083333333334,
    "raw_correct": 290
  },
  "candidate": {
    "budget_accuracy": 0.8958333333333334,
    "budget_correct": 344,
    "completion_tokens": {
      "max": 8192,
      "mean": 1891.1432291666667,
      "p50": 1050.0,
      "p90": 4987.099999999999,
      "total": 726199
    },
    "finish_reason_counts": {
      "length": 14,
      "stop": 370
    },
    "format_ok": 382,
    "format_rate": 0.9947916666666666,
    "n": 384,
    "raw_accuracy": 0.8984375,
    "raw_correct": 345
  },
  "candidate_frozen_before_access": true,
  "candidate_id": "g7-nes-d04-p",
  "dataset": {
    "eligibility_proofs": {
      "four_axis_disjoint_from_distill_train_and_randopt_search": true,
      "four_axis_disjoint_from_prior_prompt_pools": true,
      "historical_trace_identity_exclusion": true,
      "prior_sft2_holdout_rejected_as_historically_opened": true,
      "randopt_source_split_randopt_search": true,
      "reserved_before_training_and_search_windows": true,
      "training_manifest_held_out_test_used_false": true,
      "training_source_split_distill_train": true
    },
    "generator_version": "sft5-diverse-synthetic-v3",
    "name": "SFT5 genre-v3 one-shot release holdout",
    "prompt_hashes_sha256": "sha256:2b4c1c2d872604201a0dcc14a16f976a8fbf2add8c9958209ce464322e18b834",
    "quota": {
      "choice": 128,
      "difficulty_counts": {
        "easy": 107,
        "hard": 90,
        "medium": 187
      },
      "exact": 128,
      "family_counts": {
        "checksum_reasoning": 21,
        "inventory_state_tracking": 22,
        "japanese_checksum_reasoning": 21,
        "japanese_inventory_state_tracking": 22,
        "japanese_modular_dynamics": 21,
        "japanese_piecewise_cost": 21,
        "japanese_rule_filtering": 22,
        "japanese_stack_execution": 21,
        "japanese_symbolic_transduction": 21,
        "japanese_temporal_intersection": 22,
        "japanese_weighted_decision": 21,
        "modular_dynamics": 21,
        "piecewise_cost": 21,
        "rule_filtering": 22,
        "stack_execution": 21,
        "symbolic_transduction": 21,
        "temporal_intersection": 22,
        "weighted_decision": 21
      },
      "grader_counts": {
        "choice": 128,
        "exact": 128,
        "numeric": 128
      },
      "japanese": 192,
      "language_counts": {
        "en": 192,
        "ja": 192
      },
      "mcq_fraction": 0.3333333333333333,
      "numeric": 128,
      "task_counts": {
        "checksum_choice_en": 21,
        "checksum_choice_ja": 21,
        "interval_overlap_en": 22,
        "interval_overlap_ja": 22,
        "inventory_ledger_en": 22,
        "inventory_ledger_ja": 22,
        "modular_recurrence_en": 21,
        "modular_recurrence_ja": 21,
        "piecewise_tariff_en": 21,
        "piecewise_tariff_ja": 21,
        "rule_filter_choice_en": 22,
        "rule_filter_choice_ja": 22,
        "stack_machine_en": 21,
        "stack_machine_ja": 21,
        "symbolic_rewrite_en": 21,
        "symbolic_rewrite_ja": 21,
        "weighted_ranking_choice_en": 21,
        "weighted_ranking_choice_ja": 21
      },
      "total": 384
    },
    "records": 384,
    "selected_ids_sha256": "sha256:c87cf96566a31216257595b9bf32874f8bb21c91422b1d422bfd8fabc59a7ae7",
    "semantic_fingerprints_sha256": "sha256:3be2d319cf591e78d9455caa5e443936043638a186a8355593580279a108af12",
    "source_sha256": "sha256:6fea94d642c5fac144639be7d99a47c803eec8065e5b11cedb53e7acb680252b",
    "window_sha256": "sha256:095055618071b39e4112e5b66705f8e0baed9cb0519ce8ffa8580a66b85b516d"
  },
  "delta": {
    "budget_accuracy": 0.16666666666666674,
    "mean_completion_token_ratio": 0.4844750290205078,
    "mean_completion_token_reduction": 0.5155249709794922,
    "raw_accuracy": 0.14322916666666663
  },
  "eligibility_audit_sha256": "sha256:336e39619eb1778a3af516b8b08f8ceaa122ee26c58b7b35ff5232318759ebfc",
  "frozen_policy": {
    "anti_retuning": {
      "candidate_and_policy_immutable": true,
      "failure_action": "publish failure transparently or stop release; do not optimize against this holdout",
      "no_candidate_selection_or_hyperparameter_change_after_open": true,
      "report_all_rows_and_subgroups": true
    },
    "decode": {
      "enable_thinking": true,
      "max_tokens": 8192,
      "presence_penalty": 0.0,
      "seed": 2026081801,
      "temperature": 0.0,
      "top_k": -1,
      "top_p": 1.0,
      "trace_policy": "full upstream reasoning plus final content durably persisted; merged full trace graded"
    },
    "metrics": {
      "bootstrap": {
        "resamples": 10000,
        "role": "report_only",
        "seed": 2026081801
      },
      "budget_correct_definition": "raw correct && finish_reason != length",
      "grader": "qet.grading.grade_explicit_final_output over merged full trace",
      "primary": "budget_accuracy",
      "secondary": [
        "raw_accuracy",
        "format_rate",
        "finish_reason_counts",
        "completion_tokens_mean_p50_p90",
        "language_grader_task_family_difficulty_subgroups",
        "paired_correctness_transitions",
        "paired_bootstrap_95pct_ci"
      ]
    },
    "release_gate": {
      "all_required": true,
      "each_grader_budget_accuracy_delta_min": -0.05,
      "each_language_budget_accuracy_delta_min": -0.04,
      "interpretation": "pass only if every gate passes; disclose every result; no post-holdout retuning",
      "mean_completion_token_ratio_max": 0.7,
      "overall_budget_accuracy_delta_min": -0.02,
      "trace_integrity_and_384_matched_rows": true
    },
    "sha256": "sha256:8a929e01ec7f5d60a5b14ff6664f74ce4d86b216d26155ca613e371183ef5701"
  },
  "kind": "qet_g7_frozen_internal_release_gate_results",
  "measurement_interpretation": "The predeclared temperature-zero decode is valid for this frozen internal acceptance rule. It is not the temperature-1.0 measurement profile used for public benchmark claims and must not be compared as though it were.",
  "paired_bootstrap": {
    "budget_accuracy_delta_95pct_ci": [
      0.125,
      0.20833333333333334
    ],
    "mean_completion_token_ratio_95pct_ci": [
      0.4463466424570172,
      0.5239079217022228
    ],
    "resamples": 10000,
    "role": "report_only",
    "seed": 2026081801
  },
  "paired_transitions": {
    "budget": {
      "base_only_loss": 6,
      "both_correct": 274,
      "both_incorrect": 34,
      "candidate_only_gain": 70
    },
    "raw": {
      "base_only_loss": 5,
      "both_correct": 285,
      "both_incorrect": 34,
      "candidate_only_gain": 60
    }
  },
  "post_result_retuning": false,
  "public_model": {
    "name": "Qwick-3.5-9B",
    "repo_id": "horiuchinobuyuki/Qwick-3.5-9B",
    "url": "https://huggingface.co/horiuchinobuyuki/Qwick-3.5-9B"
  },
  "public_performance_benchmark": false,
  "redistribution": "aggregate metrics and identities only; prompts, answer keys, and full traces are not included",
  "release_gate_pass": true,
  "release_gates": {
    "each_grader_budget_accuracy_delta_ge_minus_0_05": true,
    "each_language_budget_accuracy_delta_ge_minus_0_04": true,
    "mean_completion_token_ratio_le_0_70": true,
    "overall_budget_accuracy_delta_ge_minus_0_02": true,
    "trace_integrity_and_384_matched_rows": true
  },
  "rerun": false,
  "role": "frozen internal candidate-acceptance release gate",
  "schema_version": 1,
  "status": "pass",
  "subgroups": {
    "difficulty": {
      "easy": {
        "base": {
          "budget_accuracy": 0.794392523364486,
          "budget_correct": 85,
          "completion_tokens": {
            "max": 8192,
            "mean": 3096.3644859813085,
            "p50": 1852.0,
            "p90": 8192.0,
            "total": 331311
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 88
          },
          "format_ok": 105,
          "format_rate": 0.9813084112149533,
          "n": 107,
          "raw_accuracy": 0.822429906542056,
          "raw_correct": 88
        },
        "candidate": {
          "budget_accuracy": 0.9252336448598131,
          "budget_correct": 99,
          "completion_tokens": {
            "max": 7655,
            "mean": 1082.196261682243,
            "p50": 730.0,
            "p90": 1571.8000000000002,
            "total": 115795
          },
          "finish_reason_counts": {
            "stop": 107
          },
          "format_ok": 107,
          "format_rate": 1.0,
          "n": 107,
          "raw_accuracy": 0.9252336448598131,
          "raw_correct": 99
        },
        "delta": {
          "budget_accuracy": 0.13084112149532712,
          "mean_completion_token_ratio": 0.34950544956249563,
          "raw_accuracy": 0.10280373831775702
        }
      },
      "hard": {
        "base": {
          "budget_accuracy": 0.6444444444444445,
          "budget_correct": 58,
          "completion_tokens": {
            "max": 8192,
            "mean": 4745.422222222222,
            "p50": 4627.5,
            "p90": 8192.0,
            "total": 427088
          },
          "finish_reason_counts": {
            "length": 31,
            "stop": 59
          },
          "format_ok": 73,
          "format_rate": 0.8111111111111111,
          "n": 90,
          "raw_accuracy": 0.6777777777777778,
          "raw_correct": 61
        },
        "candidate": {
          "budget_accuracy": 0.8333333333333334,
          "budget_correct": 75,
          "completion_tokens": {
            "max": 8192,
            "mean": 2927.9666666666667,
            "p50": 1595.5,
            "p90": 7849.400000000003,
            "total": 263517
          },
          "finish_reason_counts": {
            "length": 8,
            "stop": 82
          },
          "format_ok": 89,
          "format_rate": 0.9888888888888889,
          "n": 90,
          "raw_accuracy": 0.8333333333333334,
          "raw_correct": 75
        },
        "delta": {
          "budget_accuracy": 0.18888888888888888,
          "mean_completion_token_ratio": 0.6170086726857228,
          "raw_accuracy": 0.15555555555555556
        }
      },
      "medium": {
        "base": {
          "budget_accuracy": 0.732620320855615,
          "budget_correct": 137,
          "completion_tokens": {
            "max": 8192,
            "mean": 3960.1122994652405,
            "p50": 2820.0,
            "p90": 8192.0,
            "total": 740541
          },
          "finish_reason_counts": {
            "length": 48,
            "stop": 139
          },
          "format_ok": 165,
          "format_rate": 0.8823529411764706,
          "n": 187,
          "raw_accuracy": 0.7540106951871658,
          "raw_correct": 141
        },
        "candidate": {
          "budget_accuracy": 0.9090909090909091,
          "budget_correct": 170,
          "completion_tokens": {
            "max": 8192,
            "mean": 1855.0106951871658,
            "p50": 1152.0,
            "p90": 3469.2000000000016,
            "total": 346887
          },
          "finish_reason_counts": {
            "length": 6,
            "stop": 181
          },
          "format_ok": 186,
          "format_rate": 0.9946524064171123,
          "n": 187,
          "raw_accuracy": 0.9144385026737968,
          "raw_correct": 171
        },
        "delta": {
          "budget_accuracy": 0.17647058823529405,
          "mean_completion_token_ratio": 0.46842376046700995,
          "raw_accuracy": 0.160427807486631
        }
      }
    },
    "family": {
      "checksum_reasoning": {
        "base": {
          "budget_accuracy": 0.6666666666666666,
          "budget_correct": 14,
          "completion_tokens": {
            "max": 8192,
            "mean": 6019.333333333333,
            "p50": 6531.0,
            "p90": 8192.0,
            "total": 126406
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 14
          },
          "format_ok": 15,
          "format_rate": 0.7142857142857143,
          "n": 21,
          "raw_accuracy": 0.7142857142857143,
          "raw_correct": 15
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 4223,
            "mean": 2599.2380952380954,
            "p50": 2980.0,
            "p90": 3389.0,
            "total": 54584
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.33333333333333337,
          "mean_completion_token_ratio": 0.43181494549309374,
          "raw_accuracy": 0.2857142857142857
        }
      },
      "inventory_state_tracking": {
        "base": {
          "budget_accuracy": 0.9545454545454546,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 8192,
            "mean": 2203.909090909091,
            "p50": 1506.5,
            "p90": 4014.6000000000004,
            "total": 48486
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 0.9545454545454546,
          "n": 22,
          "raw_accuracy": 0.9545454545454546,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1720,
            "mean": 825.5454545454545,
            "p50": 699.5,
            "p90": 1282.4,
            "total": 18162
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.045454545454545414,
          "mean_completion_token_ratio": 0.3745823536691003,
          "raw_accuracy": 0.045454545454545414
        }
      },
      "japanese_checksum_reasoning": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 5677,
            "mean": 3506.5714285714284,
            "p50": 3495.0,
            "p90": 5114.0,
            "total": 73638
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 3544,
            "mean": 1774.3333333333333,
            "p50": 1648.0,
            "p90": 2809.0,
            "total": 37261
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5060023357505635,
          "raw_accuracy": 0.0
        }
      },
      "japanese_inventory_state_tracking": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 2791,
            "mean": 1189.8181818181818,
            "p50": 1084.0,
            "p90": 1929.5000000000007,
            "total": 26176
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1554,
            "mean": 785.5454545454545,
            "p50": 771.0,
            "p90": 1096.2,
            "total": 17282
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.6602231051344744,
          "raw_accuracy": 0.0
        }
      },
      "japanese_modular_dynamics": {
        "base": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 3452.0476190476193,
            "p50": 3171.0,
            "p90": 5387.0,
            "total": 72493
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 6960,
            "mean": 2382.809523809524,
            "p50": 1892.0,
            "p90": 4668.0,
            "total": 50039
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.6902597492171658,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "japanese_piecewise_cost": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 2393,
            "mean": 1676.904761904762,
            "p50": 1754.0,
            "p90": 2146.0,
            "total": 35215
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1164,
            "mean": 929.3333333333334,
            "p50": 970.0,
            "p90": 1109.0,
            "total": 19516
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5541956552605424,
          "raw_accuracy": 0.0
        }
      },
      "japanese_rule_filtering": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1757,
            "mean": 1193.0454545454545,
            "p50": 1175.0,
            "p90": 1434.7000000000003,
            "total": 26247
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 539,
            "mean": 417.5,
            "p50": 444.0,
            "p90": 457.7,
            "total": 9185
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.3499447555911152,
          "raw_accuracy": 0.0
        }
      },
      "japanese_stack_execution": {
        "base": {
          "budget_accuracy": 0.8571428571428571,
          "budget_correct": 18,
          "completion_tokens": {
            "max": 7759,
            "mean": 3738.095238095238,
            "p50": 3432.0,
            "p90": 5497.0,
            "total": 78500
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.8571428571428571,
          "raw_correct": 18
        },
        "candidate": {
          "budget_accuracy": 0.9047619047619048,
          "budget_correct": 19,
          "completion_tokens": {
            "max": 2265,
            "mean": 1380.857142857143,
            "p50": 1471.0,
            "p90": 1685.0,
            "total": 28998
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.9047619047619048,
          "raw_correct": 19
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.3694012738853503,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "japanese_symbolic_transduction": {
        "base": {
          "budget_accuracy": 0.5238095238095238,
          "budget_correct": 11,
          "completion_tokens": {
            "max": 8192,
            "mean": 5240.9047619047615,
            "p50": 5309.0,
            "p90": 8192.0,
            "total": 110059
          },
          "finish_reason_counts": {
            "length": 9,
            "stop": 12
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.5238095238095238,
          "raw_correct": 11
        },
        "candidate": {
          "budget_accuracy": 0.5714285714285714,
          "budget_correct": 12,
          "completion_tokens": {
            "max": 8192,
            "mean": 1825.6190476190477,
            "p50": 1285.0,
            "p90": 2885.0,
            "total": 38338
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.5714285714285714,
          "raw_correct": 12
        },
        "delta": {
          "budget_accuracy": 0.04761904761904756,
          "mean_completion_token_ratio": 0.34834043558455013,
          "raw_accuracy": 0.04761904761904756
        }
      },
      "japanese_temporal_intersection": {
        "base": {
          "budget_accuracy": 0.6363636363636364,
          "budget_correct": 14,
          "completion_tokens": {
            "max": 8192,
            "mean": 5000.5,
            "p50": 4310.5,
            "p90": 8192.0,
            "total": 110011
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 15
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.6363636363636364,
          "raw_correct": 14
        },
        "candidate": {
          "budget_accuracy": 0.9090909090909091,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 2350.2272727272725,
            "p50": 1612.0,
            "p90": 4764.200000000001,
            "total": 51705
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 21
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.9090909090909091,
          "raw_correct": 20
        },
        "delta": {
          "budget_accuracy": 0.2727272727272727,
          "mean_completion_token_ratio": 0.4699984546999845,
          "raw_accuracy": 0.2727272727272727
        }
      },
      "japanese_weighted_decision": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 4228,
            "mean": 1455.142857142857,
            "p50": 1345.0,
            "p90": 1434.0,
            "total": 30558
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1773,
            "mean": 861.5238095238095,
            "p50": 821.0,
            "p90": 885.0,
            "total": 18092
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5920544538255121,
          "raw_accuracy": 0.0
        }
      },
      "modular_dynamics": {
        "base": {
          "budget_accuracy": 0.23809523809523808,
          "budget_correct": 5,
          "completion_tokens": {
            "max": 8192,
            "mean": 7234.761904761905,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 151930
          },
          "finish_reason_counts": {
            "length": 16,
            "stop": 5
          },
          "format_ok": 9,
          "format_rate": 0.42857142857142855,
          "n": 21,
          "raw_accuracy": 0.42857142857142855,
          "raw_correct": 9
        },
        "candidate": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 3903.809523809524,
            "p50": 3424.0,
            "p90": 7788.0,
            "total": 81980
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "delta": {
          "budget_accuracy": 0.7142857142857142,
          "mean_completion_token_ratio": 0.539590600934641,
          "raw_accuracy": 0.5238095238095237
        }
      },
      "piecewise_cost": {
        "base": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 2132.0476190476193,
            "p50": 1593.0,
            "p90": 3281.0,
            "total": 44773
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1116,
            "mean": 701.5714285714286,
            "p50": 681.0,
            "p90": 880.0,
            "total": 14733
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.32905992450807403,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "rule_filtering": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1007,
            "mean": 805.3181818181819,
            "p50": 803.5,
            "p90": 947.5000000000001,
            "total": 17717
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 518,
            "mean": 417.3181818181818,
            "p50": 410.5,
            "p90": 505.6000000000001,
            "total": 9181
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5182028560139978,
          "raw_accuracy": 0.0
        }
      },
      "stack_execution": {
        "base": {
          "budget_accuracy": 0.09523809523809523,
          "budget_correct": 2,
          "completion_tokens": {
            "max": 8192,
            "mean": 7805.047619047619,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 163906
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 2
          },
          "format_ok": 15,
          "format_rate": 0.7142857142857143,
          "n": 21,
          "raw_accuracy": 0.2857142857142857,
          "raw_correct": 6
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 6896,
            "mean": 2170.1428571428573,
            "p50": 1617.0,
            "p90": 4988.0,
            "total": 45573
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.9047619047619048,
          "mean_completion_token_ratio": 0.27804351274511,
          "raw_accuracy": 0.7142857142857143
        }
      },
      "symbolic_transduction": {
        "base": {
          "budget_accuracy": 0.23809523809523808,
          "budget_correct": 5,
          "completion_tokens": {
            "max": 8192,
            "mean": 7272.333333333333,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 152719
          },
          "finish_reason_counts": {
            "length": 16,
            "stop": 5
          },
          "format_ok": 10,
          "format_rate": 0.47619047619047616,
          "n": 21,
          "raw_accuracy": 0.23809523809523808,
          "raw_correct": 5
        },
        "candidate": {
          "budget_accuracy": 0.6190476190476191,
          "budget_correct": 13,
          "completion_tokens": {
            "max": 8192,
            "mean": 3189.6190476190477,
            "p50": 1507.0,
            "p90": 8192.0,
            "total": 66982
          },
          "finish_reason_counts": {
            "length": 4,
            "stop": 17
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.6666666666666666,
          "raw_correct": 14
        },
        "delta": {
          "budget_accuracy": 0.380952380952381,
          "mean_completion_token_ratio": 0.43859637635133814,
          "raw_accuracy": 0.42857142857142855
        }
      },
      "temporal_intersection": {
        "base": {
          "budget_accuracy": 0.09090909090909091,
          "budget_correct": 2,
          "completion_tokens": {
            "max": 8192,
            "mean": 8035.909090909091,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 176790
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 3
          },
          "format_ok": 20,
          "format_rate": 0.9090909090909091,
          "n": 22,
          "raw_accuracy": 0.09090909090909091,
          "raw_correct": 2
        },
        "candidate": {
          "budget_accuracy": 0.18181818181818182,
          "budget_correct": 4,
          "completion_tokens": {
            "max": 8192,
            "mean": 6816.545454545455,
            "p50": 7679.0,
            "p90": 8192.0,
            "total": 149964
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 15
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.18181818181818182,
          "raw_correct": 4
        },
        "delta": {
          "budget_accuracy": 0.09090909090909091,
          "mean_completion_token_ratio": 0.8482606482267097,
          "raw_accuracy": 0.09090909090909091
        }
      },
      "weighted_decision": {
        "base": {
          "budget_accuracy": 0.9047619047619048,
          "budget_correct": 19,
          "completion_tokens": {
            "max": 8192,
            "mean": 2538.8571428571427,
            "p50": 1653.0,
            "p90": 5727.0,
            "total": 53316
          },
          "finish_reason_counts": {
            "length": 2,
            "stop": 19
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1050,
            "mean": 696.3809523809524,
            "p50": 659.0,
            "p90": 801.0,
            "total": 14624
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.09523809523809523,
          "mean_completion_token_ratio": 0.2742891439717909,
          "raw_accuracy": 0.04761904761904767
        }
      }
    },
    "grader": {
      "choice": {
        "base": {
          "budget_accuracy": 0.9296875,
          "budget_correct": 119,
          "completion_tokens": {
            "max": 8192,
            "mean": 2561.578125,
            "p50": 1455.0,
            "p90": 6424.6,
            "total": 327882
          },
          "finish_reason_counts": {
            "length": 9,
            "stop": 119
          },
          "format_ok": 121,
          "format_rate": 0.9453125,
          "n": 128,
          "raw_accuracy": 0.9453125,
          "raw_correct": 121
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 128,
          "completion_tokens": {
            "max": 4223,
            "mean": 1116.6171875,
            "p50": 745.5,
            "p90": 2911.2,
            "total": 142927
          },
          "finish_reason_counts": {
            "stop": 128
          },
          "format_ok": 128,
          "format_rate": 1.0,
          "n": 128,
          "raw_accuracy": 1.0,
          "raw_correct": 128
        },
        "delta": {
          "budget_accuracy": 0.0703125,
          "mean_completion_token_ratio": 0.43590987001421244,
          "raw_accuracy": 0.0546875
        }
      },
      "exact": {
        "base": {
          "budget_accuracy": 0.40625,
          "budget_correct": 52,
          "completion_tokens": {
            "max": 8192,
            "mean": 6187.3828125,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 791985
          },
          "finish_reason_counts": {
            "length": 70,
            "stop": 58
          },
          "format_ok": 109,
          "format_rate": 0.8515625,
          "n": 128,
          "raw_accuracy": 0.4375,
          "raw_correct": 56
        },
        "candidate": {
          "budget_accuracy": 0.6953125,
          "budget_correct": 89,
          "completion_tokens": {
            "max": 8192,
            "mean": 2980.9375,
            "p50": 1622.0,
            "p90": 8180.8,
            "total": 381560
          },
          "finish_reason_counts": {
            "length": 13,
            "stop": 115
          },
          "format_ok": 127,
          "format_rate": 0.9921875,
          "n": 128,
          "raw_accuracy": 0.703125,
          "raw_correct": 90
        },
        "delta": {
          "budget_accuracy": 0.2890625,
          "mean_completion_token_ratio": 0.48177680132830797,
          "raw_accuracy": 0.265625
        }
      },
      "numeric": {
        "base": {
          "budget_accuracy": 0.8515625,
          "budget_correct": 109,
          "completion_tokens": {
            "max": 8192,
            "mean": 2961.5078125,
            "p50": 1924.5,
            "p90": 8192.0,
            "total": 379073
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 109
          },
          "format_ok": 113,
          "format_rate": 0.8828125,
          "n": 128,
          "raw_accuracy": 0.8828125,
          "raw_correct": 113
        },
        "candidate": {
          "budget_accuracy": 0.9921875,
          "budget_correct": 127,
          "completion_tokens": {
            "max": 8192,
            "mean": 1575.875,
            "p50": 968.5,
            "p90": 3492.999999999999,
            "total": 201712
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 127
          },
          "format_ok": 127,
          "format_rate": 0.9921875,
          "n": 128,
          "raw_accuracy": 0.9921875,
          "raw_correct": 127
        },
        "delta": {
          "budget_accuracy": 0.140625,
          "mean_completion_token_ratio": 0.5321191432784714,
          "raw_accuracy": 0.109375
        }
      }
    },
    "language": {
      "en": {
        "base": {
          "budget_accuracy": 0.5729166666666666,
          "budget_correct": 110,
          "completion_tokens": {
            "max": 8192,
            "mean": 4875.223958333333,
            "p50": 5655.5,
            "p90": 8192.0,
            "total": 936043
          },
          "finish_reason_counts": {
            "length": 81,
            "stop": 111
          },
          "format_ok": 152,
          "format_rate": 0.7916666666666666,
          "n": 192,
          "raw_accuracy": 0.625,
          "raw_correct": 120
        },
        "candidate": {
          "budget_accuracy": 0.859375,
          "budget_correct": 165,
          "completion_tokens": {
            "max": 8192,
            "mean": 2373.8697916666665,
            "p50": 1065.0,
            "p90": 6953.6,
            "total": 455783
          },
          "finish_reason_counts": {
            "length": 12,
            "stop": 180
          },
          "format_ok": 190,
          "format_rate": 0.9895833333333334,
          "n": 192,
          "raw_accuracy": 0.8645833333333334,
          "raw_correct": 166
        },
        "delta": {
          "budget_accuracy": 0.28645833333333337,
          "mean_completion_token_ratio": 0.4869252801420448,
          "raw_accuracy": 0.23958333333333337
        }
      },
      "ja": {
        "base": {
          "budget_accuracy": 0.8854166666666666,
          "budget_correct": 170,
          "completion_tokens": {
            "max": 8192,
            "mean": 2931.7552083333335,
            "p50": 2246.5,
            "p90": 7086.500000000002,
            "total": 562897
          },
          "finish_reason_counts": {
            "length": 17,
            "stop": 175
          },
          "format_ok": 191,
          "format_rate": 0.9947916666666666,
          "n": 192,
          "raw_accuracy": 0.8854166666666666,
          "raw_correct": 170
        },
        "candidate": {
          "budget_accuracy": 0.9322916666666666,
          "budget_correct": 179,
          "completion_tokens": {
            "max": 8192,
            "mean": 1408.4166666666667,
            "p50": 1023.5,
            "p90": 2437.9,
            "total": 270416
          },
          "finish_reason_counts": {
            "length": 2,
            "stop": 190
          },
          "format_ok": 192,
          "format_rate": 1.0,
          "n": 192,
          "raw_accuracy": 0.9322916666666666,
          "raw_correct": 179
        },
        "delta": {
          "budget_accuracy": 0.046875,
          "mean_completion_token_ratio": 0.4804004995585338,
          "raw_accuracy": 0.046875
        }
      }
    },
    "task": {
      "checksum_choice_en": {
        "base": {
          "budget_accuracy": 0.6666666666666666,
          "budget_correct": 14,
          "completion_tokens": {
            "max": 8192,
            "mean": 6019.333333333333,
            "p50": 6531.0,
            "p90": 8192.0,
            "total": 126406
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 14
          },
          "format_ok": 15,
          "format_rate": 0.7142857142857143,
          "n": 21,
          "raw_accuracy": 0.7142857142857143,
          "raw_correct": 15
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 4223,
            "mean": 2599.2380952380954,
            "p50": 2980.0,
            "p90": 3389.0,
            "total": 54584
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.33333333333333337,
          "mean_completion_token_ratio": 0.43181494549309374,
          "raw_accuracy": 0.2857142857142857
        }
      },
      "checksum_choice_ja": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 5677,
            "mean": 3506.5714285714284,
            "p50": 3495.0,
            "p90": 5114.0,
            "total": 73638
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 3544,
            "mean": 1774.3333333333333,
            "p50": 1648.0,
            "p90": 2809.0,
            "total": 37261
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5060023357505635,
          "raw_accuracy": 0.0
        }
      },
      "interval_overlap_en": {
        "base": {
          "budget_accuracy": 0.09090909090909091,
          "budget_correct": 2,
          "completion_tokens": {
            "max": 8192,
            "mean": 8035.909090909091,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 176790
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 3
          },
          "format_ok": 20,
          "format_rate": 0.9090909090909091,
          "n": 22,
          "raw_accuracy": 0.09090909090909091,
          "raw_correct": 2
        },
        "candidate": {
          "budget_accuracy": 0.18181818181818182,
          "budget_correct": 4,
          "completion_tokens": {
            "max": 8192,
            "mean": 6816.545454545455,
            "p50": 7679.0,
            "p90": 8192.0,
            "total": 149964
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 15
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.18181818181818182,
          "raw_correct": 4
        },
        "delta": {
          "budget_accuracy": 0.09090909090909091,
          "mean_completion_token_ratio": 0.8482606482267097,
          "raw_accuracy": 0.09090909090909091
        }
      },
      "interval_overlap_ja": {
        "base": {
          "budget_accuracy": 0.6363636363636364,
          "budget_correct": 14,
          "completion_tokens": {
            "max": 8192,
            "mean": 5000.5,
            "p50": 4310.5,
            "p90": 8192.0,
            "total": 110011
          },
          "finish_reason_counts": {
            "length": 7,
            "stop": 15
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.6363636363636364,
          "raw_correct": 14
        },
        "candidate": {
          "budget_accuracy": 0.9090909090909091,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 2350.2272727272725,
            "p50": 1612.0,
            "p90": 4764.200000000001,
            "total": 51705
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 21
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 0.9090909090909091,
          "raw_correct": 20
        },
        "delta": {
          "budget_accuracy": 0.2727272727272727,
          "mean_completion_token_ratio": 0.4699984546999845,
          "raw_accuracy": 0.2727272727272727
        }
      },
      "inventory_ledger_en": {
        "base": {
          "budget_accuracy": 0.9545454545454546,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 8192,
            "mean": 2203.909090909091,
            "p50": 1506.5,
            "p90": 4014.6000000000004,
            "total": 48486
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 0.9545454545454546,
          "n": 22,
          "raw_accuracy": 0.9545454545454546,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1720,
            "mean": 825.5454545454545,
            "p50": 699.5,
            "p90": 1282.4,
            "total": 18162
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.045454545454545414,
          "mean_completion_token_ratio": 0.3745823536691003,
          "raw_accuracy": 0.045454545454545414
        }
      },
      "inventory_ledger_ja": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 2791,
            "mean": 1189.8181818181818,
            "p50": 1084.0,
            "p90": 1929.5000000000007,
            "total": 26176
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1554,
            "mean": 785.5454545454545,
            "p50": 771.0,
            "p90": 1096.2,
            "total": 17282
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.6602231051344744,
          "raw_accuracy": 0.0
        }
      },
      "modular_recurrence_en": {
        "base": {
          "budget_accuracy": 0.23809523809523808,
          "budget_correct": 5,
          "completion_tokens": {
            "max": 8192,
            "mean": 7234.761904761905,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 151930
          },
          "finish_reason_counts": {
            "length": 16,
            "stop": 5
          },
          "format_ok": 9,
          "format_rate": 0.42857142857142855,
          "n": 21,
          "raw_accuracy": 0.42857142857142855,
          "raw_correct": 9
        },
        "candidate": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 3903.809523809524,
            "p50": 3424.0,
            "p90": 7788.0,
            "total": 81980
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "delta": {
          "budget_accuracy": 0.7142857142857142,
          "mean_completion_token_ratio": 0.539590600934641,
          "raw_accuracy": 0.5238095238095237
        }
      },
      "modular_recurrence_ja": {
        "base": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 3452.0476190476193,
            "p50": 3171.0,
            "p90": 5387.0,
            "total": 72493
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 6960,
            "mean": 2382.809523809524,
            "p50": 1892.0,
            "p90": 4668.0,
            "total": 50039
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.6902597492171658,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "piecewise_tariff_en": {
        "base": {
          "budget_accuracy": 0.9523809523809523,
          "budget_correct": 20,
          "completion_tokens": {
            "max": 8192,
            "mean": 2132.0476190476193,
            "p50": 1593.0,
            "p90": 3281.0,
            "total": 44773
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1116,
            "mean": 701.5714285714286,
            "p50": 681.0,
            "p90": 880.0,
            "total": 14733
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.32905992450807403,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "piecewise_tariff_ja": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 2393,
            "mean": 1676.904761904762,
            "p50": 1754.0,
            "p90": 2146.0,
            "total": 35215
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1164,
            "mean": 929.3333333333334,
            "p50": 970.0,
            "p90": 1109.0,
            "total": 19516
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5541956552605424,
          "raw_accuracy": 0.0
        }
      },
      "rule_filter_choice_en": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1007,
            "mean": 805.3181818181819,
            "p50": 803.5,
            "p90": 947.5000000000001,
            "total": 17717
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 518,
            "mean": 417.3181818181818,
            "p50": 410.5,
            "p90": 505.6000000000001,
            "total": 9181
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5182028560139978,
          "raw_accuracy": 0.0
        }
      },
      "rule_filter_choice_ja": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 1757,
            "mean": 1193.0454545454545,
            "p50": 1175.0,
            "p90": 1434.7000000000003,
            "total": 26247
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 22,
          "completion_tokens": {
            "max": 539,
            "mean": 417.5,
            "p50": 444.0,
            "p90": 457.7,
            "total": 9185
          },
          "finish_reason_counts": {
            "stop": 22
          },
          "format_ok": 22,
          "format_rate": 1.0,
          "n": 22,
          "raw_accuracy": 1.0,
          "raw_correct": 22
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.3499447555911152,
          "raw_accuracy": 0.0
        }
      },
      "stack_machine_en": {
        "base": {
          "budget_accuracy": 0.09523809523809523,
          "budget_correct": 2,
          "completion_tokens": {
            "max": 8192,
            "mean": 7805.047619047619,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 163906
          },
          "finish_reason_counts": {
            "length": 19,
            "stop": 2
          },
          "format_ok": 15,
          "format_rate": 0.7142857142857143,
          "n": 21,
          "raw_accuracy": 0.2857142857142857,
          "raw_correct": 6
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 6896,
            "mean": 2170.1428571428573,
            "p50": 1617.0,
            "p90": 4988.0,
            "total": 45573
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.9047619047619048,
          "mean_completion_token_ratio": 0.27804351274511,
          "raw_accuracy": 0.7142857142857143
        }
      },
      "stack_machine_ja": {
        "base": {
          "budget_accuracy": 0.8571428571428571,
          "budget_correct": 18,
          "completion_tokens": {
            "max": 7759,
            "mean": 3738.095238095238,
            "p50": 3432.0,
            "p90": 5497.0,
            "total": 78500
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.8571428571428571,
          "raw_correct": 18
        },
        "candidate": {
          "budget_accuracy": 0.9047619047619048,
          "budget_correct": 19,
          "completion_tokens": {
            "max": 2265,
            "mean": 1380.857142857143,
            "p50": 1471.0,
            "p90": 1685.0,
            "total": 28998
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.9047619047619048,
          "raw_correct": 19
        },
        "delta": {
          "budget_accuracy": 0.04761904761904767,
          "mean_completion_token_ratio": 0.3694012738853503,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "symbolic_rewrite_en": {
        "base": {
          "budget_accuracy": 0.23809523809523808,
          "budget_correct": 5,
          "completion_tokens": {
            "max": 8192,
            "mean": 7272.333333333333,
            "p50": 8192.0,
            "p90": 8192.0,
            "total": 152719
          },
          "finish_reason_counts": {
            "length": 16,
            "stop": 5
          },
          "format_ok": 10,
          "format_rate": 0.47619047619047616,
          "n": 21,
          "raw_accuracy": 0.23809523809523808,
          "raw_correct": 5
        },
        "candidate": {
          "budget_accuracy": 0.6190476190476191,
          "budget_correct": 13,
          "completion_tokens": {
            "max": 8192,
            "mean": 3189.6190476190477,
            "p50": 1507.0,
            "p90": 8192.0,
            "total": 66982
          },
          "finish_reason_counts": {
            "length": 4,
            "stop": 17
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.6666666666666666,
          "raw_correct": 14
        },
        "delta": {
          "budget_accuracy": 0.380952380952381,
          "mean_completion_token_ratio": 0.43859637635133814,
          "raw_accuracy": 0.42857142857142855
        }
      },
      "symbolic_rewrite_ja": {
        "base": {
          "budget_accuracy": 0.5238095238095238,
          "budget_correct": 11,
          "completion_tokens": {
            "max": 8192,
            "mean": 5240.9047619047615,
            "p50": 5309.0,
            "p90": 8192.0,
            "total": 110059
          },
          "finish_reason_counts": {
            "length": 9,
            "stop": 12
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.5238095238095238,
          "raw_correct": 11
        },
        "candidate": {
          "budget_accuracy": 0.5714285714285714,
          "budget_correct": 12,
          "completion_tokens": {
            "max": 8192,
            "mean": 1825.6190476190477,
            "p50": 1285.0,
            "p90": 2885.0,
            "total": 38338
          },
          "finish_reason_counts": {
            "length": 1,
            "stop": 20
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 0.5714285714285714,
          "raw_correct": 12
        },
        "delta": {
          "budget_accuracy": 0.04761904761904756,
          "mean_completion_token_ratio": 0.34834043558455013,
          "raw_accuracy": 0.04761904761904756
        }
      },
      "weighted_ranking_choice_en": {
        "base": {
          "budget_accuracy": 0.9047619047619048,
          "budget_correct": 19,
          "completion_tokens": {
            "max": 8192,
            "mean": 2538.8571428571427,
            "p50": 1653.0,
            "p90": 5727.0,
            "total": 53316
          },
          "finish_reason_counts": {
            "length": 2,
            "stop": 19
          },
          "format_ok": 20,
          "format_rate": 0.9523809523809523,
          "n": 21,
          "raw_accuracy": 0.9523809523809523,
          "raw_correct": 20
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1050,
            "mean": 696.3809523809524,
            "p50": 659.0,
            "p90": 801.0,
            "total": 14624
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.09523809523809523,
          "mean_completion_token_ratio": 0.2742891439717909,
          "raw_accuracy": 0.04761904761904767
        }
      },
      "weighted_ranking_choice_ja": {
        "base": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 4228,
            "mean": 1455.142857142857,
            "p50": 1345.0,
            "p90": 1434.0,
            "total": 30558
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "candidate": {
          "budget_accuracy": 1.0,
          "budget_correct": 21,
          "completion_tokens": {
            "max": 1773,
            "mean": 861.5238095238095,
            "p50": 821.0,
            "p90": 885.0,
            "total": 18092
          },
          "finish_reason_counts": {
            "stop": 21
          },
          "format_ok": 21,
          "format_rate": 1.0,
          "n": 21,
          "raw_accuracy": 1.0,
          "raw_correct": 21
        },
        "delta": {
          "budget_accuracy": 0.0,
          "mean_completion_token_ratio": 0.5920544538255121,
          "raw_accuracy": 0.0
        }
      }
    }
  },
  "trace_integrity": {
    "base_raw_proxy_traces_sha256": "sha256:a026f73a07b184cf7fe11739f16d67a6ec3a4b95c7743b30ee3abe2d2018b1cc",
    "base_samples_sha256": "sha256:c1927c0f5f5f0c8369de97bbe02ae3c5427e715c97cdbfc595d4e5712f0fcba9",
    "candidate_raw_proxy_traces_sha256": "sha256:dd7a5f3952ac8c753596603e401a8720bad7be63f72ce4701e949e0f9143f5e8",
    "candidate_samples_sha256": "sha256:cae4e7b155372b0b26ce5d042e3453a0f51b136f1097913feb1097304ee4c00a",
    "full_trace_merge": true,
    "matched_rows_per_model": 384
  },
  "used_for_training_or_selection": false
}