| { |
| "analysis_sha256": "sha256:a7584f10c35ed6eb60056da52c65d5719de730e7e6568eee184db9f1864567fe", |
| "attempted_once": true, |
| "author": { |
| "name": "Nobuyuki Horiuchi", |
| "profile": "https://huggingface.co/horiuchinobuyuki" |
| }, |
| "base": { |
| "budget_accuracy": 0.7291666666666666, |
| "budget_correct": 280, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3903.4895833333335, |
| "p50": 2545.0, |
| "p90": 8192.0, |
| "total": 1498940 |
| }, |
| "finish_reason_counts": { |
| "length": 98, |
| "stop": 286 |
| }, |
| "format_ok": 343, |
| "format_rate": 0.8932291666666666, |
| "n": 384, |
| "raw_accuracy": 0.7552083333333334, |
| "raw_correct": 290 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.8958333333333334, |
| "budget_correct": 344, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1891.1432291666667, |
| "p50": 1050.0, |
| "p90": 4987.099999999999, |
| "total": 726199 |
| }, |
| "finish_reason_counts": { |
| "length": 14, |
| "stop": 370 |
| }, |
| "format_ok": 382, |
| "format_rate": 0.9947916666666666, |
| "n": 384, |
| "raw_accuracy": 0.8984375, |
| "raw_correct": 345 |
| }, |
| "candidate_frozen_before_access": true, |
| "candidate_id": "g7-nes-d04-p", |
| "dataset": { |
| "eligibility_proofs": { |
| "four_axis_disjoint_from_distill_train_and_randopt_search": true, |
| "four_axis_disjoint_from_prior_prompt_pools": true, |
| "historical_trace_identity_exclusion": true, |
| "prior_sft2_holdout_rejected_as_historically_opened": true, |
| "randopt_source_split_randopt_search": true, |
| "reserved_before_training_and_search_windows": true, |
| "training_manifest_held_out_test_used_false": true, |
| "training_source_split_distill_train": true |
| }, |
| "generator_version": "sft5-diverse-synthetic-v3", |
| "name": "SFT5 genre-v3 one-shot release holdout", |
| "prompt_hashes_sha256": "sha256:2b4c1c2d872604201a0dcc14a16f976a8fbf2add8c9958209ce464322e18b834", |
| "quota": { |
| "choice": 128, |
| "difficulty_counts": { |
| "easy": 107, |
| "hard": 90, |
| "medium": 187 |
| }, |
| "exact": 128, |
| "family_counts": { |
| "checksum_reasoning": 21, |
| "inventory_state_tracking": 22, |
| "japanese_checksum_reasoning": 21, |
| "japanese_inventory_state_tracking": 22, |
| "japanese_modular_dynamics": 21, |
| "japanese_piecewise_cost": 21, |
| "japanese_rule_filtering": 22, |
| "japanese_stack_execution": 21, |
| "japanese_symbolic_transduction": 21, |
| "japanese_temporal_intersection": 22, |
| "japanese_weighted_decision": 21, |
| "modular_dynamics": 21, |
| "piecewise_cost": 21, |
| "rule_filtering": 22, |
| "stack_execution": 21, |
| "symbolic_transduction": 21, |
| "temporal_intersection": 22, |
| "weighted_decision": 21 |
| }, |
| "grader_counts": { |
| "choice": 128, |
| "exact": 128, |
| "numeric": 128 |
| }, |
| "japanese": 192, |
| "language_counts": { |
| "en": 192, |
| "ja": 192 |
| }, |
| "mcq_fraction": 0.3333333333333333, |
| "numeric": 128, |
| "task_counts": { |
| "checksum_choice_en": 21, |
| "checksum_choice_ja": 21, |
| "interval_overlap_en": 22, |
| "interval_overlap_ja": 22, |
| "inventory_ledger_en": 22, |
| "inventory_ledger_ja": 22, |
| "modular_recurrence_en": 21, |
| "modular_recurrence_ja": 21, |
| "piecewise_tariff_en": 21, |
| "piecewise_tariff_ja": 21, |
| "rule_filter_choice_en": 22, |
| "rule_filter_choice_ja": 22, |
| "stack_machine_en": 21, |
| "stack_machine_ja": 21, |
| "symbolic_rewrite_en": 21, |
| "symbolic_rewrite_ja": 21, |
| "weighted_ranking_choice_en": 21, |
| "weighted_ranking_choice_ja": 21 |
| }, |
| "total": 384 |
| }, |
| "records": 384, |
| "selected_ids_sha256": "sha256:c87cf96566a31216257595b9bf32874f8bb21c91422b1d422bfd8fabc59a7ae7", |
| "semantic_fingerprints_sha256": "sha256:3be2d319cf591e78d9455caa5e443936043638a186a8355593580279a108af12", |
| "source_sha256": "sha256:6fea94d642c5fac144639be7d99a47c803eec8065e5b11cedb53e7acb680252b", |
| "window_sha256": "sha256:095055618071b39e4112e5b66705f8e0baed9cb0519ce8ffa8580a66b85b516d" |
| }, |
| "delta": { |
| "budget_accuracy": 0.16666666666666674, |
| "mean_completion_token_ratio": 0.4844750290205078, |
| "mean_completion_token_reduction": 0.5155249709794922, |
| "raw_accuracy": 0.14322916666666663 |
| }, |
| "eligibility_audit_sha256": "sha256:336e39619eb1778a3af516b8b08f8ceaa122ee26c58b7b35ff5232318759ebfc", |
| "frozen_policy": { |
| "anti_retuning": { |
| "candidate_and_policy_immutable": true, |
| "failure_action": "publish failure transparently or stop release; do not optimize against this holdout", |
| "no_candidate_selection_or_hyperparameter_change_after_open": true, |
| "report_all_rows_and_subgroups": true |
| }, |
| "decode": { |
| "enable_thinking": true, |
| "max_tokens": 8192, |
| "presence_penalty": 0.0, |
| "seed": 2026081801, |
| "temperature": 0.0, |
| "top_k": -1, |
| "top_p": 1.0, |
| "trace_policy": "full upstream reasoning plus final content durably persisted; merged full trace graded" |
| }, |
| "metrics": { |
| "bootstrap": { |
| "resamples": 10000, |
| "role": "report_only", |
| "seed": 2026081801 |
| }, |
| "budget_correct_definition": "raw correct && finish_reason != length", |
| "grader": "qet.grading.grade_explicit_final_output over merged full trace", |
| "primary": "budget_accuracy", |
| "secondary": [ |
| "raw_accuracy", |
| "format_rate", |
| "finish_reason_counts", |
| "completion_tokens_mean_p50_p90", |
| "language_grader_task_family_difficulty_subgroups", |
| "paired_correctness_transitions", |
| "paired_bootstrap_95pct_ci" |
| ] |
| }, |
| "release_gate": { |
| "all_required": true, |
| "each_grader_budget_accuracy_delta_min": -0.05, |
| "each_language_budget_accuracy_delta_min": -0.04, |
| "interpretation": "pass only if every gate passes; disclose every result; no post-holdout retuning", |
| "mean_completion_token_ratio_max": 0.7, |
| "overall_budget_accuracy_delta_min": -0.02, |
| "trace_integrity_and_384_matched_rows": true |
| }, |
| "sha256": "sha256:8a929e01ec7f5d60a5b14ff6664f74ce4d86b216d26155ca613e371183ef5701" |
| }, |
| "kind": "qet_g7_frozen_internal_release_gate_results", |
| "measurement_interpretation": "The predeclared temperature-zero decode is valid for this frozen internal acceptance rule. It is not the temperature-1.0 measurement profile used for public benchmark claims and must not be compared as though it were.", |
| "paired_bootstrap": { |
| "budget_accuracy_delta_95pct_ci": [ |
| 0.125, |
| 0.20833333333333334 |
| ], |
| "mean_completion_token_ratio_95pct_ci": [ |
| 0.4463466424570172, |
| 0.5239079217022228 |
| ], |
| "resamples": 10000, |
| "role": "report_only", |
| "seed": 2026081801 |
| }, |
| "paired_transitions": { |
| "budget": { |
| "base_only_loss": 6, |
| "both_correct": 274, |
| "both_incorrect": 34, |
| "candidate_only_gain": 70 |
| }, |
| "raw": { |
| "base_only_loss": 5, |
| "both_correct": 285, |
| "both_incorrect": 34, |
| "candidate_only_gain": 60 |
| } |
| }, |
| "post_result_retuning": false, |
| "public_model": { |
| "name": "Qwick-3.5-9B", |
| "repo_id": "horiuchinobuyuki/Qwick-3.5-9B", |
| "url": "https://huggingface.co/horiuchinobuyuki/Qwick-3.5-9B" |
| }, |
| "public_performance_benchmark": false, |
| "redistribution": "aggregate metrics and identities only; prompts, answer keys, and full traces are not included", |
| "release_gate_pass": true, |
| "release_gates": { |
| "each_grader_budget_accuracy_delta_ge_minus_0_05": true, |
| "each_language_budget_accuracy_delta_ge_minus_0_04": true, |
| "mean_completion_token_ratio_le_0_70": true, |
| "overall_budget_accuracy_delta_ge_minus_0_02": true, |
| "trace_integrity_and_384_matched_rows": true |
| }, |
| "rerun": false, |
| "role": "frozen internal candidate-acceptance release gate", |
| "schema_version": 1, |
| "status": "pass", |
| "subgroups": { |
| "difficulty": { |
| "easy": { |
| "base": { |
| "budget_accuracy": 0.794392523364486, |
| "budget_correct": 85, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3096.3644859813085, |
| "p50": 1852.0, |
| "p90": 8192.0, |
| "total": 331311 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 88 |
| }, |
| "format_ok": 105, |
| "format_rate": 0.9813084112149533, |
| "n": 107, |
| "raw_accuracy": 0.822429906542056, |
| "raw_correct": 88 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9252336448598131, |
| "budget_correct": 99, |
| "completion_tokens": { |
| "max": 7655, |
| "mean": 1082.196261682243, |
| "p50": 730.0, |
| "p90": 1571.8000000000002, |
| "total": 115795 |
| }, |
| "finish_reason_counts": { |
| "stop": 107 |
| }, |
| "format_ok": 107, |
| "format_rate": 1.0, |
| "n": 107, |
| "raw_accuracy": 0.9252336448598131, |
| "raw_correct": 99 |
| }, |
| "delta": { |
| "budget_accuracy": 0.13084112149532712, |
| "mean_completion_token_ratio": 0.34950544956249563, |
| "raw_accuracy": 0.10280373831775702 |
| } |
| }, |
| "hard": { |
| "base": { |
| "budget_accuracy": 0.6444444444444445, |
| "budget_correct": 58, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 4745.422222222222, |
| "p50": 4627.5, |
| "p90": 8192.0, |
| "total": 427088 |
| }, |
| "finish_reason_counts": { |
| "length": 31, |
| "stop": 59 |
| }, |
| "format_ok": 73, |
| "format_rate": 0.8111111111111111, |
| "n": 90, |
| "raw_accuracy": 0.6777777777777778, |
| "raw_correct": 61 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.8333333333333334, |
| "budget_correct": 75, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2927.9666666666667, |
| "p50": 1595.5, |
| "p90": 7849.400000000003, |
| "total": 263517 |
| }, |
| "finish_reason_counts": { |
| "length": 8, |
| "stop": 82 |
| }, |
| "format_ok": 89, |
| "format_rate": 0.9888888888888889, |
| "n": 90, |
| "raw_accuracy": 0.8333333333333334, |
| "raw_correct": 75 |
| }, |
| "delta": { |
| "budget_accuracy": 0.18888888888888888, |
| "mean_completion_token_ratio": 0.6170086726857228, |
| "raw_accuracy": 0.15555555555555556 |
| } |
| }, |
| "medium": { |
| "base": { |
| "budget_accuracy": 0.732620320855615, |
| "budget_correct": 137, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3960.1122994652405, |
| "p50": 2820.0, |
| "p90": 8192.0, |
| "total": 740541 |
| }, |
| "finish_reason_counts": { |
| "length": 48, |
| "stop": 139 |
| }, |
| "format_ok": 165, |
| "format_rate": 0.8823529411764706, |
| "n": 187, |
| "raw_accuracy": 0.7540106951871658, |
| "raw_correct": 141 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9090909090909091, |
| "budget_correct": 170, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1855.0106951871658, |
| "p50": 1152.0, |
| "p90": 3469.2000000000016, |
| "total": 346887 |
| }, |
| "finish_reason_counts": { |
| "length": 6, |
| "stop": 181 |
| }, |
| "format_ok": 186, |
| "format_rate": 0.9946524064171123, |
| "n": 187, |
| "raw_accuracy": 0.9144385026737968, |
| "raw_correct": 171 |
| }, |
| "delta": { |
| "budget_accuracy": 0.17647058823529405, |
| "mean_completion_token_ratio": 0.46842376046700995, |
| "raw_accuracy": 0.160427807486631 |
| } |
| } |
| }, |
| "family": { |
| "checksum_reasoning": { |
| "base": { |
| "budget_accuracy": 0.6666666666666666, |
| "budget_correct": 14, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 6019.333333333333, |
| "p50": 6531.0, |
| "p90": 8192.0, |
| "total": 126406 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 14 |
| }, |
| "format_ok": 15, |
| "format_rate": 0.7142857142857143, |
| "n": 21, |
| "raw_accuracy": 0.7142857142857143, |
| "raw_correct": 15 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 4223, |
| "mean": 2599.2380952380954, |
| "p50": 2980.0, |
| "p90": 3389.0, |
| "total": 54584 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.33333333333333337, |
| "mean_completion_token_ratio": 0.43181494549309374, |
| "raw_accuracy": 0.2857142857142857 |
| } |
| }, |
| "inventory_state_tracking": { |
| "base": { |
| "budget_accuracy": 0.9545454545454546, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2203.909090909091, |
| "p50": 1506.5, |
| "p90": 4014.6000000000004, |
| "total": 48486 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 0.9545454545454546, |
| "n": 22, |
| "raw_accuracy": 0.9545454545454546, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1720, |
| "mean": 825.5454545454545, |
| "p50": 699.5, |
| "p90": 1282.4, |
| "total": 18162 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.045454545454545414, |
| "mean_completion_token_ratio": 0.3745823536691003, |
| "raw_accuracy": 0.045454545454545414 |
| } |
| }, |
| "japanese_checksum_reasoning": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 5677, |
| "mean": 3506.5714285714284, |
| "p50": 3495.0, |
| "p90": 5114.0, |
| "total": 73638 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 3544, |
| "mean": 1774.3333333333333, |
| "p50": 1648.0, |
| "p90": 2809.0, |
| "total": 37261 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5060023357505635, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "japanese_inventory_state_tracking": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 2791, |
| "mean": 1189.8181818181818, |
| "p50": 1084.0, |
| "p90": 1929.5000000000007, |
| "total": 26176 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1554, |
| "mean": 785.5454545454545, |
| "p50": 771.0, |
| "p90": 1096.2, |
| "total": 17282 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.6602231051344744, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "japanese_modular_dynamics": { |
| "base": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3452.0476190476193, |
| "p50": 3171.0, |
| "p90": 5387.0, |
| "total": 72493 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 6960, |
| "mean": 2382.809523809524, |
| "p50": 1892.0, |
| "p90": 4668.0, |
| "total": 50039 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.6902597492171658, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "japanese_piecewise_cost": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 2393, |
| "mean": 1676.904761904762, |
| "p50": 1754.0, |
| "p90": 2146.0, |
| "total": 35215 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1164, |
| "mean": 929.3333333333334, |
| "p50": 970.0, |
| "p90": 1109.0, |
| "total": 19516 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5541956552605424, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "japanese_rule_filtering": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1757, |
| "mean": 1193.0454545454545, |
| "p50": 1175.0, |
| "p90": 1434.7000000000003, |
| "total": 26247 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 539, |
| "mean": 417.5, |
| "p50": 444.0, |
| "p90": 457.7, |
| "total": 9185 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.3499447555911152, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "japanese_stack_execution": { |
| "base": { |
| "budget_accuracy": 0.8571428571428571, |
| "budget_correct": 18, |
| "completion_tokens": { |
| "max": 7759, |
| "mean": 3738.095238095238, |
| "p50": 3432.0, |
| "p90": 5497.0, |
| "total": 78500 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.8571428571428571, |
| "raw_correct": 18 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9047619047619048, |
| "budget_correct": 19, |
| "completion_tokens": { |
| "max": 2265, |
| "mean": 1380.857142857143, |
| "p50": 1471.0, |
| "p90": 1685.0, |
| "total": 28998 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.9047619047619048, |
| "raw_correct": 19 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.3694012738853503, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "japanese_symbolic_transduction": { |
| "base": { |
| "budget_accuracy": 0.5238095238095238, |
| "budget_correct": 11, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 5240.9047619047615, |
| "p50": 5309.0, |
| "p90": 8192.0, |
| "total": 110059 |
| }, |
| "finish_reason_counts": { |
| "length": 9, |
| "stop": 12 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.5238095238095238, |
| "raw_correct": 11 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.5714285714285714, |
| "budget_correct": 12, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1825.6190476190477, |
| "p50": 1285.0, |
| "p90": 2885.0, |
| "total": 38338 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.5714285714285714, |
| "raw_correct": 12 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904756, |
| "mean_completion_token_ratio": 0.34834043558455013, |
| "raw_accuracy": 0.04761904761904756 |
| } |
| }, |
| "japanese_temporal_intersection": { |
| "base": { |
| "budget_accuracy": 0.6363636363636364, |
| "budget_correct": 14, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 5000.5, |
| "p50": 4310.5, |
| "p90": 8192.0, |
| "total": 110011 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 15 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.6363636363636364, |
| "raw_correct": 14 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9090909090909091, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2350.2272727272725, |
| "p50": 1612.0, |
| "p90": 4764.200000000001, |
| "total": 51705 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 21 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.9090909090909091, |
| "raw_correct": 20 |
| }, |
| "delta": { |
| "budget_accuracy": 0.2727272727272727, |
| "mean_completion_token_ratio": 0.4699984546999845, |
| "raw_accuracy": 0.2727272727272727 |
| } |
| }, |
| "japanese_weighted_decision": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 4228, |
| "mean": 1455.142857142857, |
| "p50": 1345.0, |
| "p90": 1434.0, |
| "total": 30558 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1773, |
| "mean": 861.5238095238095, |
| "p50": 821.0, |
| "p90": 885.0, |
| "total": 18092 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5920544538255121, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "modular_dynamics": { |
| "base": { |
| "budget_accuracy": 0.23809523809523808, |
| "budget_correct": 5, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7234.761904761905, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 151930 |
| }, |
| "finish_reason_counts": { |
| "length": 16, |
| "stop": 5 |
| }, |
| "format_ok": 9, |
| "format_rate": 0.42857142857142855, |
| "n": 21, |
| "raw_accuracy": 0.42857142857142855, |
| "raw_correct": 9 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3903.809523809524, |
| "p50": 3424.0, |
| "p90": 7788.0, |
| "total": 81980 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "delta": { |
| "budget_accuracy": 0.7142857142857142, |
| "mean_completion_token_ratio": 0.539590600934641, |
| "raw_accuracy": 0.5238095238095237 |
| } |
| }, |
| "piecewise_cost": { |
| "base": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2132.0476190476193, |
| "p50": 1593.0, |
| "p90": 3281.0, |
| "total": 44773 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1116, |
| "mean": 701.5714285714286, |
| "p50": 681.0, |
| "p90": 880.0, |
| "total": 14733 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.32905992450807403, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "rule_filtering": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1007, |
| "mean": 805.3181818181819, |
| "p50": 803.5, |
| "p90": 947.5000000000001, |
| "total": 17717 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 518, |
| "mean": 417.3181818181818, |
| "p50": 410.5, |
| "p90": 505.6000000000001, |
| "total": 9181 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5182028560139978, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "stack_execution": { |
| "base": { |
| "budget_accuracy": 0.09523809523809523, |
| "budget_correct": 2, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7805.047619047619, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 163906 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 2 |
| }, |
| "format_ok": 15, |
| "format_rate": 0.7142857142857143, |
| "n": 21, |
| "raw_accuracy": 0.2857142857142857, |
| "raw_correct": 6 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 6896, |
| "mean": 2170.1428571428573, |
| "p50": 1617.0, |
| "p90": 4988.0, |
| "total": 45573 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.9047619047619048, |
| "mean_completion_token_ratio": 0.27804351274511, |
| "raw_accuracy": 0.7142857142857143 |
| } |
| }, |
| "symbolic_transduction": { |
| "base": { |
| "budget_accuracy": 0.23809523809523808, |
| "budget_correct": 5, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7272.333333333333, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 152719 |
| }, |
| "finish_reason_counts": { |
| "length": 16, |
| "stop": 5 |
| }, |
| "format_ok": 10, |
| "format_rate": 0.47619047619047616, |
| "n": 21, |
| "raw_accuracy": 0.23809523809523808, |
| "raw_correct": 5 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.6190476190476191, |
| "budget_correct": 13, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3189.6190476190477, |
| "p50": 1507.0, |
| "p90": 8192.0, |
| "total": 66982 |
| }, |
| "finish_reason_counts": { |
| "length": 4, |
| "stop": 17 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.6666666666666666, |
| "raw_correct": 14 |
| }, |
| "delta": { |
| "budget_accuracy": 0.380952380952381, |
| "mean_completion_token_ratio": 0.43859637635133814, |
| "raw_accuracy": 0.42857142857142855 |
| } |
| }, |
| "temporal_intersection": { |
| "base": { |
| "budget_accuracy": 0.09090909090909091, |
| "budget_correct": 2, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 8035.909090909091, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 176790 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 3 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9090909090909091, |
| "n": 22, |
| "raw_accuracy": 0.09090909090909091, |
| "raw_correct": 2 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.18181818181818182, |
| "budget_correct": 4, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 6816.545454545455, |
| "p50": 7679.0, |
| "p90": 8192.0, |
| "total": 149964 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 15 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.18181818181818182, |
| "raw_correct": 4 |
| }, |
| "delta": { |
| "budget_accuracy": 0.09090909090909091, |
| "mean_completion_token_ratio": 0.8482606482267097, |
| "raw_accuracy": 0.09090909090909091 |
| } |
| }, |
| "weighted_decision": { |
| "base": { |
| "budget_accuracy": 0.9047619047619048, |
| "budget_correct": 19, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2538.8571428571427, |
| "p50": 1653.0, |
| "p90": 5727.0, |
| "total": 53316 |
| }, |
| "finish_reason_counts": { |
| "length": 2, |
| "stop": 19 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1050, |
| "mean": 696.3809523809524, |
| "p50": 659.0, |
| "p90": 801.0, |
| "total": 14624 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.09523809523809523, |
| "mean_completion_token_ratio": 0.2742891439717909, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| } |
| }, |
| "grader": { |
| "choice": { |
| "base": { |
| "budget_accuracy": 0.9296875, |
| "budget_correct": 119, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2561.578125, |
| "p50": 1455.0, |
| "p90": 6424.6, |
| "total": 327882 |
| }, |
| "finish_reason_counts": { |
| "length": 9, |
| "stop": 119 |
| }, |
| "format_ok": 121, |
| "format_rate": 0.9453125, |
| "n": 128, |
| "raw_accuracy": 0.9453125, |
| "raw_correct": 121 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 128, |
| "completion_tokens": { |
| "max": 4223, |
| "mean": 1116.6171875, |
| "p50": 745.5, |
| "p90": 2911.2, |
| "total": 142927 |
| }, |
| "finish_reason_counts": { |
| "stop": 128 |
| }, |
| "format_ok": 128, |
| "format_rate": 1.0, |
| "n": 128, |
| "raw_accuracy": 1.0, |
| "raw_correct": 128 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0703125, |
| "mean_completion_token_ratio": 0.43590987001421244, |
| "raw_accuracy": 0.0546875 |
| } |
| }, |
| "exact": { |
| "base": { |
| "budget_accuracy": 0.40625, |
| "budget_correct": 52, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 6187.3828125, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 791985 |
| }, |
| "finish_reason_counts": { |
| "length": 70, |
| "stop": 58 |
| }, |
| "format_ok": 109, |
| "format_rate": 0.8515625, |
| "n": 128, |
| "raw_accuracy": 0.4375, |
| "raw_correct": 56 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.6953125, |
| "budget_correct": 89, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2980.9375, |
| "p50": 1622.0, |
| "p90": 8180.8, |
| "total": 381560 |
| }, |
| "finish_reason_counts": { |
| "length": 13, |
| "stop": 115 |
| }, |
| "format_ok": 127, |
| "format_rate": 0.9921875, |
| "n": 128, |
| "raw_accuracy": 0.703125, |
| "raw_correct": 90 |
| }, |
| "delta": { |
| "budget_accuracy": 0.2890625, |
| "mean_completion_token_ratio": 0.48177680132830797, |
| "raw_accuracy": 0.265625 |
| } |
| }, |
| "numeric": { |
| "base": { |
| "budget_accuracy": 0.8515625, |
| "budget_correct": 109, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2961.5078125, |
| "p50": 1924.5, |
| "p90": 8192.0, |
| "total": 379073 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 109 |
| }, |
| "format_ok": 113, |
| "format_rate": 0.8828125, |
| "n": 128, |
| "raw_accuracy": 0.8828125, |
| "raw_correct": 113 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9921875, |
| "budget_correct": 127, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1575.875, |
| "p50": 968.5, |
| "p90": 3492.999999999999, |
| "total": 201712 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 127 |
| }, |
| "format_ok": 127, |
| "format_rate": 0.9921875, |
| "n": 128, |
| "raw_accuracy": 0.9921875, |
| "raw_correct": 127 |
| }, |
| "delta": { |
| "budget_accuracy": 0.140625, |
| "mean_completion_token_ratio": 0.5321191432784714, |
| "raw_accuracy": 0.109375 |
| } |
| } |
| }, |
| "language": { |
| "en": { |
| "base": { |
| "budget_accuracy": 0.5729166666666666, |
| "budget_correct": 110, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 4875.223958333333, |
| "p50": 5655.5, |
| "p90": 8192.0, |
| "total": 936043 |
| }, |
| "finish_reason_counts": { |
| "length": 81, |
| "stop": 111 |
| }, |
| "format_ok": 152, |
| "format_rate": 0.7916666666666666, |
| "n": 192, |
| "raw_accuracy": 0.625, |
| "raw_correct": 120 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.859375, |
| "budget_correct": 165, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2373.8697916666665, |
| "p50": 1065.0, |
| "p90": 6953.6, |
| "total": 455783 |
| }, |
| "finish_reason_counts": { |
| "length": 12, |
| "stop": 180 |
| }, |
| "format_ok": 190, |
| "format_rate": 0.9895833333333334, |
| "n": 192, |
| "raw_accuracy": 0.8645833333333334, |
| "raw_correct": 166 |
| }, |
| "delta": { |
| "budget_accuracy": 0.28645833333333337, |
| "mean_completion_token_ratio": 0.4869252801420448, |
| "raw_accuracy": 0.23958333333333337 |
| } |
| }, |
| "ja": { |
| "base": { |
| "budget_accuracy": 0.8854166666666666, |
| "budget_correct": 170, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2931.7552083333335, |
| "p50": 2246.5, |
| "p90": 7086.500000000002, |
| "total": 562897 |
| }, |
| "finish_reason_counts": { |
| "length": 17, |
| "stop": 175 |
| }, |
| "format_ok": 191, |
| "format_rate": 0.9947916666666666, |
| "n": 192, |
| "raw_accuracy": 0.8854166666666666, |
| "raw_correct": 170 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9322916666666666, |
| "budget_correct": 179, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1408.4166666666667, |
| "p50": 1023.5, |
| "p90": 2437.9, |
| "total": 270416 |
| }, |
| "finish_reason_counts": { |
| "length": 2, |
| "stop": 190 |
| }, |
| "format_ok": 192, |
| "format_rate": 1.0, |
| "n": 192, |
| "raw_accuracy": 0.9322916666666666, |
| "raw_correct": 179 |
| }, |
| "delta": { |
| "budget_accuracy": 0.046875, |
| "mean_completion_token_ratio": 0.4804004995585338, |
| "raw_accuracy": 0.046875 |
| } |
| } |
| }, |
| "task": { |
| "checksum_choice_en": { |
| "base": { |
| "budget_accuracy": 0.6666666666666666, |
| "budget_correct": 14, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 6019.333333333333, |
| "p50": 6531.0, |
| "p90": 8192.0, |
| "total": 126406 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 14 |
| }, |
| "format_ok": 15, |
| "format_rate": 0.7142857142857143, |
| "n": 21, |
| "raw_accuracy": 0.7142857142857143, |
| "raw_correct": 15 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 4223, |
| "mean": 2599.2380952380954, |
| "p50": 2980.0, |
| "p90": 3389.0, |
| "total": 54584 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.33333333333333337, |
| "mean_completion_token_ratio": 0.43181494549309374, |
| "raw_accuracy": 0.2857142857142857 |
| } |
| }, |
| "checksum_choice_ja": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 5677, |
| "mean": 3506.5714285714284, |
| "p50": 3495.0, |
| "p90": 5114.0, |
| "total": 73638 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 3544, |
| "mean": 1774.3333333333333, |
| "p50": 1648.0, |
| "p90": 2809.0, |
| "total": 37261 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5060023357505635, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "interval_overlap_en": { |
| "base": { |
| "budget_accuracy": 0.09090909090909091, |
| "budget_correct": 2, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 8035.909090909091, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 176790 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 3 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9090909090909091, |
| "n": 22, |
| "raw_accuracy": 0.09090909090909091, |
| "raw_correct": 2 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.18181818181818182, |
| "budget_correct": 4, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 6816.545454545455, |
| "p50": 7679.0, |
| "p90": 8192.0, |
| "total": 149964 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 15 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.18181818181818182, |
| "raw_correct": 4 |
| }, |
| "delta": { |
| "budget_accuracy": 0.09090909090909091, |
| "mean_completion_token_ratio": 0.8482606482267097, |
| "raw_accuracy": 0.09090909090909091 |
| } |
| }, |
| "interval_overlap_ja": { |
| "base": { |
| "budget_accuracy": 0.6363636363636364, |
| "budget_correct": 14, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 5000.5, |
| "p50": 4310.5, |
| "p90": 8192.0, |
| "total": 110011 |
| }, |
| "finish_reason_counts": { |
| "length": 7, |
| "stop": 15 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.6363636363636364, |
| "raw_correct": 14 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9090909090909091, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2350.2272727272725, |
| "p50": 1612.0, |
| "p90": 4764.200000000001, |
| "total": 51705 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 21 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 0.9090909090909091, |
| "raw_correct": 20 |
| }, |
| "delta": { |
| "budget_accuracy": 0.2727272727272727, |
| "mean_completion_token_ratio": 0.4699984546999845, |
| "raw_accuracy": 0.2727272727272727 |
| } |
| }, |
| "inventory_ledger_en": { |
| "base": { |
| "budget_accuracy": 0.9545454545454546, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2203.909090909091, |
| "p50": 1506.5, |
| "p90": 4014.6000000000004, |
| "total": 48486 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 0.9545454545454546, |
| "n": 22, |
| "raw_accuracy": 0.9545454545454546, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1720, |
| "mean": 825.5454545454545, |
| "p50": 699.5, |
| "p90": 1282.4, |
| "total": 18162 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.045454545454545414, |
| "mean_completion_token_ratio": 0.3745823536691003, |
| "raw_accuracy": 0.045454545454545414 |
| } |
| }, |
| "inventory_ledger_ja": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 2791, |
| "mean": 1189.8181818181818, |
| "p50": 1084.0, |
| "p90": 1929.5000000000007, |
| "total": 26176 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1554, |
| "mean": 785.5454545454545, |
| "p50": 771.0, |
| "p90": 1096.2, |
| "total": 17282 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.6602231051344744, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "modular_recurrence_en": { |
| "base": { |
| "budget_accuracy": 0.23809523809523808, |
| "budget_correct": 5, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7234.761904761905, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 151930 |
| }, |
| "finish_reason_counts": { |
| "length": 16, |
| "stop": 5 |
| }, |
| "format_ok": 9, |
| "format_rate": 0.42857142857142855, |
| "n": 21, |
| "raw_accuracy": 0.42857142857142855, |
| "raw_correct": 9 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3903.809523809524, |
| "p50": 3424.0, |
| "p90": 7788.0, |
| "total": 81980 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "delta": { |
| "budget_accuracy": 0.7142857142857142, |
| "mean_completion_token_ratio": 0.539590600934641, |
| "raw_accuracy": 0.5238095238095237 |
| } |
| }, |
| "modular_recurrence_ja": { |
| "base": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3452.0476190476193, |
| "p50": 3171.0, |
| "p90": 5387.0, |
| "total": 72493 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 6960, |
| "mean": 2382.809523809524, |
| "p50": 1892.0, |
| "p90": 4668.0, |
| "total": 50039 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.6902597492171658, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "piecewise_tariff_en": { |
| "base": { |
| "budget_accuracy": 0.9523809523809523, |
| "budget_correct": 20, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2132.0476190476193, |
| "p50": 1593.0, |
| "p90": 3281.0, |
| "total": 44773 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1116, |
| "mean": 701.5714285714286, |
| "p50": 681.0, |
| "p90": 880.0, |
| "total": 14733 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.32905992450807403, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "piecewise_tariff_ja": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 2393, |
| "mean": 1676.904761904762, |
| "p50": 1754.0, |
| "p90": 2146.0, |
| "total": 35215 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1164, |
| "mean": 929.3333333333334, |
| "p50": 970.0, |
| "p90": 1109.0, |
| "total": 19516 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5541956552605424, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "rule_filter_choice_en": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1007, |
| "mean": 805.3181818181819, |
| "p50": 803.5, |
| "p90": 947.5000000000001, |
| "total": 17717 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 518, |
| "mean": 417.3181818181818, |
| "p50": 410.5, |
| "p90": 505.6000000000001, |
| "total": 9181 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5182028560139978, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "rule_filter_choice_ja": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 1757, |
| "mean": 1193.0454545454545, |
| "p50": 1175.0, |
| "p90": 1434.7000000000003, |
| "total": 26247 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 22, |
| "completion_tokens": { |
| "max": 539, |
| "mean": 417.5, |
| "p50": 444.0, |
| "p90": 457.7, |
| "total": 9185 |
| }, |
| "finish_reason_counts": { |
| "stop": 22 |
| }, |
| "format_ok": 22, |
| "format_rate": 1.0, |
| "n": 22, |
| "raw_accuracy": 1.0, |
| "raw_correct": 22 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.3499447555911152, |
| "raw_accuracy": 0.0 |
| } |
| }, |
| "stack_machine_en": { |
| "base": { |
| "budget_accuracy": 0.09523809523809523, |
| "budget_correct": 2, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7805.047619047619, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 163906 |
| }, |
| "finish_reason_counts": { |
| "length": 19, |
| "stop": 2 |
| }, |
| "format_ok": 15, |
| "format_rate": 0.7142857142857143, |
| "n": 21, |
| "raw_accuracy": 0.2857142857142857, |
| "raw_correct": 6 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 6896, |
| "mean": 2170.1428571428573, |
| "p50": 1617.0, |
| "p90": 4988.0, |
| "total": 45573 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.9047619047619048, |
| "mean_completion_token_ratio": 0.27804351274511, |
| "raw_accuracy": 0.7142857142857143 |
| } |
| }, |
| "stack_machine_ja": { |
| "base": { |
| "budget_accuracy": 0.8571428571428571, |
| "budget_correct": 18, |
| "completion_tokens": { |
| "max": 7759, |
| "mean": 3738.095238095238, |
| "p50": 3432.0, |
| "p90": 5497.0, |
| "total": 78500 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.8571428571428571, |
| "raw_correct": 18 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.9047619047619048, |
| "budget_correct": 19, |
| "completion_tokens": { |
| "max": 2265, |
| "mean": 1380.857142857143, |
| "p50": 1471.0, |
| "p90": 1685.0, |
| "total": 28998 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.9047619047619048, |
| "raw_correct": 19 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904767, |
| "mean_completion_token_ratio": 0.3694012738853503, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "symbolic_rewrite_en": { |
| "base": { |
| "budget_accuracy": 0.23809523809523808, |
| "budget_correct": 5, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 7272.333333333333, |
| "p50": 8192.0, |
| "p90": 8192.0, |
| "total": 152719 |
| }, |
| "finish_reason_counts": { |
| "length": 16, |
| "stop": 5 |
| }, |
| "format_ok": 10, |
| "format_rate": 0.47619047619047616, |
| "n": 21, |
| "raw_accuracy": 0.23809523809523808, |
| "raw_correct": 5 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.6190476190476191, |
| "budget_correct": 13, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 3189.6190476190477, |
| "p50": 1507.0, |
| "p90": 8192.0, |
| "total": 66982 |
| }, |
| "finish_reason_counts": { |
| "length": 4, |
| "stop": 17 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.6666666666666666, |
| "raw_correct": 14 |
| }, |
| "delta": { |
| "budget_accuracy": 0.380952380952381, |
| "mean_completion_token_ratio": 0.43859637635133814, |
| "raw_accuracy": 0.42857142857142855 |
| } |
| }, |
| "symbolic_rewrite_ja": { |
| "base": { |
| "budget_accuracy": 0.5238095238095238, |
| "budget_correct": 11, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 5240.9047619047615, |
| "p50": 5309.0, |
| "p90": 8192.0, |
| "total": 110059 |
| }, |
| "finish_reason_counts": { |
| "length": 9, |
| "stop": 12 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.5238095238095238, |
| "raw_correct": 11 |
| }, |
| "candidate": { |
| "budget_accuracy": 0.5714285714285714, |
| "budget_correct": 12, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 1825.6190476190477, |
| "p50": 1285.0, |
| "p90": 2885.0, |
| "total": 38338 |
| }, |
| "finish_reason_counts": { |
| "length": 1, |
| "stop": 20 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 0.5714285714285714, |
| "raw_correct": 12 |
| }, |
| "delta": { |
| "budget_accuracy": 0.04761904761904756, |
| "mean_completion_token_ratio": 0.34834043558455013, |
| "raw_accuracy": 0.04761904761904756 |
| } |
| }, |
| "weighted_ranking_choice_en": { |
| "base": { |
| "budget_accuracy": 0.9047619047619048, |
| "budget_correct": 19, |
| "completion_tokens": { |
| "max": 8192, |
| "mean": 2538.8571428571427, |
| "p50": 1653.0, |
| "p90": 5727.0, |
| "total": 53316 |
| }, |
| "finish_reason_counts": { |
| "length": 2, |
| "stop": 19 |
| }, |
| "format_ok": 20, |
| "format_rate": 0.9523809523809523, |
| "n": 21, |
| "raw_accuracy": 0.9523809523809523, |
| "raw_correct": 20 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1050, |
| "mean": 696.3809523809524, |
| "p50": 659.0, |
| "p90": 801.0, |
| "total": 14624 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.09523809523809523, |
| "mean_completion_token_ratio": 0.2742891439717909, |
| "raw_accuracy": 0.04761904761904767 |
| } |
| }, |
| "weighted_ranking_choice_ja": { |
| "base": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 4228, |
| "mean": 1455.142857142857, |
| "p50": 1345.0, |
| "p90": 1434.0, |
| "total": 30558 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "candidate": { |
| "budget_accuracy": 1.0, |
| "budget_correct": 21, |
| "completion_tokens": { |
| "max": 1773, |
| "mean": 861.5238095238095, |
| "p50": 821.0, |
| "p90": 885.0, |
| "total": 18092 |
| }, |
| "finish_reason_counts": { |
| "stop": 21 |
| }, |
| "format_ok": 21, |
| "format_rate": 1.0, |
| "n": 21, |
| "raw_accuracy": 1.0, |
| "raw_correct": 21 |
| }, |
| "delta": { |
| "budget_accuracy": 0.0, |
| "mean_completion_token_ratio": 0.5920544538255121, |
| "raw_accuracy": 0.0 |
| } |
| } |
| } |
| }, |
| "trace_integrity": { |
| "base_raw_proxy_traces_sha256": "sha256:a026f73a07b184cf7fe11739f16d67a6ec3a4b95c7743b30ee3abe2d2018b1cc", |
| "base_samples_sha256": "sha256:c1927c0f5f5f0c8369de97bbe02ae3c5427e715c97cdbfc595d4e5712f0fcba9", |
| "candidate_raw_proxy_traces_sha256": "sha256:dd7a5f3952ac8c753596603e401a8720bad7be63f72ce4701e949e0f9143f5e8", |
| "candidate_samples_sha256": "sha256:cae4e7b155372b0b26ce5d042e3453a0f51b136f1097913feb1097304ee4c00a", |
| "full_trace_merge": true, |
| "matched_rows_per_model": 384 |
| }, |
| "used_for_training_or_selection": false |
| } |
|
|
