{ "artifact": { "format": "merged BF16 safetensors", "standalone": true }, "author": { "name": "Nobuyuki Horiuchi", "profile": "https://huggingface.co/horiuchinobuyuki" }, "base_model": { "id": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a" }, "benchmarks": { "gpqa_diamond": { "accuracy_delta": 0.020202020202020204, "base": { "accuracy": 0.7777777777777778, "correct": 154 }, "base_mean_completion_tokens": 10042.040404040405, "candidate": { "accuracy": 0.797979797979798, "correct": 158 }, "candidate_mean_completion_tokens": 7132.131313131313, "decode_profile": "thinking_general_32768", "extractor": "flexible-extract", "mean_completion_token_ratio": 0.7102273070183732, "n": 198, "role": "primary selection target", "shortening_fraction": 0.2897726929816268, "task": "gpqa_diamond_cot_zeroshot" }, "hmmt_2025_public_60x4": { "base": { "budget_correct": 187, "raw_correct": 187 }, "budget_accuracy_delta": -0.0625, "candidate": { "budget_correct": 172, "raw_correct": 172 }, "decode_profile": "thinking_general_81920_four_seed", "mean_completion_token_ratio": 0.6794268679708825, "n": 240, "role": "report-only capability risk under the amended practical objective", "shortening_fraction": 0.3205731320291175 }, "ifeval": { "base": { "inst_level_loose_acc": 0.9424460431654677, "inst_level_strict_acc": 0.9220623501199041, "prompt_level_loose_acc": 0.9242144177449169, "prompt_level_strict_acc": 0.8927911275415896 }, "base_empty_final_content": 8, "base_length_stops": 8, "base_mean_fulltrace_tokens": 4461.646950092421, "candidate": { "inst_level_loose_acc": 0.9400479616306955, "inst_level_strict_acc": 0.920863309352518, "prompt_level_loose_acc": 0.9242144177449169, "prompt_level_strict_acc": 0.8964879852125693 }, "candidate_empty_final_content": 13, "candidate_length_stops": 12, "candidate_mean_fulltrace_tokens": 3445.1940850277265, "comparable_to_upstream_model_card_score": false, "dataset_revision": "966cd89545d6b6acfd7638bc708b98261ca58e84", "decode_profile": "thinking_general_32768", "delta": { "inst_level_loose_acc": -0.0023980815347721673, "inst_level_strict_acc": -0.0011990407673860837, "prompt_level_loose_acc": 0.0, "prompt_level_strict_acc": 0.003696857670979714 }, "fulltrace_mean_token_ratio": 0.772179897595071, "grading": "official-compatible final content; full trace retained separately", "headline_claim_eligible": true, "n_prompts": 541, "official_model_card_prompt_strict_acc": 0.915, "role": "corrected matched report; no post-hoc numeric gate or reselection", "shortening_fraction": 0.227820102404929, "temperature_zero_predecessor_invalidated": true, "upstream_comparability_note": "The local Base sanity check uses the official-recommended sampling values, but the upstream card does not fully specify the IFEval harness and grader conditions." }, "ifeval_base_length_stop_128k_overlay": { "candidate_reselection": false, "decode_profile": "thinking_general_131072_base_length_overlay", "delta": { "inst_level_loose_acc": 0.00959232613908867, "inst_level_strict_acc": 0.00959232613908867, "prompt_level_loose_acc": 0.007393715341959317, "prompt_level_strict_acc": 0.007393715341959317 }, "empty_final_content_delta": -6, "empty_final_content_transitions": { "content->content": 533, "empty->content": 6, "empty->empty": 2 }, "finish_reason_transitions": { "length->length": 2, "length->stop": 6, "stop->stop": 533 }, "higher": { "empty_final_content": 2, "finish_reasons": { "length": 2, "stop": 539 }, "full_trace_tokens": { "maximum": 131072, "mean": 4832.584103512015, "n": 541, "p50": 3592, "p90": 7430.0, "p95": 8815.0, "p99": 14310.800000000043 }, "graded_content_tokens": { "maximum": 2348, "mean": 276.1312384473198, "n": 541, "p50": 169, "p90": 639.0, "p95": 829.0, "p99": 1555.0000000000005 }, "metrics": { "inst_level_loose_acc": 0.9520383693045563, "inst_level_strict_acc": 0.9316546762589928, "prompt_level_loose_acc": 0.9316081330868762, "prompt_level_strict_acc": 0.9001848428835489 } }, "higher_budget": 131072, "length_stop_delta": -6, "lower": { "empty_final_content": 8, "finish_reasons": { "length": 8, "stop": 533 }, "full_trace_tokens": { "maximum": 32769, "mean": 4461.646950092421, "n": 541, "p50": 3637, "p90": 7495.0, "p95": 9238.0, "p99": 32768.0 }, "graded_content_tokens": { "maximum": 2348, "mean": 271.7985212569316, "n": 541, "p50": 167, "p90": 636.0, "p95": 826.0, "p99": 1522.400000000001 }, "metrics": { "inst_level_loose_acc": 0.9424460431654677, "inst_level_strict_acc": 0.9220623501199041, "prompt_level_loose_acc": 0.9242144177449169, "prompt_level_strict_acc": 0.8927911275415896 } }, "lower_budget": 32768, "matched_candidate_comparison": false, "n_prompts_logical_overlay": 541, "n_regenerated": 8, "pure_token_budget_causal_estimate": false, "role": "Qwen3.5-9B-only report-only budget/runtime sensitivity diagnostic", "runtime_conditions": { "higher_overlay": { "harness": "lm-eval==0.4.12", "max_gen_toks": 131072, "max_length": 262144, "num_concurrent": 2, "seed": 2026080901, "sha256": "sha256:fe5e8e0df2e4e6589715dd0302fc1ba84f4d9a55ffddce4b7bd8f9b97580b40e", "started_at_utc": "20260823T123340Z", "tasks": "ifeval_qwen_bf16_official_sampling_32k_length_8_128k", "trace_policy": "full_trace_merge" }, "interpretation": "The generation cap and feasible runtime concurrency/replica layout both differ. Sampled trajectories can be runtime-layout-sensitive, so this is not a pure causal token-budget estimate.", "lower": { "harness": "lm-eval==0.4.12", "max_gen_toks": 32768, "max_length": 65536, "num_concurrent": 96, "seed": 2026080901, "sha256": "sha256:4cdce65876faacc53aea1a86f7e8510ec210fe156641371ff8830ca6e365efd4", "started_at_utc": "20260823T103850Z", "tasks": "ifeval_pinned_966cd895", "trace_policy": "full_trace_merge" } }, "synthetic_holdout_rerun": false, "trace_integrity": { "higher_full_trace_rows": 541, "lower_full_trace_rows": 541, "missing_full_trace_rows": 0, "reasoning_fallback_when_final_content_empty": false } }, "jmmlu_common_clean": { "base": { "length_stops": 0, "mean_completion_tokens": 4530.474164133739, "raw_accuracy": 0.8794326241134752, "raw_correct": 868 }, "candidate": { "budget_correct": 858, "length_stops": 1, "mean_completion_tokens": 2081.7507598784196, "raw_accuracy": 0.8693009118541033, "raw_correct": 858 }, "decode_profile": "thinking_general_32768", "mean_completion_token_ratio": 0.4594995323798444, "n": 987, "raw_accuracy_delta": -0.010131712259371928, "role": "Japanese generalization", "shortening_fraction": 0.5405004676201556 }, "livecodebench_v6": { "base": { "budget_correct": 702, "length_stops": 278, "mean_completion_tokens": 35512.85402843602, "raw_correct": 766 }, "budget_accuracy_delta": 0.0729857819905213, "candidate": { "budget_correct": 779, "length_stops": 76, "mean_completion_tokens": 17327.68720379147, "raw_correct": 785 }, "decode_profile": "thinking_coding_81920", "grader": { "generation_errors": 0, "network_isolation": "bubblewrap network namespace unshared", "runner_commit": "28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24" }, "mean_completion_token_ratio": 0.48792719362732045, "n": 1055, "paired_transitions": { "budget_base_only_loss": 79, "budget_candidate_only_gain": 156, "raw_base_only_loss": 88, "raw_candidate_only_gain": 107 }, "raw_accuracy_delta": 0.018009478672985857, "role": "code generalization", "shortening_fraction": 0.5120728063726796 }, "mmlu_pro": { "accuracy_delta": -0.0026595744680851063, "base": { "accuracy": 0.8138297872340425, "correct": 9792 }, "base_mean_completion_tokens": 4115.58751662234, "candidate": { "accuracy": 0.8111702127659575, "correct": 9760 }, "candidate_mean_completion_tokens": 2489.4774767287236, "decode_profile": "thinking_general_32768", "diagnostic_extractor": "first match", "mean_completion_token_ratio": 0.6048899377486293, "n": 12032, "primary_extractor": "last match", "role": "primary selection target", "shortening_fraction": 0.3951100622513707, "task": "mmlu_pro_fulltrace, 5-shot" }, "mmmu_quantized_vision_validation_900": { "candidate_reselection": false, "claim_scope": "complete 900-item MMMU validation split; not the 10,500-item test split", "comparisons_to_qwick_bf16": { "qwick_fp8_minus_qwick_bf16": { "candidate": "qwick_fp8", "candidate_mean_completion_token_change_pct": 4.6625715448075145, "candidate_minus_source_bf16_official_budget_accuracy_pp": -1.7777777777777781, "candidate_source_bf16_mean_completion_token_ratio": 1.0466257154480751, "discordant_exact_mcnemar_two_sided_p": 0.19818134136296867, "per_subject": { "Accounting": { "candidate_correct": 26, "n": 30, "source_bf16_correct": 24 }, "Agriculture": { "candidate_correct": 15, "n": 30, "source_bf16_correct": 19 }, "Architecture_and_Engineering": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 24 }, "Art": { "candidate_correct": 19, "n": 30, "source_bf16_correct": 20 }, "Art_Theory": { "candidate_correct": 27, "n": 30, "source_bf16_correct": 25 }, "Basic_Medical_Science": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 23 }, "Biology": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 24 }, "Chemistry": { "candidate_correct": 15, "n": 30, "source_bf16_correct": 21 }, "Clinical_Medicine": { "candidate_correct": 15, "n": 30, "source_bf16_correct": 17 }, "Computer_Science": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 24 }, "Design": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 24 }, "Diagnostics_and_Laboratory_Medicine": { "candidate_correct": 14, "n": 30, "source_bf16_correct": 12 }, "Economics": { "candidate_correct": 25, "n": 30, "source_bf16_correct": 24 }, "Electronics": { "candidate_correct": 15, "n": 30, "source_bf16_correct": 21 }, "Energy_and_Power": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 24 }, "Finance": { "candidate_correct": 26, "n": 30, "source_bf16_correct": 25 }, "Geography": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 20 }, "History": { "candidate_correct": 22, "n": 30, "source_bf16_correct": 23 }, "Literature": { "candidate_correct": 27, "n": 30, "source_bf16_correct": 26 }, "Manage": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 20 }, "Marketing": { "candidate_correct": 29, "n": 30, "source_bf16_correct": 27 }, "Materials": { "candidate_correct": 22, "n": 30, "source_bf16_correct": 22 }, "Math": { "candidate_correct": 25, "n": 30, "source_bf16_correct": 24 }, "Mechanical_Engineering": { "candidate_correct": 19, "n": 30, "source_bf16_correct": 16 }, "Music": { "candidate_correct": 11, "n": 30, "source_bf16_correct": 13 }, "Pharmacy": { "candidate_correct": 27, "n": 30, "source_bf16_correct": 28 }, "Physics": { "candidate_correct": 28, "n": 30, "source_bf16_correct": 28 }, "Psychology": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 25 }, "Public_Health": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 29 }, "Sociology": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 19 } }, "stratified_bootstrap_95pct_ci": { "high_pp": 0.7777777777777778, "low_pp": -4.222222222222222, "method": "paired bootstrap, resampled within each subject stratum", "resamples": 10000, "role": "descriptive interval for one-seed sampled generation", "seed": 2026082403 }, "transitions": { "both_correct": 595, "both_wrong": 169, "candidate_only": 60, "source_bf16_only": 76 } }, "qwick_nvfp4_minus_qwick_bf16": { "candidate": "qwick_nvfp4", "candidate_mean_completion_token_change_pct": 13.31993608405304, "candidate_minus_source_bf16_official_budget_accuracy_pp": -3.9999999999999925, "candidate_source_bf16_mean_completion_token_ratio": 1.1331993608405304, "discordant_exact_mcnemar_two_sided_p": 0.006751369163205467, "per_subject": { "Accounting": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 24 }, "Agriculture": { "candidate_correct": 18, "n": 30, "source_bf16_correct": 19 }, "Architecture_and_Engineering": { "candidate_correct": 20, "n": 30, "source_bf16_correct": 24 }, "Art": { "candidate_correct": 19, "n": 30, "source_bf16_correct": 20 }, "Art_Theory": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 25 }, "Basic_Medical_Science": { "candidate_correct": 23, "n": 30, "source_bf16_correct": 23 }, "Biology": { "candidate_correct": 19, "n": 30, "source_bf16_correct": 24 }, "Chemistry": { "candidate_correct": 16, "n": 30, "source_bf16_correct": 21 }, "Clinical_Medicine": { "candidate_correct": 17, "n": 30, "source_bf16_correct": 17 }, "Computer_Science": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 24 }, "Design": { "candidate_correct": 25, "n": 30, "source_bf16_correct": 24 }, "Diagnostics_and_Laboratory_Medicine": { "candidate_correct": 12, "n": 30, "source_bf16_correct": 12 }, "Economics": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 24 }, "Electronics": { "candidate_correct": 16, "n": 30, "source_bf16_correct": 21 }, "Energy_and_Power": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 24 }, "Finance": { "candidate_correct": 26, "n": 30, "source_bf16_correct": 25 }, "Geography": { "candidate_correct": 20, "n": 30, "source_bf16_correct": 20 }, "History": { "candidate_correct": 22, "n": 30, "source_bf16_correct": 23 }, "Literature": { "candidate_correct": 26, "n": 30, "source_bf16_correct": 26 }, "Manage": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 20 }, "Marketing": { "candidate_correct": 28, "n": 30, "source_bf16_correct": 27 }, "Materials": { "candidate_correct": 21, "n": 30, "source_bf16_correct": 22 }, "Math": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 24 }, "Mechanical_Engineering": { "candidate_correct": 17, "n": 30, "source_bf16_correct": 16 }, "Music": { "candidate_correct": 6, "n": 30, "source_bf16_correct": 13 }, "Pharmacy": { "candidate_correct": 27, "n": 30, "source_bf16_correct": 28 }, "Physics": { "candidate_correct": 28, "n": 30, "source_bf16_correct": 28 }, "Psychology": { "candidate_correct": 24, "n": 30, "source_bf16_correct": 25 }, "Public_Health": { "candidate_correct": 26, "n": 30, "source_bf16_correct": 29 }, "Sociology": { "candidate_correct": 18, "n": 30, "source_bf16_correct": 19 } }, "stratified_bootstrap_95pct_ci": { "high_pp": -1.2222222222222223, "low_pp": -6.777777777777778, "method": "paired bootstrap, resampled within each subject stratum", "resamples": 10000, "role": "descriptive interval for one-seed sampled generation", "seed": 2026082404 }, "transitions": { "both_correct": 569, "both_wrong": 163, "candidate_only": 66, "source_bf16_only": 102 } } }, "complete_validation_split": true, "dataset": { "id": "MMMU/MMMU", "license": "apache-2.0", "manifest_sha256": "5fd7d207b1b1c248987e0f79f98060a2c5b5eba55a07fa430f2d3ee92b601013", "question_type_counts": { "multiple-choice": 847, "open": 53 }, "records": 900, "revision": "98e6ac0cb9b7b2cd2c991b85a50762edc4aedc68", "rows_per_subject": 30, "selection_policy": "complete validation split; no item selection", "split": "validation", "subjects": 30, "window_sha256": "56615c75e3d93c71148e570283451e90dd36450a8cef2ef6dac1425ee8634244" }, "decode_profile": "vision_thinking_general_32768", "full_mmmu_test_score": false, "grading": { "full_trace_retained": true, "official_compatible": "MMMU parser/evaluator at the pinned commit, including independent per-lane random.seed(42) multiple-choice parse fallback", "primary": "official-compatible budget accuracy = correct and finish_reason != length", "secondary": "deterministic no-random-fallback strict raw/budget accuracy", "source": "final content only" }, "limitations": [ "This is the complete validation split, not the larger released test split.", "Generation uses one matched sampling seed per item and model.", "The official multiple-choice parser guesses on parse misses; fallback counts and deterministic strict scores are reported.", "The quantized checkpoints exclude the visual tower from quantization, while language-model Linear layers are quantized.", "The result is descriptive and is published regardless of direction." ], "models": { "base_bf16": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a", "quantization_scope": "language-model Linear layers; visual tower excluded", "qwick_bf16": "Qwick-3.5-9B BF16 g7-nes-d04-p; training-update sha256:2a25037cd9c7bbdacdbefba3b09424e74f7759cb024acfe56f3a412db0f92e0e", "qwick_fp8": "Qwick-3.5-9B-FP8; ModelOpt 0.45.0; weights sha256:e783c584408b2cf24e9332774071ea11e9fa0be5d3c80c86d200381fa000006e,sha256:dd63614f1dc80dce2d83be3f3e69af1f8a0ebc8add9b8e0abf3910f563e44344", "qwick_nvfp4": "Qwick-3.5-9B-NVFP4; ModelOpt 0.45.0; weights sha256:274fbcfc303b88fd2bcbc84a28a726514c86f5c97a439d9322c98d1d586246ab" }, "role": "report_only; disclosed regardless of direction; not a release gate", "runs": { "base_bf16": { "by_question_type": { "multiple-choice": { "budget_official_correct": 634, "budget_strict_correct": 632, "official_correct": 638, "records": 847, "strict_correct": 633 }, "open": { "budget_official_correct": 30, "budget_strict_correct": 30, "official_correct": 30, "records": 53, "strict_correct": 30 } }, "completion_tokens": { "max": 32768, "mean": 6476.481111111111, "p50": 2922.5, "p90": 18142.600000000002 }, "empty_final_contents": 11, "errors": 0, "finish_reason_counts": { "length": 12, "stop": 888 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7377777777777778, "official_budget_correct": 664, "official_random_fallbacks": 14, "official_raw_accuracy": 0.7422222222222222, "official_raw_correct": 668, "records": 900, "strict_budget_accuracy": 0.7355555555555555, "strict_budget_correct": 662, "strict_raw_accuracy": 0.7366666666666667, "strict_raw_correct": 663, "strict_unparsed_final_contents": 14 }, "qwick_bf16": { "by_question_type": { "multiple-choice": { "budget_official_correct": 638, "budget_strict_correct": 634, "official_correct": 645, "records": 847, "strict_correct": 636 }, "open": { "budget_official_correct": 33, "budget_strict_correct": 33, "official_correct": 33, "records": 53, "strict_correct": 33 } }, "completion_tokens": { "max": 32768, "mean": 4798.663333333333, "p50": 1567.5, "p90": 14666.800000000007 }, "empty_final_contents": 12, "errors": 0, "finish_reason_counts": { "length": 15, "stop": 885 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7455555555555555, "official_budget_correct": 671, "official_random_fallbacks": 21, "official_raw_accuracy": 0.7533333333333333, "official_raw_correct": 678, "records": 900, "strict_budget_accuracy": 0.7411111111111112, "strict_budget_correct": 667, "strict_raw_accuracy": 0.7433333333333333, "strict_raw_correct": 669, "strict_unparsed_final_contents": 21 }, "qwick_fp8": { "by_question_type": { "multiple-choice": { "budget_official_correct": 626, "budget_strict_correct": 626, "official_correct": 629, "records": 847, "strict_correct": 626 }, "open": { "budget_official_correct": 29, "budget_strict_correct": 29, "official_correct": 29, "records": 53, "strict_correct": 29 } }, "completion_tokens": { "max": 32768, "mean": 5022.404444444444, "p50": 1607.0, "p90": 16310.100000000002 }, "empty_final_contents": 14, "errors": 0, "finish_reason_counts": { "length": 16, "stop": 884 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7277777777777777, "official_budget_correct": 655, "official_random_fallbacks": 17, "official_raw_accuracy": 0.7311111111111112, "official_raw_correct": 658, "records": 900, "strict_budget_accuracy": 0.7277777777777777, "strict_budget_correct": 655, "strict_raw_accuracy": 0.7277777777777777, "strict_raw_correct": 655, "strict_unparsed_final_contents": 17 }, "qwick_nvfp4": { "by_question_type": { "multiple-choice": { "budget_official_correct": 607, "budget_strict_correct": 598, "official_correct": 620, "records": 847, "strict_correct": 600 }, "open": { "budget_official_correct": 28, "budget_strict_correct": 28, "official_correct": 28, "records": 53, "strict_correct": 28 } }, "completion_tokens": { "max": 32768, "mean": 5437.8422222222225, "p50": 1299.0, "p90": 18765.9 }, "empty_final_contents": 46, "errors": 0, "finish_reason_counts": { "length": 48, "stop": 852 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7055555555555556, "official_budget_correct": 635, "official_random_fallbacks": 79, "official_raw_accuracy": 0.72, "official_raw_correct": 648, "records": 900, "strict_budget_accuracy": 0.6955555555555556, "strict_budget_correct": 626, "strict_raw_accuracy": 0.6977777777777778, "strict_raw_correct": 628, "strict_unparsed_final_contents": 79 } }, "synthetic_holdout_rerun": false, "trace_integrity": { "matched_prompt_hash_seeds_across_qwick_bf16_fp8_nvfp4": true, "missing_full_traces": 0, "proxies": { "qwick_bf16": { "content_mode": "merge", "decode_violations": 0, "image_parts_max": 5, "image_parts_min": 1, "image_parts_total": 980, "missing_full_responses": 0, "path_sha256": "7aeaf383241c88b56c6178f5a8fd051b4bf13d729508f0275925d8db7bd81e75", "records": 900, "unique_items": 900 }, "qwick_fp8": { "content_mode": "merge", "decode_violations": 0, "image_parts_max": 5, "image_parts_min": 1, "image_parts_total": 980, "missing_full_responses": 0, "path_sha256": "4dcc53bd02f832e64b06bd0a0bb86cc6439e13f49bd9a00320a3c3e2e98e3217", "records": 900, "unique_items": 900 }, "qwick_nvfp4": { "content_mode": "merge", "decode_violations": 0, "image_parts_max": 5, "image_parts_min": 1, "image_parts_total": 980, "missing_full_responses": 0, "path_sha256": "5075bc701be128cf5fc49c74fa05a8baf08e8de14f7c575de67c1a33c5420f80", "records": 900, "unique_items": 900 } }, "qwick_bf16_samples_sha256": "92faede14af458abc4b9b4f2fd526c54eabe65b491e731b05d6c258e890a29b0", "qwick_fp8_samples_sha256": "68cdd094bfdb10de75019e6ab01f359b520aac070bf4861b33e4f46e6a90e3c6", "qwick_nvfp4_samples_sha256": "37c5f5090f4af57a2d79de38157da34c862a6724b45628ce083aa1e2bd939583", "source_bf16_analysis_sha256": "1430028ffeb3d2b317150f8b1f48d46f3c63251633c146d28c68fe9be0586250" } }, "mmmu_vision_validation_900": { "base": { "by_question_type": { "multiple-choice": { "budget_official_correct": 634, "budget_strict_correct": 632, "official_correct": 638, "records": 847, "strict_correct": 633 }, "open": { "budget_official_correct": 30, "budget_strict_correct": 30, "official_correct": 30, "records": 53, "strict_correct": 30 } }, "completion_tokens": { "max": 32768, "mean": 6476.481111111111, "p50": 2922.5, "p90": 18142.600000000002 }, "empty_final_contents": 11, "errors": 0, "finish_reason_counts": { "length": 12, "stop": 888 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7377777777777778, "official_budget_correct": 664, "official_random_fallbacks": 14, "official_raw_accuracy": 0.7422222222222222, "official_raw_correct": 668, "records": 900, "strict_budget_accuracy": 0.7355555555555555, "strict_budget_correct": 662, "strict_raw_accuracy": 0.7366666666666667, "strict_raw_correct": 663, "strict_unparsed_final_contents": 14 }, "candidate": { "by_question_type": { "multiple-choice": { "budget_official_correct": 638, "budget_strict_correct": 634, "official_correct": 645, "records": 847, "strict_correct": 636 }, "open": { "budget_official_correct": 33, "budget_strict_correct": 33, "official_correct": 33, "records": 53, "strict_correct": 33 } }, "completion_tokens": { "max": 32768, "mean": 4798.663333333333, "p50": 1567.5, "p90": 14666.800000000007 }, "empty_final_contents": 12, "errors": 0, "finish_reason_counts": { "length": 15, "stop": 885 }, "missing_full_traces": 0, "official_budget_accuracy": 0.7455555555555555, "official_budget_correct": 671, "official_random_fallbacks": 21, "official_raw_accuracy": 0.7533333333333333, "official_raw_correct": 678, "records": 900, "strict_budget_accuracy": 0.7411111111111112, "strict_budget_correct": 667, "strict_raw_accuracy": 0.7433333333333333, "strict_raw_correct": 669, "strict_unparsed_final_contents": 21 }, "candidate_reselection": false, "claim_scope": "complete 900-item MMMU validation split; not the 10,500-item test split", "complete_validation_split": true, "dataset": { "id": "MMMU/MMMU", "license": "apache-2.0", "manifest_sha256": "5fd7d207b1b1c248987e0f79f98060a2c5b5eba55a07fa430f2d3ee92b601013", "question_type_counts": { "multiple-choice": 847, "open": 53 }, "records": 900, "revision": "98e6ac0cb9b7b2cd2c991b85a50762edc4aedc68", "rows_per_subject": 30, "selection_policy": "complete validation split; no item selection", "split": "validation", "subjects": 30, "window_sha256": "56615c75e3d93c71148e570283451e90dd36450a8cef2ef6dac1425ee8634244" }, "decode_profile": "vision_thinking_general_32768", "full_mmmu_test_score": false, "grading": { "full_trace_retained": true, "official_compatible": "MMMU parser/evaluator at the pinned commit, including independent per-lane random.seed(42) multiple-choice parse fallback", "primary": "official-compatible budget accuracy = correct and finish_reason != length", "secondary": "deterministic no-random-fallback strict raw/budget accuracy", "source": "final content only" }, "limitations": [ "This is the complete validation split, not the larger released test split.", "Generation uses one paired sampling seed per item.", "The official multiple-choice parser guesses on parse misses; fallback counts and deterministic strict scores are reported.", "This evaluates BF16 Base-to-Qwick retention, not FP8 or NVFP4 vision behavior." ], "models": { "base": "Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a", "comparison": "BF16 Base versus BF16 Qwick; quantized variants not run", "qwick": "Qwick-3.5-9B g7-nes-d04-p; training-update sha256:2a25037cd9c7bbdacdbefba3b09424e74f7759cb024acfe56f3a412db0f92e0e" }, "paired": { "discordant_exact_mcnemar_two_sided_p": 0.6321848782933385, "qwick_base_mean_completion_token_ratio": 0.740936822173495, "qwick_mean_completion_token_change_pct": -25.9063177826505, "qwick_minus_base_official_budget_accuracy_pp": 0.7777777777777772, "stratified_bootstrap_95pct_ci": { "high_pp": 3.4444444444444446, "low_pp": -1.8888888888888888, "method": "paired bootstrap, resampled within each subject stratum", "resamples": 10000, "role": "descriptive interval for one-seed sampled generation", "seed": 2026082402 }, "transitions": { "base_only": 75, "both_correct": 589, "both_wrong": 154, "qwick_only": 82 } }, "role": "report_only; disclosed regardless of direction; not a release gate", "synthetic_holdout_rerun": false, "trace_integrity": { "base_proxy": { "content_mode": "merge", "decode_violations": 0, "image_parts_max": 5, "image_parts_min": 1, "image_parts_total": 980, "missing_full_responses": 0, "path_sha256": "fccf239f0f0f9a628d09eb2835ad1a52d05fc15ef5412c6a1b91f75c463775ff", "records": 900, "unique_items": 900 }, "base_samples_sha256": "941b7581e5d219a561e98ec5cb3a9e1e2392cad9c96826e9358d282a17e4f0f8", "matched_prompt_hash_seeds": true, "missing_full_traces": 0, "qwick_proxy": { "content_mode": "merge", "decode_violations": 0, "image_parts_max": 5, "image_parts_min": 1, "image_parts_total": 980, "missing_full_responses": 0, "path_sha256": "7aeaf383241c88b56c6178f5a8fd051b4bf13d729508f0275925d8db7bd81e75", "records": 900, "unique_items": 900 }, "qwick_samples_sha256": "92faede14af458abc4b9b4f2fd526c54eabe65b491e731b05d6c258e890a29b0" } } }, "candidate_id": "g7-nes-d04-p", "claim_scope": "matched Base-relative public-development measurements; not claimed as bit-exact reproduction of the upstream model card", "correctness_definitions": { "budget_correct": "raw_correct && finish_reason != length", "raw_correct": "correct regardless of finish reason" }, "decode_profiles": { "thinking_coding_81920": { "enable_thinking": true, "max_new_tokens": 81920, "presence_penalty": 0.0, "seed": 2026081101, "temperature": 0.6, "top_k": 20, "top_p": 0.95 }, "thinking_general_131072_base_length_overlay": { "enable_thinking": true, "max_new_tokens": 131072, "min_p": 0.0, "presence_penalty": 1.5, "repetition_penalty": 1.0, "scope": "upstream BF16 32k length-stop rows only", "temperature": 1.0, "top_k": 20, "top_p": 0.95 }, "thinking_general_32768": { "enable_thinking": true, "max_new_tokens": 32768, "min_p": 0.0, "presence_penalty": 1.5, "repetition_penalty": 1.0, "temperature": 1.0, "top_k": 20, "top_p": 0.95 }, "thinking_general_81920_four_seed": { "enable_thinking": true, "max_new_tokens": 81920, "presence_penalty": 1.5, "seeds": [ 2026081101, 2026081102, 2026081103, 2026081104 ], "temperature": 1.0, "top_k": 20, "top_p": 0.95 }, "vision_thinking_general_32768": { "enable_thinking": true, "max_tokens": 32768, "min_p": 0.0, "presence_penalty": 1.5, "repetition_penalty": 1.0, "request_seed_pairing": "model-independent prompt hash", "server_context_tokens": 131072, "temperature": 1.0, "top_k": 20, "top_p": 0.95 } }, "heldout_known_residual_risk": "The one-shot holdout is synthetic, bilingual, and structurally balanced; it does not establish uniform performance on every natural or deployment domain.", "heldout_public_benchmark": false, "heldout_release_gate_pass": true, "heldout_used": true, "heldout_used_only_after_candidate_and_policy_freeze": true, "kind": "qwick_3_5_9b_bf16_public_benchmark_comparison", "known_risks": { "hmmt81920_observed_gate_pass": false, "hmmt81920_policy": "report_only_by_explicit_release_objective_amendment", "mmlu_pro_worst_subject": { "accuracy_delta": -0.02063983488132095, "base_correct": 753, "candidate_correct": 733, "n": 969, "policy": "reported subgroup risk; aggregate MMLU-Pro remains the frozen selection gate", "subject": "engineering" } }, "length_definition": "completion tokens over the persisted full generation trace (thinking plus final answer)", "public_model": { "name": "Qwick-3.5-9B", "repo_id": "horiuchinobuyuki/Qwick-3.5-9B", "url": "https://huggingface.co/horiuchinobuyuki/Qwick-3.5-9B" }, "schema_version": 1, "selection_gates": { "sft5_genre_v3_heldout": { "decode": { "enable_thinking": true, "max_tokens": 8192, "presence_penalty": 0.0, "seed": 2026081801, "temperature": 0.0, "top_k": -1, "top_p": 1.0, "trace_policy": "full upstream reasoning plus final content durably persisted; merged full trace graded" }, "interpretation": "The predeclared temperature-zero decode is valid for this frozen internal acceptance rule. It is not the temperature-1.0 measurement profile used for public benchmark claims and must not be compared as though it were.", "policy_sha256": "sha256:8a929e01ec7f5d60a5b14ff6664f74ce4d86b216d26155ca613e371183ef5701", "public_performance_benchmark": false, "release_gate_pass": true, "result_file": "heldout_results.json", "role": "frozen internal candidate-acceptance release gate" } }, "selection_objective": { "gpqa_mean_token_ratio": 0.7102273070183732, "mmlu_mean_token_ratio": 0.6048899377486293, "pass": true, "rule": "GPQA-Diamond or MMLU-Pro mean full-trace token ratio versus Base must be <=0.70, with frozen benchmark-specific accuracy guards" } }