eval_compare: upload findings run 20260426_083700

findings/20260426_083700/comparison_tables.md

@@ -0,0 +1,46 @@
+# Model Comparison: fine-tuned vs base Qwen2.5-7B
+
+- **Fine-tuned** : `Yaswanth-Bolla/qwen-merged`
+- **Base**       : `Qwen/Qwen2.5-7B-Instruct`
+
+## Claim 1 — Orchestrator stopping behaviour (Pool C)
+
+| Metric | fine-tuned | base | Δ (fine − base) |
+| --- | --- | --- | --- |
+| mean_final | 0.4537 | 0.4495 | ▲ 0.0042 |
+| mean_p1_steps | 15.75 | 16.62 | ▼ 0.87 |
+| mean_p2_steps | 6.5 | 5.62 | ▲ 0.88 |
+| mean_r_cross | 0.4662 | 0.4412 | ▲ 0.025 |
+| ECE | 0.8125 | 0.5813 | ▲ 0.2312 |
+| stopping_distribution (histogram) | see JSON | see JSON | — |
+
+## Claim 2 — r_cross ablation (Pool C)
+
+| Condition | Metric | fine-tuned | base | Δ |
+| --- | --- | --- | --- | --- |
+| r_cross_on | mean_final | 0.4994 | 0.4738 | ▲ 0.0256 |
+| r_cross_on | mean_r_cross | 0.585 | 0.4163 | ▲ 0.1687 |
+| r_cross_on | p2_steps_to_correct_patch | 6.86 | 5.86 | ▲ 1.0 |
+| r_cross_off | mean_final | 0.4867 | 0.4813 | ▲ 0.0054 |
+| r_cross_off | mean_r_cross | 0.0 | 0.0 | = 0.0 |
+| r_cross_off | p2_steps_to_correct_patch | 7.29 | 4.29 | ▲ 3.0 |
+
+## Claim 3 — Convergence proxy (Pool C)
+
+| Metric | fine-tuned | base | Δ |
+| --- | --- | --- | --- |
+| mean_final | 0.4675 | 0.4798 | ▼ 0.0123 |
+| stdev_final | 0.2184 | 0.2201 | ▼ 0.0017 |
+
+**fine-tuned convergence curve**: [0.7475, 0.4255]
+
+**base convergence curve**: [0.6425, 0.462]
+
+## Claim 4 — Held-out generalization (Pool D)
+
+| Metric | fine-tuned | base | Δ |
+| --- | --- | --- | --- |
+| mean_final | 0.5284 | 0.5565 | ▼ 0.0281 |
+| stdev_final | 0.071 | 0.0696 | ▲ 0.0014 |
+| ECE | 0.925 | 0.8187 | ▲ 0.1063 |
+| pearson_r (P2 breadth) | -0.3637 | 0.4951 | ▼ 0.8588 |

findings/20260426_083700/manifest.json

@@ -0,0 +1,14 @@
+{
+  "outdir": "eval_output",
+  "files": {
+    "results_finetuned": "eval_output/results_Yaswanth-Bolla_qwen-merged.json",
+    "results_base": "eval_output/results_Qwen_Qwen2.5-7B-Instruct.json",
+    "comparison_tables": "eval_output/comparison_tables.md",
+    "plots": [
+      "eval_output/mean_final_comparison.png",
+      "eval_output/stopping_distribution.png",
+      "eval_output/calibration.png",
+      "eval_output/convergence.png"
+    ]
+  }
+}

findings/20260426_083700/results_Qwen_Qwen2.5-7B-Instruct.json

@@ -0,0 +1,506 @@
+{
+  "claim1": {
+    "claim": "1_orchestrator_stopping",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.4495,
+      "stdev_final": 0.1976,
+      "mean_r_cross": 0.4412,
+      "mean_p1_steps": 16.62,
+      "mean_p2_steps": 5.62
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "model": {
+          "[0,4)": 9.999920000639996e-07,
+          "[4,6)": 9.999920000639996e-07,
+          "[6,8)": 0.24999900000799996,
+          "[8,10)": 9.999920000639996e-07,
+          "[10,13)": 9.999920000639996e-07,
+          "[13,17)": 0.24999900000799996,
+          "[17,25)": 0.24999900000799996,
+          "[25,inf)": 0.24999900000799996
+        },
+        "kl_pairwise": {}
+      },
+      "action_position[check_dependencies]": {
+        "model": {
+          "frequency": 1.0,
+          "mean_position": 3.12,
+          "median_position": 3.0,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "model": {
+          "ece": 0.5813,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 2,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 6,
+              "mean_conf": 0.942,
+              "accuracy": 0.167
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "model": {
+          "pearson_r": 0.0,
+          "n": 6
+        }
+      },
+      "n_episodes_per_condition": {
+        "model": 8
+      },
+      "n_total": 8
+    }
+  },
+  "claim2": {
+    "claim": "2_r_cross_ablation",
+    "n_per_condition": 8,
+    "aggregate": {
+      "r_cross_on": {
+        "n": 8,
+        "mean_final": 0.4738,
+        "stdev_final": 0.179,
+        "mean_r_cross": 0.4163,
+        "mean_p1_steps": 15.75,
+        "mean_p2_steps": 6.12,
+        "p2_steps_to_correct_patch": 5.86
+      },
+      "r_cross_off": {
+        "n": 8,
+        "mean_final": 0.4813,
+        "stdev_final": 0.183,
+        "mean_r_cross": 0.0,
+        "mean_p1_steps": 18,
+        "mean_p2_steps": 4.75,
+        "p2_steps_to_correct_patch": 4.29
+      }
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "r_cross_on": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.2499990000079999,
+          "[8,10)": 0.125,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 9.999920000639994e-07,
+          "[17,25)": 0.6249960000319997,
+          "[25,inf)": 9.999920000639994e-07
+        },
+        "r_cross_off": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 9.999920000639994e-07,
+          "[8,10)": 0.2499990000079999,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 9.999920000639994e-07,
+          "[17,25)": 0.6249960000319997,
+          "[25,inf)": 0.125
+        },
+        "kl_pairwise": {
+          "r_cross_on_vs_r_cross_off": 3.0206
+        }
+      },
+      "action_position[check_dependencies]": {
+        "r_cross_on": {
+          "frequency": 1.0,
+          "mean_position": 3,
+          "median_position": 3.0,
+          "n": 8
+        },
+        "r_cross_off": {
+          "frequency": 1.0,
+          "mean_position": 2.5,
+          "median_position": 2.0,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "r_cross_on": {
+          "ece": 0.8063,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 1,
+              "mean_conf": 0.85,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 7,
+              "mean_conf": 0.943,
+              "accuracy": 0.143
+            }
+          ],
+          "n": 8
+        },
+        "r_cross_off": {
+          "ece": 0.8125,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 1,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 1,
+              "mean_conf": 0.85,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 6,
+              "mean_conf": 0.942,
+              "accuracy": 0.0
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "r_cross_on": {
+          "pearson_r": -0.3637,
+          "n": 8,
+          "mean_breadth": 2.75,
+          "mean_confidence": 0.888
+        },
+        "r_cross_off": {
+          "pearson_r": -0.0747,
+          "n": 7,
+          "mean_breadth": 2.86,
+          "mean_confidence": 0.879
+        }
+      },
+      "n_episodes_per_condition": {
+        "r_cross_on": 8,
+        "r_cross_off": 8
+      },
+      "n_total": 16
+    }
+  },
+  "claim3": {
+    "claim": "3_convergence_proxy",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.4798,
+      "stdev_final": 0.2201,
+      "mean_r_cross": 0.4912,
+      "mean_p1_steps": 15.5,
+      "mean_p2_steps": 6.5
+    },
+    "convergence_curve": {
+      "model": [
+        0.6425,
+        0.462
+      ]
+    }
+  },
+  "claim4": {
+    "claim": "4_held_out_generalization",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.5565,
+      "stdev_final": 0.0696,
+      "mean_r_cross": 0.4875,
+      "mean_p1_steps": 11.75,
+      "mean_p2_steps": 7.38
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "model": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.4999970000239998,
+          "[8,10)": 0.125,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 0.125,
+          "[17,25)": 0.125,
+          "[25,inf)": 0.125
+        },
+        "kl_pairwise": {}
+      },
+      "action_position[check_dependencies]": {
+        "model": {
+          "frequency": 1.0,
+          "mean_position": 2,
+          "median_position": 2.0,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "model": {
+          "ece": 0.8187,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 1,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 1,
+              "mean_conf": 0.85,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 6,
+              "mean_conf": 0.95,
+              "accuracy": 0.0
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "model": {
+          "pearson_r": 0.4951,
+          "n": 7,
+          "mean_breadth": 3.43,
+          "mean_confidence": 0.886
+        }
+      },
+      "n_episodes_per_condition": {
+        "model": 8
+      },
+      "n_total": 8
+    }
+  }
+}

findings/20260426_083700/results_Yaswanth-Bolla_qwen-merged.json

@@ -0,0 +1,506 @@
+{
+  "claim1": {
+    "claim": "1_orchestrator_stopping",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.4537,
+      "stdev_final": 0.1904,
+      "mean_r_cross": 0.4662,
+      "mean_p1_steps": 15.75,
+      "mean_p2_steps": 6.5
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "model": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.2499990000079999,
+          "[8,10)": 9.999920000639994e-07,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 0.125,
+          "[17,25)": 0.6249960000319997,
+          "[25,inf)": 9.999920000639994e-07
+        },
+        "kl_pairwise": {}
+      },
+      "action_position[check_dependencies]": {
+        "model": {
+          "frequency": 1.0,
+          "mean_position": 2.75,
+          "median_position": 2.5,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "model": {
+          "ece": 0.8125,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 8,
+              "mean_conf": 0.938,
+              "accuracy": 0.125
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "model": {
+          "pearson_r": -0.4211,
+          "n": 8,
+          "mean_breadth": 2.62,
+          "mean_confidence": 0.912
+        }
+      },
+      "n_episodes_per_condition": {
+        "model": 8
+      },
+      "n_total": 8
+    }
+  },
+  "claim2": {
+    "claim": "2_r_cross_ablation",
+    "n_per_condition": 8,
+    "aggregate": {
+      "r_cross_on": {
+        "n": 8,
+        "mean_final": 0.4994,
+        "stdev_final": 0.1266,
+        "mean_r_cross": 0.585,
+        "mean_p1_steps": 13,
+        "mean_p2_steps": 7.25,
+        "p2_steps_to_correct_patch": 6.86
+      },
+      "r_cross_off": {
+        "n": 8,
+        "mean_final": 0.4867,
+        "stdev_final": 0.2132,
+        "mean_r_cross": 0.0,
+        "mean_p1_steps": 13,
+        "mean_p2_steps": 6.38,
+        "p2_steps_to_correct_patch": 7.29
+      }
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "r_cross_on": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.2499990000079999,
+          "[8,10)": 0.125,
+          "[10,13)": 0.2499990000079999,
+          "[13,17)": 0.125,
+          "[17,25)": 0.125,
+          "[25,inf)": 0.125
+        },
+        "r_cross_off": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.4999970000239998,
+          "[8,10)": 0.125,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 9.999920000639994e-07,
+          "[17,25)": 0.2499990000079999,
+          "[25,inf)": 0.125
+        },
+        "kl_pairwise": {
+          "r_cross_on_vs_r_cross_off": 4.3144
+        }
+      },
+      "action_position[check_dependencies]": {
+        "r_cross_on": {
+          "frequency": 1.0,
+          "mean_position": 2.75,
+          "median_position": 3.0,
+          "n": 8
+        },
+        "r_cross_off": {
+          "frequency": 1.0,
+          "mean_position": 2.5,
+          "median_position": 2.5,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "r_cross_on": {
+          "ece": 0.8312,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 1,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 7,
+              "mean_conf": 0.95,
+              "accuracy": 0.0
+            }
+          ],
+          "n": 8
+        },
+        "r_cross_off": {
+          "ece": 0.6875,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 1,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 1,
+              "mean_conf": 0.85,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 6,
+              "mean_conf": 0.942,
+              "accuracy": 0.167
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "r_cross_on": {
+          "pearson_r": 0.0,
+          "n": 7
+        },
+        "r_cross_off": {
+          "pearson_r": -0.0202,
+          "n": 7,
+          "mean_breadth": 3.29,
+          "mean_confidence": 0.879
+        }
+      },
+      "n_episodes_per_condition": {
+        "r_cross_on": 8,
+        "r_cross_off": 8
+      },
+      "n_total": 16
+    }
+  },
+  "claim3": {
+    "claim": "3_convergence_proxy",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.4675,
+      "stdev_final": 0.2184,
+      "mean_r_cross": 0.4662,
+      "mean_p1_steps": 17.38,
+      "mean_p2_steps": 4.88
+    },
+    "convergence_curve": {
+      "model": [
+        0.7475,
+        0.4255
+      ]
+    }
+  },
+  "claim4": {
+    "claim": "4_held_out_generalization",
+    "n": 8,
+    "aggregate": {
+      "n": 8,
+      "mean_final": 0.5284,
+      "stdev_final": 0.071,
+      "mean_r_cross": 0.5,
+      "mean_p1_steps": 17.88,
+      "mean_p2_steps": 5.25
+    },
+    "behavioral": {
+      "stopping_distribution": {
+        "model": {
+          "[0,4)": 9.999920000639994e-07,
+          "[4,6)": 9.999920000639994e-07,
+          "[6,8)": 0.125,
+          "[8,10)": 9.999920000639994e-07,
+          "[10,13)": 9.999920000639994e-07,
+          "[13,17)": 0.125,
+          "[17,25)": 0.7499950000399997,
+          "[25,inf)": 9.999920000639994e-07
+        },
+        "kl_pairwise": {}
+      },
+      "action_position[check_dependencies]": {
+        "model": {
+          "frequency": 1.0,
+          "mean_position": 1.75,
+          "median_position": 2.0,
+          "n": 8
+        }
+      },
+      "confidence_calibration": {
+        "model": {
+          "ece": 0.925,
+          "buckets": [
+            {
+              "bucket_low": 0.0,
+              "bucket_high": 0.1,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.1,
+              "bucket_high": 0.2,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.2,
+              "bucket_high": 0.3,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.3,
+              "bucket_high": 0.4,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.4,
+              "bucket_high": 0.5,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.5,
+              "bucket_high": 0.6,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.6,
+              "bucket_high": 0.7,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.7,
+              "bucket_high": 0.8,
+              "n": 0,
+              "mean_conf": 0.0,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.8,
+              "bucket_high": 0.9,
+              "n": 2,
+              "mean_conf": 0.85,
+              "accuracy": 0.0
+            },
+            {
+              "bucket_low": 0.9,
+              "bucket_high": 1.0,
+              "n": 6,
+              "mean_conf": 0.95,
+              "accuracy": 0.0
+            }
+          ],
+          "n": 8
+        }
+      },
+      "p2_search_breadth_correlation": {
+        "model": {
+          "pearson_r": -0.3637,
+          "n": 8,
+          "mean_breadth": 2.38,
+          "mean_confidence": 0.875
+        }
+      },
+      "n_episodes_per_condition": {
+        "model": 8
+      },
+      "n_total": 8
+    }
+  }
+}