logbook-files/cap_repro_results.json

{
  "C1": {
    "claim": "68% execute at most 10 steps before human intervention",
    "denominator_N": 60,
    "count_le10_steps": 41,
    "recomputed_pct_le10": 68.33,
    "paper_reported_pct_le10": 68,
    "recomputed_pct_le5": 46.67,
    "paper_reported_pct_le5": 47,
    "figure7a_counts_sum_to_N": true,
    "alldata_N": 89,
    "alldata_pct_le10": 59.55,
    "verdict_pass": true
  },
  "C2a": {
    "claim": "70% rely on prompting off-the-shelf models rather than weight tuning",
    "denominator_N": 20,
    "count_offshelf_prompting": 14,
    "recomputed_pct": 70.0,
    "paper_reported_pct": 70,
    "figure5_counts_sum_to_N": true,
    "verdict_pass": true
  },
  "C2b": {
    "claim": "74% depend primarily on human evaluation",
    "denominator_N": 31,
    "multi_select": true,
    "count_manual_human_in_loop": 23,
    "recomputed_pct": 74.19,
    "paper_reported_pct": 74,
    "is_primary_method": true,
    "alldata_manual_pct": 63.3,
    "verdict_pass": true
  },
  "C3": {
    "claim": "Reliability is the dominant challenge in production agent deployment",
    "denominator_N": 29,
    "core_technical_perf_count": 11,
    "recomputed_pct": 37.93,
    "paper_reported_pct": 38,
    "rank1_ranking": [
      [
        "Core Technical Performance (reliability/robustness/scalability)",
        11,
        37.9
      ],
      [
        "Compliance and User Trust",
        5,
        17.2
      ],
      [
        "Transparency and Governance",
        1,
        3.4
      ]
    ],
    "core_exceeds_compliance_plus_governance": true,
    "definition": "Core Technical Performance encompasses robustness, reliability, and scalability (Table: 'Robustness and reliability - ensuring consistent, correct behavior')",
    "verdict_pass": true
  },
  "scope": {
    "total_valid_responses": 306,
    "deployed_subset": 86,
    "case_studies": 20,
    "high_level_domains_count": 8,
    "table1_outlier_domains_count": 16,
    "enumerated_domains_total": 24,
    "claimed_distinct_domains": 26,
    "enumerated_vs_claimed_delta": 2,
    "note": "8 high-level + 16 Table-1 outliers (mentioned exactly once) = 24 explicitly enumerated; the paper's stated total of 26 distinct domains is consistent with 2 additional Other sub-domains that were mentioned more than once (Table 1 lists only 'mentioned only once' domains)."
  }
}