#!/usr/bin/env python3 """ Create Combined Goodhart Gap Benchmark Combines: 1. cgrt-consensus-5model data (61,678 problems, ~$1000 in API calls) 2. Programmatic multi-domain problems (101 problems) Focus: Disagreement cases where models show the "Goodhart Gap" - understanding procedures but failing execution. """ import json from pathlib import Path from collections import defaultdict # Paths CONSENSUS_DATA = Path("/home/adam/Mojo/Research/experiments/tau-bench/cgrt/data/consensus_cli_labels_enriched.jsonl") PROGRAMMATIC_DATA = Path("data/test.jsonl") OUTPUT_DIR = Path("data") def load_consensus_data(): """Load the cgrt-consensus-5model dataset.""" data = [] with open(CONSENSUS_DATA) as f: for line in f: data.append(json.loads(line)) return data def load_programmatic_data(): """Load our programmatic test problems.""" data = [] with open(PROGRAMMATIC_DATA) as f: for line in f: data.append(json.loads(line)) return data def extract_goodhart_examples(consensus_data): """ Extract examples that demonstrate the Goodhart Gap: - Models that show work but get wrong answers - Disagreement between models despite similar reasoning - Contested problems where execution differs """ goodhart_examples = [] for item in consensus_data: # Focus on disagreement cases if item.get('all_agree', True): continue # Get all model answers answers = {} for model in ['claude', 'codex', 'gemini', 'deepseek', 'qwen']: ans_key = f'{model}_answer' resp_key = f'{model}_response' if ans_key in item and resp_key in item: answers[model] = { 'answer': item[ans_key], 'response': item[resp_key] } if len(answers) < 3: continue # Classify the type of disagreement unique_answers = set(a['answer'] for a in answers.values() if a['answer']) example = { 'id': f"consensus_{item['idx']}", 'source': 'cgrt-consensus-5model', 'question': item['question'], 'majority_answer': item.get('majority_answer', ''), 'agreement_score': item.get('agreement_score', 0), 'consensus_tier': item.get('consensus_tier', 'unknown'), 'num_unique_answers': len(unique_answers), 'model_responses': answers, 'outlier_models': item.get('outlier_models', []), 'difficulty_signal': item.get('difficulty_signal', 0), 'goodhart_type': classify_goodhart_type(answers, item) } goodhart_examples.append(example) return goodhart_examples def classify_goodhart_type(answers, item): """Classify the type of Goodhart Gap exhibited.""" # Check if models show similar reasoning but different answers responses = [a['response'] for a in answers.values() if a['response']] answer_set = set(a['answer'] for a in answers.values() if a['answer']) if len(answer_set) == 1: return 'agreement' # Shouldn't happen in disagreement set tier = item.get('consensus_tier', '') if tier == 'contested': return 'execution_divergence' # Strong Goodhart Gap elif tier == 'bronze': return 'partial_agreement' elif tier == 'silver': return 'minor_disagreement' else: return 'calculation_error' def create_combined_dataset(): """Create the combined benchmark dataset.""" print("Loading consensus data...") consensus_data = load_consensus_data() print(f" Loaded {len(consensus_data)} consensus problems") print("\nLoading programmatic data...") programmatic_data = load_programmatic_data() print(f" Loaded {len(programmatic_data)} programmatic problems") print("\nExtracting Goodhart Gap examples...") goodhart_examples = extract_goodhart_examples(consensus_data) print(f" Found {len(goodhart_examples)} disagreement cases") # Categorize by tier by_tier = defaultdict(list) for ex in goodhart_examples: by_tier[ex['consensus_tier']].append(ex) print("\n By tier:") for tier, examples in sorted(by_tier.items()): print(f" {tier}: {len(examples)}") # Create output datasets OUTPUT_DIR.mkdir(exist_ok=True) # 1. Full disagreement dataset print("\nWriting full disagreement dataset...") with open(OUTPUT_DIR / "goodhart_disagreements.jsonl", 'w') as f: for ex in goodhart_examples: f.write(json.dumps(ex) + '\n') print(f" Wrote {len(goodhart_examples)} examples to goodhart_disagreements.jsonl") # 2. Contested subset (strongest Goodhart Gap cases) contested = by_tier.get('contested', []) print(f"\nWriting contested subset ({len(contested)} examples)...") with open(OUTPUT_DIR / "goodhart_contested.jsonl", 'w') as f: for ex in contested: f.write(json.dumps(ex) + '\n') # 3. Combined test set (contested + programmatic) print("\nCreating combined test set...") combined = [] # Add contested examples (reformatted for evaluation) for ex in contested: combined.append({ 'id': ex['id'], 'domain': 'math_consensus', 'problem': ex['question'], 'correct_answer': ex['majority_answer'], 'source': 'cgrt-consensus-5model', 'consensus_tier': ex['consensus_tier'], 'model_responses': ex['model_responses'], 'difficulty': 'hard', 'steps': 3 # Estimate }) # Add programmatic examples for ex in programmatic_data: ex['source'] = 'programmatic' combined.append(ex) with open(OUTPUT_DIR / "combined_test.jsonl", 'w') as f: for ex in combined: f.write(json.dumps(ex) + '\n') print(f" Wrote {len(combined)} examples to combined_test.jsonl") # 4. Summary statistics summary = { 'total_consensus_problems': len(consensus_data), 'total_disagreements': len(goodhart_examples), 'contested_count': len(contested), 'programmatic_count': len(programmatic_data), 'combined_test_count': len(combined), 'by_tier': {k: len(v) for k, v in by_tier.items()}, 'sources': { 'cgrt-consensus-5model': 'https://huggingface.co/datasets/Adam1010/cgrt-consensus-5model', 'programmatic': 'Python-generated multi-domain problems' }, 'cost_estimate': '$1000+ in API calls for consensus data' } with open(OUTPUT_DIR / "combined_summary.json", 'w') as f: json.dump(summary, f, indent=2) print("\n" + "="*50) print("COMBINED DATASET SUMMARY") print("="*50) print(f"Consensus source problems: {len(consensus_data)}") print(f"Disagreement cases: {len(goodhart_examples)}") print(f"Contested (strongest): {len(contested)}") print(f"Programmatic problems: {len(programmatic_data)}") print(f"Combined test set: {len(combined)}") print("="*50) return summary if __name__ == "__main__": create_combined_dataset()