def codeforge_reward_fn(prompts, completions, *, task_meta, **kwargs): """Score completions using CodeForge's full pipeline.""" from codeforge.sandbox.sandbox import run_sandbox from codeforge.grounder import ground from codeforge.grader import compute_reward rewards = [] for completion, meta in zip(completions, task_meta): # Extract code (strip markdown blocks if model adds them despite prompt) code = completion if "```python" in code: code = code.split("```python")[1].split("```")[0].strip() elif "```" in code: code = code.split("```")[1].split("```")[0].strip() files = {"main.py": code} # 1. Sandbox: ruff + mypy + pytest sandbox_files = {**files, **meta["hidden_tests"]} try: # CodeForge run_sandbox signature: result = run_sandbox(files=sandbox_files, tools=meta["tools"]) sandbox_score = result.composite_score except Exception as e: print(f"Sandbox error: {e}") sandbox_score = 0.0 # 2. AST Grounding: verify all imports/attrs are real try: report = ground("\n".join(files.values())) groundedness = report.groundedness except Exception as e: print(f"Grounding error: {e}") groundedness = 0.0 # 3. Brier-calibrated reward reward = compute_reward( sandbox_score=sandbox_score, groundedness=groundedness, confidence=0.7, # Default fixed confidence since model doesn't declare it here ) rewards.append(reward) return rewards