Phase 4: CRMAPI with 3 drifts + scenarios E1/E2/E3

scenarios.py

@@ -1,4 +1,124 @@
-"""Scenario definitions (E1-E3 easy, M1-M3 medium, H1-H3 hard).
+"""Scenario definitions (E1-E3 easy). Medium/hard added in Phase 11."""
+from __future__ import annotations
 
-Will be filled in Phase 4 (easy) and Phase 11 (medium+).
-"""
+from models import DriftEvent
+
+
+SCENARIOS: dict[str, dict] = {
+    "E1_onboard_new_hire": {
+        "difficulty": "easy",
+        "max_steps": 8,
+        "token_budget": 4000,
+        "task_description": (
+            "A new employee named Priya Sharma (priya@company.com) starts on "
+            "Monday April 27. Send her a welcome email at priya@company.com "
+            "with subject containing 'welcome', then create a calendar event "
+            "titled 'New Hire Orientation' on Monday April 27 at 10:00 AM for "
+            "1 hour, inviting both Priya and her manager alex@company.com."
+        ),
+        "success_criteria": [
+            "Welcome email sent to priya@company.com with 'welcome' in subject",
+            "Calendar event created for Monday April 27 10am with both Priya "
+            "and Alex attending",
+        ],
+        "seed_data": {
+            "mail": {"messages": []},
+            "calendar": {"events": []},
+        },
+        "drift_plan": [
+            DriftEvent(
+                tool="calendar",
+                endpoint="create_event",
+                kind="field_rename",
+                fires_at_step=3,
+                details={"from": "attendees", "to": "participants"},
+            ),
+        ],
+        "ground_truth_final_state": {
+            "mail.sent_count": 1,
+            "mail.last_sent_to": "priya@company.com",
+            "mail.last_subject_contains_welcome": True,
+            "calendar.events_count": 1,
+            "calendar.last_event_has_both_attendees": True,
+        },
+        "required_tools": ["mail", "calendar"],
+    },
+
+    "E2_meeting_invite_blast": {
+        "difficulty": "easy",
+        "max_steps": 6,
+        "token_budget": 4000,
+        "task_description": (
+            "Send calendar invite emails to three team members for an "
+            "all-hands meeting at 3:00 PM Friday. Recipients: "
+            "alex@company.com, jordan@company.com, sam@company.com. "
+            "Subject: 'All-Hands: Friday 3pm'. "
+            "Body should mention the time and agenda."
+        ),
+        "success_criteria": [
+            "Three emails sent to the three listed recipients",
+            "All three emails have subject containing 'All-Hands'",
+        ],
+        "seed_data": {
+            "mail": {"messages": []},
+        },
+        "drift_plan": [
+            DriftEvent(
+                tool="mail",
+                endpoint="send_message",
+                kind="endpoint_deprecation",
+                fires_at_step=1,
+                details={"replacement": "messages.send"},
+            ),
+        ],
+        "ground_truth_final_state": {
+            "mail.sent_count": 3,
+            "mail.sent_to_all_three_recipients": True,
+        },
+        "required_tools": ["mail"],
+    },
+
+    "E3_customer_lookup": {
+        "difficulty": "easy",
+        "max_steps": 8,
+        "token_budget": 4000,
+        "task_description": (
+            "A customer emailed support asking about their account: "
+            "bob@customer.com. Look them up in the CRM by email, retrieve "
+            "their full contact record, and update their status field to "
+            "'support_in_progress'. Report back their company name in your "
+            "final summary."
+        ),
+        "success_criteria": [
+            "Customer bob@customer.com found in CRM",
+            "Contact status updated to 'support_in_progress'",
+            "Completion summary mentions the customer's company",
+        ],
+        "seed_data": {
+            "crm": {
+                "contacts": [
+                    {"contact_id": "c_1", "customer_email": "alice@customer.com",
+                     "name": "Alice Nguyen", "company": "Acme Corp", "status": "active"},
+                    {"contact_id": "c_2", "customer_email": "bob@customer.com",
+                     "name": "Bob Taylor", "company": "Globex Industries", "status": "active"},
+                    {"contact_id": "c_3", "customer_email": "carol@customer.com",
+                     "name": "Carol Davis", "company": "Initech", "status": "inactive"},
+                ],
+            },
+        },
+        "drift_plan": [
+            DriftEvent(
+                tool="crm",
+                endpoint=None,
+                kind="field_rename",
+                fires_at_step=2,
+                details={"from": "customer_email", "to": "email_address"},
+            ),
+        ],
+        "ground_truth_final_state": {
+            "crm.contact_c_2_status": "support_in_progress",
+            "complete_summary_mentions_company": True,
+        },
+        "required_tools": ["crm"],
+    },
+}

tests/test_crm.py

@@ -0,0 +1,142 @@
+"""CRMAPI acceptance tests — Phase 4."""
+from __future__ import annotations
+
+import pytest
+
+from models import DriftEvent
+from tools.crm import CRMAPI
+
+
+def _fresh_crm() -> CRMAPI:
+    return CRMAPI(seed_data={
+        "contacts": [
+            {"contact_id": "c_1", "customer_email": "alice@customer.com",
+             "name": "Alice Nguyen", "company": "Acme Corp", "status": "active"},
+            {"contact_id": "c_2", "customer_email": "bob@customer.com",
+             "name": "Bob Taylor", "company": "Globex Industries", "status": "active"},
+            {"contact_id": "c_3", "customer_email": "carol@customer.com",
+             "name": "Carol Davis", "company": "Initech", "status": "inactive"},
+        ]
+    })
+
+
+def test_baseline_search_contacts() -> None:
+    crm = _fresh_crm()
+    by_email = crm.call("search_contacts", {"customer_email": "bob@customer.com"})
+    assert by_email.ok is True
+    assert by_email.status == 200
+    assert by_email.body is not None
+    assert by_email.body["total"] == 1
+    assert by_email.body["contacts"][0]["name"] == "Bob Taylor"
+    assert by_email.body["contacts"][0]["customer_email"] == "bob@customer.com"
+
+    by_name = crm.call("search_contacts", {"name": "Alice"})
+    assert by_name.ok is True
+    assert by_name.body["total"] == 1
+    assert by_name.body["contacts"][0]["contact_id"] == "c_1"
+
+    all_contacts = crm.call("search_contacts", {})
+    assert all_contacts.ok is True
+    assert all_contacts.body["total"] == 3
+
+
+def test_baseline_create_and_get() -> None:
+    crm = _fresh_crm()
+    created = crm.call("create_contact", {
+        "customer_email": "dan@customer.com",
+        "name": "Dan Ellis",
+        "company": "DEMO Corp",
+    })
+    assert created.ok is True
+    assert created.status == 200
+    cid = created.body["contact_id"]
+    assert cid.startswith("c_")
+
+    got = crm.call("get_contact", {"contact_id": cid})
+    assert got.ok is True
+    assert got.status == 200
+    assert got.body is not None
+    assert got.body["name"] == "Dan Ellis"
+    assert got.body["customer_email"] == "dan@customer.com"
+    assert got.body["company"] == "DEMO Corp"
+
+
+def test_drift_field_rename_crm_tool_wide() -> None:
+    crm = _fresh_crm()
+    event = DriftEvent(
+        tool="crm", endpoint=None, kind="field_rename",
+        fires_at_step=1, details={},
+    )
+    crm.apply_drift(event)
+
+    for ep in ("search_contacts", "get_contact", "create_contact", "update_contact"):
+        schema = crm.get_schema(ep)
+        assert "customer_email" not in schema["params"]
+        assert "customer_email" not in schema["required"]
+        assert "customer_email" not in schema["response_shape"]
+
+    assert "email_address" in crm.get_schema("search_contacts")["params"]
+    assert "email_address" in crm.get_schema("create_contact")["required"]
+    assert "email_address" in crm.get_schema("get_contact")["response_shape"]
+
+    bad = crm.call("search_contacts", {"customer_email": "bob@customer.com"})
+    assert bad.ok is False
+    assert bad.status == 400
+
+    ok = crm.call("search_contacts", {"email_address": "bob@customer.com"})
+    assert ok.ok is True
+    assert ok.status == 200
+    assert ok.body is not None
+    assert ok.body["total"] == 1
+    contact = ok.body["contacts"][0]
+    assert "email_address" in contact
+    assert "customer_email" not in contact
+    assert contact["email_address"] == "bob@customer.com"
+
+
+def test_drift_rate_limit_tightening() -> None:
+    crm = _fresh_crm()
+    event = DriftEvent(
+        tool="crm", endpoint=None, kind="rate_limit_tightening",
+        fires_at_step=1, details={},
+    )
+    crm.apply_drift(event)
+
+    r1 = crm.call("search_contacts", {})
+    assert r1.ok is True
+
+    r2 = crm.call("search_contacts", {})
+    assert r2.ok is True
+
+    r3 = crm.call("search_contacts", {})
+    assert r3.ok is False
+    assert r3.status == 429
+
+
+def test_drift_endpoint_deprecation_update_contact() -> None:
+    crm = _fresh_crm()
+    event = DriftEvent(
+        tool="crm", endpoint="update_contact", kind="endpoint_deprecation",
+        fires_at_step=1, details={"replacement": "contacts.patch"},
+    )
+    crm.apply_drift(event)
+
+    old = crm.call("update_contact", {"contact_id": "c_1", "status": "updated"})
+    assert old.ok is False
+    assert old.status == 410
+
+    new = crm.call("contacts.patch", {"contact_id": "c_1", "status": "updated"})
+    assert new.ok is True
+    assert new.status == 200
+    assert new.body is not None
+    assert new.body["status"] == "updated"
+
+
+def test_crm_unknown_drift_raises() -> None:
+    crm = _fresh_crm()
+    event = DriftEvent(
+        tool="crm", endpoint="search_contacts", kind="tool_removal",
+        fires_at_step=1, details={},
+    )
+    with pytest.raises(ValueError):
+        crm.apply_drift(event)

tests/test_scenarios.py

@@ -0,0 +1,65 @@
+"""Scenario structure tests — Phase 4."""
+from __future__ import annotations
+
+from models import DriftEvent
+from scenarios import SCENARIOS
+
+
+REQUIRED_KEYS = {
+    "difficulty",
+    "max_steps",
+    "token_budget",
+    "task_description",
+    "success_criteria",
+    "seed_data",
+    "drift_plan",
+    "ground_truth_final_state",
+    "required_tools",
+}
+
+
+def test_all_three_scenarios_present() -> None:
+    assert set(SCENARIOS.keys()) == {
+        "E1_onboard_new_hire",
+        "E2_meeting_invite_blast",
+        "E3_customer_lookup",
+    }
+
+
+def test_each_scenario_has_required_fields() -> None:
+    for name, sc in SCENARIOS.items():
+        missing = REQUIRED_KEYS - set(sc.keys())
+        assert not missing, f"{name} missing keys: {missing}"
+
+
+def test_drift_plans_contain_valid_events() -> None:
+    valid_tools = {"mail", "calendar", "crm", "chat", "docs"}
+    for name, sc in SCENARIOS.items():
+        plan = sc["drift_plan"]
+        assert isinstance(plan, list)
+        assert len(plan) > 0, f"{name}: drift_plan must have at least one event"
+        for d in plan:
+            assert isinstance(d, DriftEvent), f"{name}: non-DriftEvent in drift_plan"
+            assert d.tool in valid_tools
+            assert isinstance(d.fires_at_step, int)
+            assert d.fires_at_step >= 0
+
+
+def test_required_tools_match_scenario_intent() -> None:
+    assert SCENARIOS["E1_onboard_new_hire"]["required_tools"] == ["mail", "calendar"]
+    assert SCENARIOS["E2_meeting_invite_blast"]["required_tools"] == ["mail"]
+    assert SCENARIOS["E3_customer_lookup"]["required_tools"] == ["crm"]
+
+    for name, sc in SCENARIOS.items():
+        seed_keys = set(sc["seed_data"].keys())
+        req = set(sc["required_tools"])
+        assert seed_keys.issubset(req), (
+            f"{name}: seed_data keys {seed_keys} not a subset of required_tools {req}"
+        )
+
+
+def test_ground_truth_keys_non_empty() -> None:
+    for name, sc in SCENARIOS.items():
+        gt = sc["ground_truth_final_state"]
+        assert isinstance(gt, dict)
+        assert len(gt) > 0, f"{name}: ground_truth_final_state is empty"

tools/base.py

@@ -40,6 +40,13 @@ class BaseTool:
                 error=f"Endpoint '{endpoint}' is no longer available on {self.name}.",
             )
         schema = self.active_schemas[endpoint]
+        unknown = [p for p in params if p not in schema.params]
+        if unknown:
+            return ToolResponse(
+                ok=False,
+                status=400,
+                error=f"Unknown params on {endpoint}: {unknown}",
+            )
         missing = [p for p in schema.required if p not in params]
         if missing:
             return ToolResponse(

tools/crm.py

@@ -1,4 +1,213 @@
-"""CRMAPI — search_contacts, get_contact, create_contact, update_contact + drifts.
+"""CRMAPI — search_contacts, get_contact, create_contact, update_contact + 3 drift handlers."""
+from __future__ import annotations
 
-Will be filled in Phase 4.
-"""
+import copy
+
+from models import DriftEvent, ToolResponse
+from tools.base import BaseTool, EndpointSchema
+
+
+_SEARCH_CONTACTS_BASE = EndpointSchema(
+    name="search_contacts",
+    params={"customer_email": "str", "name": "str", "limit": "int"},
+    required=[],
+    response_shape={"contacts": "list", "total": "int"},
+    error_codes={200: "ok", 400: "bad_request", 429: "rate_limit"},
+)
+
+_GET_CONTACT_BASE = EndpointSchema(
+    name="get_contact",
+    params={"contact_id": "str"},
+    required=["contact_id"],
+    response_shape={
+        "contact_id": "str", "name": "str", "customer_email": "str",
+        "company": "str", "status": "str",
+    },
+    error_codes={200: "ok", 404: "not_found", 429: "rate_limit"},
+)
+
+_CREATE_CONTACT_BASE = EndpointSchema(
+    name="create_contact",
+    params={"customer_email": "str", "name": "str", "company": "str"},
+    required=["customer_email", "name"],
+    response_shape={
+        "contact_id": "str", "name": "str",
+        "customer_email": "str", "status": "str",
+    },
+    error_codes={200: "ok", 400: "bad_request", 422: "validation", 429: "rate_limit"},
+)
+
+_UPDATE_CONTACT_BASE = EndpointSchema(
+    name="update_contact",
+    params={
+        "contact_id": "str", "customer_email": "str", "name": "str",
+        "company": "str", "status": "str",
+    },
+    required=["contact_id"],
+    response_shape={"contact_id": "str", "status": "str"},
+    error_codes={200: "ok", 404: "not_found", 422: "validation", 429: "rate_limit"},
+)
+
+
+class CRMAPI(BaseTool):
+    name = "crm"
+    baseline_schemas = {
+        "search_contacts": _SEARCH_CONTACTS_BASE,
+        "get_contact": _GET_CONTACT_BASE,
+        "create_contact": _CREATE_CONTACT_BASE,
+        "update_contact": _UPDATE_CONTACT_BASE,
+    }
+
+    def __init__(self, seed_data: dict) -> None:
+        super().__init__()
+        self.contacts: list[dict] = list(seed_data.get("contacts", []))
+        self._next_id: int = len(self.contacts) + 1
+        self._call_count: int = 0
+        self._rate_limit_active: bool = False
+        self.handlers = {
+            "search_contacts": self._search_contacts,
+            "get_contact": self._get_contact,
+            "create_contact": self._create_contact,
+            "update_contact": self._update_contact,
+        }
+
+    def call(self, endpoint: str, params: dict) -> ToolResponse:
+        rl = self._check_rate_limit()
+        if rl is not None:
+            return rl
+        return super().call(endpoint, params)
+
+    def _check_rate_limit(self) -> ToolResponse | None:
+        self._call_count += 1
+        if self._rate_limit_active and self._call_count > 2:
+            return ToolResponse(ok=False, status=429, error="Rate limit exceeded")
+        return None
+
+    def _email_field_active(self) -> str:
+        any_schema = next(iter(self.active_schemas.values()))
+        if "email_address" in any_schema.response_shape or "email_address" in any_schema.params:
+            return "email_address"
+        for s in self.active_schemas.values():
+            if "email_address" in s.response_shape or "email_address" in s.params:
+                return "email_address"
+        return "customer_email"
+
+    def _normalize_email_in(self, params: dict) -> dict:
+        out = dict(params)
+        if "email_address" in out and "customer_email" not in out:
+            out["customer_email"] = out.pop("email_address")
+        return out
+
+    def _project_contact(self, contact: dict) -> dict:
+        out = dict(contact)
+        drifted = self._email_field_active() == "email_address"
+        if drifted and "customer_email" in out:
+            out["email_address"] = out.pop("customer_email")
+        elif not drifted and "email_address" in out:
+            out["customer_email"] = out.pop("email_address")
+        return out
+
+    def _search_contacts(self, params: dict) -> ToolResponse:
+        email = params.get("email_address") or params.get("customer_email")
+        name = params.get("name")
+
+        if email is None and name is None:
+            matched = list(self.contacts)
+        else:
+            matched = []
+            for c in self.contacts:
+                email_match = email is not None and c.get("customer_email", "").lower() == email.lower()
+                name_match = name is not None and name.lower() in c.get("name", "").lower()
+                if email_match or name_match:
+                    matched.append(c)
+
+        projected = [self._project_contact(c) for c in matched]
+        return ToolResponse(
+            ok=True, status=200,
+            body={"contacts": projected, "total": len(projected)},
+        )
+
+    def _get_contact(self, params: dict) -> ToolResponse:
+        cid = params["contact_id"]
+        for c in self.contacts:
+            if c.get("contact_id") == cid:
+                return ToolResponse(ok=True, status=200, body=self._project_contact(c))
+        return ToolResponse(ok=False, status=404, error=f"contact {cid} not found")
+
+    def _create_contact(self, params: dict) -> ToolResponse:
+        normalized = self._normalize_email_in(params)
+        cid = f"c_{self._next_id}"
+        self._next_id += 1
+        contact = {
+            "contact_id": cid,
+            "customer_email": normalized.get("customer_email", ""),
+            "name": normalized.get("name", ""),
+            "company": normalized.get("company", ""),
+            "status": normalized.get("status", "active"),
+        }
+        self.contacts.append(contact)
+        return ToolResponse(ok=True, status=200, body=self._project_contact(contact))
+
+    def _update_contact(self, params: dict) -> ToolResponse:
+        cid = params["contact_id"]
+        normalized = self._normalize_email_in(params)
+        for c in self.contacts:
+            if c.get("contact_id") == cid:
+                for field in ("customer_email", "name", "company", "status"):
+                    if field in normalized:
+                        c[field] = normalized[field]
+                return ToolResponse(ok=True, status=200, body=self._project_contact(c))
+        return ToolResponse(ok=False, status=404, error=f"contact {cid} not found")
+
+    def apply_drift(self, event: DriftEvent) -> None:
+        if event.kind == "field_rename":
+            targets = (
+                "search_contacts", "get_contact", "create_contact",
+                "update_contact", "contacts.patch",
+            )
+            for ep in targets:
+                if ep not in self.active_schemas:
+                    continue
+                schema = self.active_schemas[ep]
+                if "customer_email" in schema.params:
+                    schema.params.pop("customer_email")
+                    schema.params["email_address"] = "str"
+                if "customer_email" in schema.required:
+                    schema.required = [
+                        "email_address" if r == "customer_email" else r
+                        for r in schema.required
+                    ]
+                if "customer_email" in schema.response_shape:
+                    schema.response_shape.pop("customer_email")
+                    schema.response_shape["email_address"] = "str"
+            return
+
+        if event.kind == "rate_limit_tightening":
+            self._rate_limit_active = True
+            self._call_count = 0
+            return
+
+        if event.kind == "endpoint_deprecation" and event.endpoint == "update_contact":
+            drifted_rename = any(
+                "email_address" in s.params or "email_address" in s.response_shape
+                for s in self.active_schemas.values()
+            )
+            self.active_schemas.pop("update_contact", None)
+            self.handlers.pop("update_contact", None)
+            new_schema = copy.deepcopy(self.baseline_schemas["update_contact"])
+            new_schema.name = "contacts.patch"
+            if drifted_rename:
+                if "customer_email" in new_schema.params:
+                    new_schema.params.pop("customer_email")
+                    new_schema.params["email_address"] = "str"
+                if "customer_email" in new_schema.response_shape:
+                    new_schema.response_shape.pop("customer_email")
+                    new_schema.response_shape["email_address"] = "str"
+            self.active_schemas["contacts.patch"] = new_schema
+            self.handlers["contacts.patch"] = self._update_contact
+            return
+
+        raise ValueError(
+            f"CRMAPI.apply_drift: unhandled drift "
+            f"kind={event.kind!r} endpoint={event.endpoint!r}"
+        )