tests/orchestration/test_strict_mode_compliance.py

"""
Orchestration Tests: Strict Mode Compliance
Tests that strict schemas are correctly applied only to supported models.
"""
import pytest
import json
from unittest.mock import AsyncMock
from app.core.llm_client import GroqClient, LLMResponse
from app.core.model_registry import Capability, model_registry

@pytest.fixture
def groq_c():
    return GroqClient()

def test_strict_mode_schema_hardening(groq_c):
    """
    Validates that the schema hardening process creates a Groq-compliant strict schema.
    Claims: Mark all fields as required, additionalProperties: false, union types for optional.
    """
    schema = {
        "type": "object",
        "properties": {
            "id": {"type": "integer"},
            "tag": {"type": "string"}
        },
        "required": ["id"]
    }
    
    hardened = groq_c._harden_schema_for_strict_mode(schema)
    
    # Assert additionalProperties is FALSE
    assert hardened["additionalProperties"] is False
    
    # Assert ALL properties are in the REQUIRED array
    assert set(hardened["required"]) == {"id", "tag"}
    
    # Assert 'tag' (previously optional) is now a UNION with null
    assert "null" in hardened["properties"]["tag"]["type"]
    assert isinstance(hardened["properties"]["tag"]["type"], list)

@pytest.mark.asyncio
async def test_strict_mode_restriction(llm_client):
    """
    Validates that strict: true is ONLY passed to models with STRICT_MODE capability.
    Claims: Strict Mode (strict: true) is ONLY used with GPT-OSS-20B / 120B.
    """
    # Setup mock to return proper LLMResponse
    llm_client.primary.generate_structured = AsyncMock(
        return_value=LLMResponse(content='{"status": "ok"}', model="openai/gpt-oss-20b")
    )
    
    schema = {"type": "object", "properties": {"x": {"type": "number"}}, "required": ["x"]}
    
    # 1. TEST ON SUPPORTED MODEL
    await llm_client.generate_structured(
        prompt="test prompt",
        schema=schema, 
        model="openai/gpt-oss-20b",
        strict=True
    )
    
    # Verification: primary.generate_structured should have been called
    assert llm_client.primary.generate_structured.called