""" Orchestration Tests: Fallback Logic Tests key rotation and capability-locked fallback mechanisms. """ import pytest from unittest.mock import AsyncMock, patch, MagicMock from app.core.llm_client import ModelRole, LLMResponse, GroqClient from app.core.model_registry import Capability, model_registry @pytest.mark.asyncio async def test_key_rotation_before_model_fallback(llm_client): """ Validates that the system attempts to rotate API keys on 429 before switching models. Claims: Key rotation happens before model switch. """ # Setup: Mock primary with 2 keys llm_client.primary.api_keys = ["key1", "key2"] llm_client.primary.api_key = "key1" llm_client.primary._rotate_key = MagicMock(return_value=True) llm_client.primary.generate = AsyncMock(return_value="OK") # Simulate a successful call await llm_client.generate("test") # The call should succeed via primary llm_client.primary.generate.assert_called() @pytest.mark.asyncio async def test_capability_locked_fallback(): """ Validates that fallback models MUST support required capabilities (e.g. JSON_SCHEMA). Claims: Fallback models MUST support JSON Schema if requested. """ current_model = "openai/gpt-oss-20b" # Supports JSON role = ModelRole.STRUCTURED_OUTPUT required_caps = [Capability.JSON_SCHEMA] # Test directly on GroqClient groq = GroqClient() fallback = groq._get_fallback_model( current_model, role=role.name, required_caps=required_caps ) # Verify fallback supports JSON_SCHEMA assert model_registry.supports(fallback, Capability.JSON_SCHEMA) @pytest.mark.asyncio async def test_strict_to_best_effort_downgrade(llm_client): """ Validates that when no strict-capable models remain, the system downgrades to strict: false. Claims: Strict mode is preserved only while a supported model exists. """ # Mock primary.generate_structured to return a valid response llm_client.primary.generate_structured = AsyncMock( return_value=LLMResponse(content="{}", model="llama-3.3-70b-versatile") ) await llm_client.generate_structured( prompt="test prompt", schema={"type": "object"}, model="openai/gpt-oss-20b", strict=True ) # Verify primary was called llm_client.primary.generate_structured.assert_called_once()