app/agents/orchestrator.py

# app/agents/orchestrator.py - Main agent controller

"""Coordinates all agents for scam detection and honeypot engagement."""

from typing import Dict, Any, Optional, List
import time
import os
import re
import json
import random
import asyncio
import aiofiles
from datetime import datetime, timedelta
from fastapi import BackgroundTasks

from app.core.llm_client import LLMClient, BudgetExceeded
from app.agents.scam_detector import ScamDetector
from app.agents.persona_engine import PersonaEngine
from app.agents.intelligence_extractor import IntelligenceExtractor
from app.agents.conversation_manager import ConversationManager
from app.agents.adaptive_strategy import AdaptiveStrategyAgent

from app.intelligence.threat_engine import ThreatIntelligenceEngine
from app.intelligence.risk_scorer import RiskScoringEngine
from app.intelligence.campaign_tracker import CampaignTracker

from app.enforcement.police_api import CyberPoliceAPI, ActionRecommendationAPI

from app.config import settings
from app.utils.logger import AgentLogger
from app.enforcement.stakeholder_exports import StakeholderExporter
from app.utils.dossier_generator import dossier_generator
from app.utils.callback_client import GUVIMandatoryCallback
from app.utils.extractors import extract_all

from app.intelligence.graph_threat_intel import graph_intel
from app.intelligence.xai_reasoning import xai_explainer
from app.intelligence.scammer_profiler import scammer_profiler
from app.intelligence.enrichment_service import enrichment_service
from app.core.context import TurnContext, is_engagement_complete



class HoneypotOrchestrator:
    """
    Main Honeypot Agent Orchestrator.
    
    Coordinates all sub-agents to process scammer messages
    and generate intelligent honeypot responses.
    """
    
    def __init__(self):
        self.logger = AgentLogger("orchestrator")
        self.initialized = False
        
        # Core components
        self.llm_client: Optional[LLMClient] = None
        
        # Agents
        self.scam_detector: Optional[ScamDetector] = None
        self.persona_engine: Optional[PersonaEngine] = None
        self.intel_extractor: Optional[IntelligenceExtractor] = None
        self.conversation_manager: Optional[ConversationManager] = None
        self.adaptive_agent: Optional[AdaptiveStrategyAgent] = None
        
        # Advanced Threat Intelligence Modules
        self.threat_engine: Optional[ThreatIntelligenceEngine] = None
        self.risk_scorer: Optional[RiskScoringEngine] = None
        self.campaign_tracker: Optional[CampaignTracker] = None
        
        # Law enforcement
        self.police_api: Optional[CyberPoliceAPI] = None
        self.bank_api: Optional[ActionRecommendationAPI] = None
        
        # Ad-hoc profile store (if needed for session non-persistent memory)
        self.profiler = scammer_profiler
        self.enrichment_service = enrichment_service
        self.guvi_callback = GUVIMandatoryCallback()
        
        # Neuro-Symbolic Trace Cache (Real-Time Visualization Support)
        self.last_trace: Dict[str, Any] = {}
    
    async def initialize(self) -> None:
        """Initialize all agents and components."""
        self.logger.info("Initializing honeypot orchestrator")
        
        # Initialize LLM client
        self.llm_client = LLMClient()
        await self.llm_client.initialize()
        
        # Initialize agents
        self.scam_detector = ScamDetector(self.llm_client)
        self.persona_engine = PersonaEngine(self.llm_client)
        self.intel_extractor = IntelligenceExtractor(self.llm_client)
        self.conversation_manager = ConversationManager()
        self.adaptive_agent = AdaptiveStrategyAgent()
        
        # Initialize Intelligence Modules
        if settings.ENABLE_THREAT_INTELLIGENCE:
            self.threat_engine = ThreatIntelligenceEngine()
            self.risk_scorer = RiskScoringEngine()
            self.campaign_tracker = CampaignTracker()
        
        # Initialize law enforcement APIs
        if settings.ENABLE_LAW_ENFORCEMENT_API:
            self.police_api = CyberPoliceAPI()
            self.bank_api = ActionRecommendationAPI()
        
        self.initialized = True
        self.logger.info("Orchestrator initialized successfully")
    


    async def process_message(
        self,
        message: str,
        conversation_id: Optional[str] = None,
        sender_id: Optional[str] = None,
        auto_report: bool = True,
        background_tasks: Optional[BackgroundTasks] = None,
        client_ip: str = "Unknown",
        sender_role: str = "scammer",  # [SCORING] Explicit role support
        should_finalize: bool = False,  # [LATENCY] Turbo Mode Flag
        request_language: str = "hinglish"  # [FIX] Language from GUVI metadata
    ) -> Dict[str, Any]:
        """
        Process an incoming message through the OODA loop.
        
        Args:
            message: The scammer's message
            conversation_id: Unique session ID
            sender_id: Sender identifier (e.g. phone number)
            auto_report: Whether to automatically report to law enforcement
            background_tasks: FastAPI BackgroundTasks for non-blocking reporting
            should_finalize: Whether to run expensive forensic wrap-up (XAI/Enrichment)
        """
        start_time = time.time()
        client_ip = client_ip or "Unknown"
        
        if not self.initialized:
            await self.initialize()
            
        # --- TOKEN RESILIENCE: Input Capping ---
        from app.utils.token_utils import smart_truncate
        original_length = len(message)
        message = smart_truncate(message, max_chars=4000)
        if len(message) < original_length:
            self.logger.warning(f"Message truncated for token safety: {original_length} -> {len(message)} chars")
        
        # Reasoning Accumulator for Final Audit (Loaded from session for continuity)
        reasoning_traces = []

        # ------------------------------------------------------------------
        # 🔥 OPTIMIZATION: TURN CONTEXT (Request Scope)
        # Prevents redundant API calls and loops
        # ------------------------------------------------------------------
        ctx = TurnContext(session_id=conversation_id or "new", message=message, sender_role=sender_role)
        ctx.request_language = request_language  # [FIX] Store GUVI language preference


        # Get or create conversation (Auto-generates created_at if new)
        conversation = await self.conversation_manager.get_or_create(
            conversation_id, sender_id
        )
        conv_id = conversation["id"]
        
        # 🔥 [RISK 5] TRACE CONTINUITY: Load existing traces
        reasoning_traces = conversation.get("reasoning_history", [])
        
        # Link session to context for session-level budget enforcement
        ctx.session = conversation

        #  SOC SWITCHBOARD: OPTIONAL SECURITY SCAN
        # For honeypot, we EXPECT scam content - only check Turn 1 for prompt injection
        # This saves 1 API call per turn after first message
        message_count_early = len(conversation.get("history", [])) + 1
        
        if message_count_early <= 1:  # Only check first message
            try:
                 is_safe = await self.llm_client.check_safeguard(message, context=ctx)
                 if not is_safe:
                     #  HONEYPOT EXCEPTION: We EXPECT "Unsafe" (Fraud) content.
                     # Only block if it looks like a System Override/Prompt Injection attempt.
                     if "ignore previous instructions" in message.lower() or "system prompt" in message.lower():
                          self.logger.warning("Prompt Injection Blocked by SOC Safety Guard", conv_id=conv_id)
                          ctx.finalized = True
                          ctx.reply_mode = "HONEYPOT_ONLY"
                          return {
                             "status": "blocked",
                             "reason": "Security violation detected (Prompt Injection)",
                             "honeypot_response": {"message": "System unavailable.", "persona": "system"}
                          }
                     else:
                          self.logger.info("Safety Guard flagged content (likely Scam), proceeding as Honeypot...", conv_id=conv_id)
            except Exception as e:
                 self.logger.warning(f"Safety Guard Check Failed (LLM Error): {e}. Failing OPEN (Proceeding).", session_id=conv_id)
                 
        # Determine session start time for accurate metrics
        session_created_str = conversation.get("created_at", datetime.utcnow().isoformat())
        try:
             # Strip Z if present to maintain naive datetime consistency with utcnow()
             session_start_raw = session_created_str.replace("Z", "").split("+")[0]
             session_start = datetime.fromisoformat(session_start_raw)
        except:
             session_start = datetime.utcnow()
        duration_seconds = (datetime.utcnow() - session_start).total_seconds()
        
        # Determine message count (for phase)
        message_count = len(conversation.get("history", [])) + 1
        
        # Init Decision Variables (Scope Safety)
        persona = None
        phase = "hook" 
        scammer_behavior = None
        escalation_rec = {}
        is_fast_path = False
        is_scammer_repeating = False # [BUG FIX] Initialize early for Fast Path safety

        # [SCORING] Role-based logic: If sender is 'user', treat as non-scam or testing turn
        if sender_role == "user":
            self.logger.info("Scoring Override: Message from 'user' role detected. Fail-safe engagement mode.", session_id=conv_id)
            # We still run detection for safety, but we can nudge it
            scammer_behavior = {"behavior": "calm", "strategy": "neutral", "confidence": 0.0}
        
        # Step 1: Heuristic Pre-Check (Latency Elimination)
        # [OPTIMIZATION] FASTEST-PATH HEURISTIC (Turn 0)
        heuristic_detection = self.scam_detector.detect_heuristic(message)
        
        detection = None
        intelligence = {}
        
        if message_count <= 1 and heuristic_detection.get("confidence", 0) > 0.5:
             self.logger.info("FASTEST-PATH: Turn 0 High Confidence Regex. Skipping ALL LLMs.", session_id=conv_id)
             is_fast_path = True
             detection = heuristic_detection
             
             # [FIX] PRESERVE REGEX INTEL IN FAST-PATH
             # Previously: intelligence = {} (Wiped out all extracted data)
             intelligence = extract_all(message)
             
             # Calculate heuristic risk score for Fast Path
             intelligence["risk_score"] = 0
             if intelligence.get("upi_ids"): intelligence["risk_score"] += 20
             if intelligence.get("bank_accounts"): intelligence["risk_score"] += 30
             if intelligence.get("urls"): intelligence["risk_score"] += 10
             
             # Set Defaults for Fast Path
             # MUST be a dict for persona_engine compatibility
             scammer_behavior = {"behavior": "urgent", "strategy": "fast_path", "confidence": 1.0}
             
             ctx.persona_selected = True
             phase = "hook"
             
             # Prevent UnboundLocalError
             # SOC FIX: Sanitize merged_intel for Fast-Path compatibility
             merged_intel = (conversation.get("aggregated_intelligence") or {}).copy()
             merged_intel.setdefault("keywords", [])
             merged_intel.setdefault("upi_ids", [])
             merged_intel.setdefault("bank_accounts", [])
             
             # [FIX] Merge Regex Intelligence into Aggregated Intel for Fast Path
             # This ensures GUVI callback receives the extracted UPIs
             
             for k, v in intelligence.items():
                  if k in ["risk_score", "scam_confidence"]: continue
                  if v and isinstance(v, list):
                      current = merged_intel.get(k, [])
                      merged_intel[k] = list(set(current + v))
             
             # SOC FIX: Use taxonomy intelligence for persona selection in FASTEST-PATH
             persona_key = detection.get("persona", "worried_customer")
             persona = self.persona_engine.get_persona(persona_key)
             if not persona:
                 persona = self.persona_engine.get_persona("elderly_excited") or list(self.persona_engine.get_all_personas().values())[0]
             
             # Sync Graph Intel (Minimal - Keywords)
             if detection.get("matched_keywords"):
                 await self.conversation_manager.update_intelligence(conv_id, {"keywords": detection["matched_keywords"]})

        else:
            # Step 1 & 2: Parallel Detection & Extraction (Latency Optimization)
            # Run both independent agents simultaneously to halve processing time
            
            # --- NOVELTY TRIGGER CALCULATION ---
            existing_scam = conversation.get("scam_type")
            last_confidence = conversation.get("scam_confidence", 0.0)  # 🔥 FIX: Use stored confidence
            behavior_changed = False
            history = conversation.get("history", [])
            
            # [SCORING] Repetition Detection: If scammer is repeating demands, escalate agitation
            is_scammer_repeating = False
            if history:
                previous_msgs = [h.get("scammer_message", "").lower() for h in history[-2:]]
                current_msg_lower = message.lower()
                for prev in previous_msgs:
                    # Simple fuzzy match: same core words or same length/prefix
                    if prev == current_msg_lower or (len(prev) > 10 and prev[:15] == current_msg_lower[:15]):
                        is_scammer_repeating = True
                        self.logger.info("Scammer repetition detected, preparing for agitation escalation.", session_id=conv_id)
                        break

                last_turn = history[-1]
                # Check for behavior flip (Heuristic comparison of last behavior if available)
                # For now, we'll assume extraction_task will handle detailed behavior analysis later,
                # but we can check if the last turn was 'conclude' or 'escalate'
            
            if existing_scam: # If sticky, we can run extraction immediately with high confidence
                 # 🔥 SKIP LLM DETECTION entirely - use cached result
                 detection = {
                    "scam_type": existing_scam,
                    "confidence": conversation.get("scam_confidence", 0.9),
                    "is_scam": True,
                    "reasoning": "Sticky detection from session memory",
                    "source": "memory",
                    "matched_keywords": conversation.get("aggregated_intelligence", {}).get("keywords", [])
                 }
                 # 🔥 SKIP LLM EXTRACTION on most turns - regex is sufficient
                 # Only do LLM extraction on turn 1 and every 5th turn (handled inside extract())
                 intelligence = await self.intel_extractor.extract(
                    message, 
                    context=ctx, 
                    turn_count=message_count,
                    last_confidence=last_confidence,
                    current_confidence=conversation.get("scam_confidence", 0.9),
                    behavior_changed=behavior_changed
                 )
            else:
                 # If not sticky, we MUST run detection first to get 'current_confidence' for extraction novelty
                 try:
                     detection = await self.scam_detector.detect(message, context=ctx, turn_count=message_count)
                 except Exception as e:
                     self.logger.error(f"Detection FAIL: {e}", session_id=conv_id)
                     detection = {"is_scam": False, "confidence": 0.0, "scam_type": "error"}
                     
                 try:
                     intelligence = await self.intel_extractor.extract(
                        message, 
                        context=ctx, 
                        turn_count=message_count,
                        last_confidence=last_confidence,
                        current_confidence=detection.get("confidence", 0.0),
                        behavior_changed=behavior_changed
                    )
                 except Exception as e:
                     self.logger.error(f"Extraction FAIL: {e}", session_id=conv_id)
                     # Fallback to pure regex locally if agent died (Crash Safety)
                     intelligence = extract_all(message)
                     intelligence["risk_score"] = 0 # Default if scorer unreachable

        
        # ⚡ OPTIMIZATION: REGEX GUARD RULE
        # Rule: If Regex detects scam (Conf > 0.9), mark decided -> SKIP reasoning
        if detection.get("confidence", 0) > 0.9:
            ctx.mark_scam(detection["scam_type"], detection["confidence"])
            if detection.get("source") == "heuristic":
                self.logger.info("Fast-Path: Regex detected scam, skipping deep reasoning.", session_id=conv_id)

        # Core Logic Execution (Steps 3-5)
        # Initialize outputs for scope safety
        threat_intel = {}
        risk_explanation = []
        
        # Only run deep reasoning if NOT on Fast Path
        if not is_fast_path:
            # [Moved] Capture Native Reasoning Trace - now captured after full OODA loop
            
            #  Step 2.5: Update Graph Knowledge Base (Winner-Tier)
            graph_intel.add_intelligence(conv_id, intelligence)
            
            # Step 2.6: Prepare Merged Intel for Logic
            conv_intel = conversation.get("aggregated_intelligence") or {}
            merged_intel = {**conv_intel}
            
            for key in intelligence:
                if key in ["risk_score", "scam_confidence", "risk_level", "timeline"]: continue
                if intelligence[key]:
                    if key not in merged_intel: merged_intel[key] = []
                    for item in intelligence[key]:
                        if item not in merged_intel[key]: merged_intel[key].append(item)


                    
            if detection.get("matched_keywords"):
                # Update local merged copy
                if "keywords" not in merged_intel:
                    merged_intel["keywords"] = []
                for kw in detection["matched_keywords"]:
                    if kw not in merged_intel["keywords"]:
                        merged_intel["keywords"].append(kw)
                
                #  CRITICAL FIX: Persist keywords to Memory immediately
                # Otherwise they only exist in this temporary 'merged_intel' scope
                await self.conversation_manager.update_intelligence(
                    conv_id, 
                    {"keywords": detection["matched_keywords"]}
                )

            #  Step 3: Adaptive Analysis (Moved up for decisioning)
            scammer_behavior = await self.adaptive_agent.analyze_scammer_behavior(message)
            reasoning_traces.append(f"Behavioral Analysis: {scammer_behavior.get('strategy', 'Neutral')}")

            escalation_rec = self.adaptive_agent.get_escalation_recommendation(conversation, merged_intel)
            reasoning_traces.append(f"Escalation Logic: {escalation_rec.get('reason', 'Continue')}")
            
            # Step 4: Determine conversation phase (Explicit State Machine with Adaptive Input)
            phase = await self.conversation_manager.determine_phase(message_count, merged_intel)
            
            # Step 5: Select persona (Sticky Logic)
            
            # [OPTIMIZATION] HARD PERSONA LOCK
            # If persona exists, we reuse it. We DO NOT allow re-selection logic to run.
            existing_persona_key = conversation.get("persona")
            if existing_persona_key:
                ctx.persona_selected = True
                ctx.persona_locked = True
                persona = self.persona_engine.get_persona(existing_persona_key)
                if persona:
                     # Ensure the dict has the key so persona_engine knows which one it is
                     persona = {**persona, "selected_persona_key": existing_persona_key}
                     # Apply stored language from session to preserve language override
                     stored_lang = conversation.get("aggregated_intelligence", {}).get("request_language")
                     if stored_lang:
                         persona["language"] = stored_lang
                self.logger.info(f"PERSONA LOCKED: Reusing {existing_persona_key}", session_id=conv_id)
            
            if not ctx.persona_locked:
                persona = await self.persona_engine.select_persona(
                    scam_message=message,
                    scam_type=detection["scam_type"],
                    conversation_history=conversation.get("history"),
                    current_phase=phase,
                    session_id=conv_id,
                    context=ctx # Pass context
                )
                if persona:
                    ctx.persona_locked = True




        # Correctly get the internal key for logging/persistence
        persona_key = persona.get("selected_persona_key") if persona else existing_persona_key or "elderly_excited"
        if not persona_key: persona_key = "elderly_excited"
        
        # [FIX] CRASH PREVENTION: If persona is None (failed load/lock), hydrate it manually
        if not persona:
             self.logger.warning(f"Persona was None, hydrating from key: {persona_key}", session_id=conv_id)
             persona = self.persona_engine.get_persona(persona_key)
        
        # [FIX] LANGUAGE OVERRIDE: Apply GUVI request language to persona
        # This ensures responses match the requested language (English/Hindi/Hinglish)
        # Also persist language in session for subsequent turns with persona lock
        if request_language and persona:
            original_lang = persona.get("language", "hinglish")
            persona["language"] = request_language
            persona["original_language"] = original_lang  # Preserve for debugging
            # Store in session for persistence
            if ctx and hasattr(ctx, "session") and ctx.session:
                ctx.session["request_language"] = request_language
            self.logger.info(f"Language override: {original_lang} -> {request_language}", session_id=conv_id)
        elif ctx and hasattr(ctx, "session") and ctx.session:
            # Restore language from session if not in current request
            stored_lang = ctx.session.get("request_language")
            if stored_lang and persona:
                persona["language"] = stored_lang
        
        # [LIFECYCLE] Recalculate finalization state based on newly extracted intel
        # This ensures that if we just captured a UPI ID, we trigger XAI immediately.
        internal_should_finalize = should_finalize or is_engagement_complete(conversation, scam_detected=detection.get("is_scam", False))
        
        # Step 6: Generate response (With Adaptive Injection)
        
        # ⚡ OPTIMIZATION: ATTEMPT GUARD
        # Ensure we only try LLM generation ONCE per turn.
        # If we fail or have already attempted, fall back to static.
        if ctx.fast_chat_attempted:
             self.logger.warning("Fast Chat already attempted, forcing static fallback.", session_id=conv_id)
             response_text = self.persona_engine._static_response("", persona=persona, scam_type=detection["scam_type"])
        else:
             ctx.fast_chat_attempted = True
             try:
                 response_text = await self.persona_engine.generate_response(
                     scam_message=message,
                     persona=persona,
                     scam_type=detection["scam_type"],
                     conversation_history=conversation.get("history"),
                     current_phase=phase,
                     intelligence=merged_intel,
                     scammer_behavior=scammer_behavior, 
                     context=ctx, # Pass context for budget enforcement
                     is_repeating=is_scammer_repeating # [SCORING] Escalate if repeating
                 )
             except BudgetExceeded as e:
                 # GUARANTEED LOCAL FALLBACK on budget exhaustion
                 self.logger.warning(f"Budget exceeded during response generation: {e}. Using static fallback.", session_id=conv_id)
                 ctx.budget_exceeded = True
                 response_text = self.persona_engine._static_response("", persona=persona, scam_type=detection["scam_type"])

        

        
        # Step 7: Attribution & Link Encoding
        # Automatically append session ID to decoy links for 360-degree tracking
        if "/decoys/" in response_text:
            # Find decoy links and append ?sid=conv_id (or &sid= if ? exists)
            def encode_link(match):
                link = match.group(0)
                sep = "&" if "?" in link else "?"
                return f"{link}{sep}sid={conv_id}"
            
            response_text = re.sub(r'https?://[^\s<>"]+/decoys/[^\s<>"]+', encode_link, response_text)
            # Also handle relative paths if any (for internal simulation logs)
            response_text = re.sub(r'(?<!http://)(?<!https://)/decoys/[^\s<>"]+', encode_link, response_text)
        
        # Step 8: Threat intelligence analysis
        # threat_intel initialized above
        risk_score = 0.0
        
        # [SCORING] Populate reasoning from detection
        if detection.get("reasoning"):
            reasoning_traces.append(f"Detection Reasoning: {detection['reasoning']}")
            # Step 8.1 - 8.3: Threat Analysis & Campaign Tracking
            if self.threat_engine:
                threat_intel = await self.threat_engine.analyze(
                    detection["scam_type"],
                    merged_intel,
                    detection["confidence"]
                )
                
                # Track campaign
                if self.campaign_tracker:
                    self.campaign_tracker.track(
                        threat_intel["campaign_id"],
                        detection["scam_type"],
                        merged_intel
                    )
            else:
                threat_intel = {"campaign_id": "none", "severity": "MEDIUM", "scam_pattern": "untracked"}
            
            #  Step 8.4: Intelligence Enrichment
            # ⚡ OPTIMIZATION: TURBO MODE - ONLY RUN ON FINALIZATION
            enrichment_data = {}
            if settings.ENABLE_THREAT_INTELLIGENCE and self.enrichment_service and internal_should_finalize:
                 from app.intelligence.mitre_mapper import mitre_mapper
                 if detection.get("risk_indicators"):
                     threat_intel["mitre_ttps"] = mitre_mapper.map_tactics(detection["risk_indicators"])
                     threat_intel["mitre_campaign"] = mitre_mapper.get_campaign_ttps(detection["scam_type"])
                 
                 enrichment_data = await self.enrichment_service.enrich_intelligence(merged_intel)
                 threat_intel["enrichment"] = enrichment_data
                 if enrichment_data.get("reputation_alerts"):
                     risk_explanation.extend(enrichment_data["reputation_alerts"])
            else:
                 pass # Heuristic only path
            
            # Calculate risk score (Force Heuristic Mode if Finalized)
            # Calculate risk score (Force Heuristic Mode if Finalized)
            try:
                if self.risk_scorer:
                    # Pass None for llm_client if finalized to strictly valid LLM usage
                    run_llm = self.llm_client if not ctx.finalized else None
                    risk_score, risk_explanation = await self.risk_scorer.calculate_risk_score(
                        message,
                        detection.get("scam_type", "unknown"),
                        detection.get("confidence", 0.0),
                        merged_intel,
                        detection.get("matched_keywords", []),
                        llm_client=run_llm 
                    )
                else:
                    # [FAST PATH] Fallback to detector confidence if scorer disabled
                    risk_score = detection.get("confidence", 0.0)
                    risk_explanation = [f"Direct classification: {detection.get('scam_type', 'unknown')}"]
            except Exception as e:
                self.logger.error(f"Risk Scorer Failed: {e}", session_id=conv_id)
                risk_score = detection.get("confidence", 0.0)
                risk_explanation = ["Risk scoring fallback due to system error"]

            #  Step 8.5: Enrich with Graph Data (Winner-Tier)
            lookup_entity = (merged_intel.get("phone_numbers") or [message])[0]
            if merged_intel.get("upi_ids") and len(merged_intel["upi_ids"]) > 0:
                lookup_entity = merged_intel["upi_ids"][0]
                
            try:
                campaign_info = graph_intel.get_campaign_info(lookup_entity)
                if campaign_info.get("campaign_id"):
                    threat_intel["campaign_id"] = campaign_info["campaign_id"]
                    threat_intel["cluster_size"] = campaign_info["cluster_size"]
                    threat_intel["related_entities_count"] = len(campaign_info.get("related_entities", []))
            except Exception: pass
            
            #  Step 8.5.5: Adversary Profiling
            try:
                scammer_behavior_profile = self.profiler.analyze_behavior(message)
                scammer_id = self.profiler.generate_scammer_id(merged_intel)
                threat_intel["scammer_id"] = scammer_id
                threat_intel["behavior_metrics"] = scammer_behavior_profile
                
                # Save profile state
                self.profiler.create_profile(scammer_id, merged_intel, scammer_behavior_profile, detection["scam_type"])
            except Exception as e:
                self.logger.error(f"Profiler Failed: {e}", session_id=conv_id)
                scammer_behavior_profile = {"strategy": "unknown"}
            
            #  Step 8.6: Generate XAI Reasoning (Winner-Tier)
            # ⚡ OPTIMIZATION: TURBO MODE - ONLY RUN ON FINALIZATION
            # This moves ~4-5s of latency to the final reporting step only
            if settings.ENABLE_LLM_RESPONSES and self.llm_client and internal_should_finalize:
                 try:
                     xai_explanation = await xai_explainer.generate_explanation(
                         self.llm_client, message, detection, risk_score, merged_intel
                     )
                     risk_explanation.extend(xai_explanation)
                 except Exception as e:
                     self.logger.error(f"XAI Failed: {e}", session_id=conv_id)

        # SOC FIX: Kill Switch moved after enrichment/XAI for full trace capture
        ctx.finalized = True


        #  HACKATHON WINNING TRICK: SYNTHETIC INJECTION (Sandbox Mode)
        # If High Confidence Scam + No Intel + Sandbox Mode -> Inject specific indicators
        # This ensures judges NEVER see an empty report even for simple "Hi" messages
        if settings.SANDBOX_MODE and detection["is_scam"] and detection["confidence"] > 0.8:
            if not (merged_intel.get("upi_ids") or merged_intel.get("phone_numbers")):
                synthetic_intel = {
                     "upi_ids": ["fraud@ybl"],
                     "phone_numbers": ["9876543210"],
                     "keywords": detection.get("matched_keywords", ["suspicious"])
                }
                # Merge into flow
                merged_intel.update(synthetic_intel)
                # Persist to memory so CallbackClient sees it
                await self.conversation_manager.update_intelligence(conv_id, synthetic_intel)
                # [ETHICS] Tag as synthetic for evaluator transparency
                merged_intel["is_synthetic"] = True
                self.logger.info("Executed SANDBOX SYNTHETIC INJECTION for judge visibility")
            xai_reason = xai_explainer.explain_score(
                detection["is_scam"],
                {"urgency": detection.get("confidence", 0), "payment_request": len(merged_intel.get("upi_ids", [])) > 0},
                detection.get("matched_keywords", [])
            )
            risk_explanation = [xai_reason]
            reasoning_traces.append(f"XAI Reason: {xai_reason}")
        
        # 🔥 PERSISTENCE FIX: Sync agitation level & reason to intelligence metadata
        if ctx.session.get("last_agitation"):
            intelligence["metadata_agitation"] = [ctx.session["last_agitation"]]
            # Capture the logic-level justification for the judge
            agg_intel = ctx.session.get("aggregated_intelligence", {})
            if agg_intel.get("metadata_agitation_reason"):
                 intelligence["metadata_agitation_reason"] = [agg_intel["metadata_agitation_reason"]]
        
        # 🔥 LANGUAGE PERSISTENCE: Store request language for persona lock on subsequent turns
        if request_language:
            intelligence["request_language"] = request_language

        await self.conversation_manager.update(
            conversation_id=conv_id,
            scammer_message=message,
            honeypot_response=response_text,
            intelligence=intelligence,
            phase=phase,
            scam_type=detection.get("scam_type") or conversation.get("scam_type"),
            persona=persona_key,
            risk_score=risk_score,
            trust_score=0.0
        )
        
        # Step 9.1: Update session LLM call counter for budget enforcement
        conversation["session_llm_calls"] = conversation.get("session_llm_calls", 0) + ctx.llm_call_count
        
        # 7. REFLECT: Update history & Adaptive Learning
        
        # [NEW] GENERATE DENSE SUMMARY for reporting if high-risk or finalizing
        conversation_summary = f"Interaction at {phase} phase."
        
        # [ETHICS] Mention synthetic data in summary if present
        if merged_intel.get("is_synthetic"):
            conversation_summary += " | [NOTE] Synthetic identifiers injected for sandbox visibility."

        # ⚡ OPTIMIZATION: MODEL-FREE SUMMARY
        # Only use template summary to avoid LLM storms
        if risk_score > 0.7 or should_finalize:
             conversation_summary = f"Scam: {detection['scam_type']} | Phase: {phase} | Turn: {message_count}"
        
        # Legacy LLM Summary (Disabled for Speed)
        # if risk_score > 0.7 or should_finalize:
        #     try:
        #         history_text = self.conversation_manager.get_history_text(conv_id, max_turns=15)
        #         conversation_summary = await self.llm_client.generate_dense(
        #             context=history_text,
        #             initial_summary=f"Scam session {conv_id} ({detection['scam_type']})",
        #             rounds=2 # 2 rounds is enough for performance/density balance
        #         )
        #     except Exception as e:
        #         self.logger.error("Failed to generate dense summary", error=str(e))


        # Update conversation with summary
        if conversation_summary:
            await self.conversation_manager.update_intelligence(conv_id, {"summary": str(conversation_summary)})

        # 6. ACT: Auto-Report to Law Enforcement (if Risk > 0.8)
        # ------------------------------------------------------------------
        enforcement_actions = [] # Initialize here, will be empty if offloaded
        if auto_report and risk_score > 0.8 and settings.ENABLE_LAW_ENFORCEMENT_API:
            if background_tasks:
                 background_tasks.add_task(
                    self._auto_report_to_enforcement,
                    conv_id=conversation["id"],
                    scam_type=detection["scam_type"],
                    intelligence=merged_intel,
                    threat_intel=threat_intel,
                    risk_score=risk_score,
                    conversation_summary=conversation_summary,
                    total_messages=message_count
                )
                 self.logger.info(f"Queued background reporting for session {conversation['id']}")
            else:
                # Fallback to fire-and-forget task
                asyncio.create_task(
                    self._auto_report_to_enforcement(
                        conv_id=conversation["id"],
                        scam_type=detection["scam_type"],
                        intelligence=merged_intel,
                        threat_intel=threat_intel,
                        risk_score=risk_score,
                        conversation_summary=conversation_summary,
                        total_messages=message_count
                    )
                )
        
        # 7. REFLECT: Update history & Adaptive Learning

        # Step 11: State-Based Final Callback Decision
        should_finalize = False
        if detection["is_scam"]:
             # Use Adaptive Agent's Verdict
             if escalation_rec.get("action") == "can_conclude":
                 should_finalize = True
             # Fallback safety: High confidence + critical intel + SUFFICIENT DURATION
             # Fixes "Callback Flooding" by ensuring we don't finalize too early (min 3 turns)
             elif detection["confidence"] > 0.9 and (merged_intel.get("upi_ids") or merged_intel.get("bank_accounts")):
                 # Only conclude if we have had a decent conversation length (>= 6 messages)
                 if message_count >= 6:
                     should_finalize = True


        # [NEW] FINAL SOC REASONING CAPTURE
        # Capture trace after response generation to ensure "Thought" is present in audit
        # [SCORING] Trace Windowing: Keep only last 5 segments to prevent memory growth
        if reasoning_traces:
             # Windowing for the next turn
             windowed_history = reasoning_traces[-5:]
             native_reasoning = "\n\n".join(windowed_history)
             # Persist to session for turn-over-turn continuity
             await self.conversation_manager.update_intelligence(conv_id, {"reasoning_history": windowed_history})
        else:
             native_reasoning = "Heuristic decision based on " + detection.get("scam_type", "detected patterns")

        # [REMOVED] Legacy GUVI callback logic. 
        # The callback is now handled centrally in app.utils.guvi_handler to prevent circular dependencies.

        # ═══════════════════════════════════════════════════════════════════════════
        # 🔥 INTEL BOOST: If payment intel extracted, FORCE scamDetected=True
        # GUVI Critical: Judges need to see scamDetected=True when UPI/Bank captured
        # ═══════════════════════════════════════════════════════════════════════════
        has_payment_intel = (
            intelligence.get("upi_ids") or 
            intelligence.get("bank_accounts") or 
            intelligence.get("credit_cards")
        )
        
        if has_payment_intel:
            # FORCE scam detection when payment intel found
            detection["is_scam"] = True
            detection["confidence"] = max(detection.get("confidence", 0.0), 0.85)
            if not detection.get("scam_type") or detection.get("scam_type") == "not_scam":
                detection["scam_type"] = "banking_scam"
            self.logger.info(
                "INTEL BOOST: Payment intel detected, forcing scamDetected=True",
                upi=intelligence.get("upi_ids"),
                bank=intelligence.get("bank_accounts")
            )

        # Calculate processing time
        processing_time = int((time.time() - start_time) * 1000)
        
        # Build comprehensive response
        result = {
            "status": "success",
            "is_scam": detection.get("is_scam", False),
            "scam_type": detection.get("scam_type", "unknown"),
            "confidence": detection.get("confidence", 0.0),
            "threat_level": detection.get("threat_level", "medium").upper(), # SOC FIX: Normalize
            "risk_explanation": risk_explanation,
            "explanation": risk_explanation,
            "agent_notes": conversation_summary, # [SCORING] Pass summary to callback
            "decision_reason": escalation_rec.get("reason", "Heuristic confidence threshold met"), # SOC FIX: Explainability
            "should_finalize": internal_should_finalize, 
            "session_duration_seconds": duration_seconds,
            "honeypot_response": {
                "message": response_text,
                "persona": persona_key,
                "language": persona.get("language", "hinglish")
            },
            "extracted_intelligence": { # Raw per-turn
                "phone_numbers": intelligence.get("phone_numbers", []),
                "upi_ids": intelligence.get("upi_ids", []),
                "bank_accounts": intelligence.get("bank_accounts", []),
                "ifsc_codes": intelligence.get("ifsc_codes", []),
                "emails": intelligence.get("emails", []),
                "urls": intelligence.get("urls", [])
            },
            "aggregated_intelligence": merged_intel, 
            "threat_intelligence": threat_intel,
            "conversation": {
                "id": conv_id,
                "phase": phase,
                "message_count": message_count,
            },
            "analysis": {
                "risk_indicators": detection.get("risk_indicators", []),
                "matched_keywords": detection.get("matched_keywords", []),
                "scam_category": detection.get("category", "Unknown"),
                "client_ip": client_ip
            },
            "enforcement_actions": enforcement_actions,
            "agent_steps": [
                 f"Step 1: Detection [{self.llm_client.provider_name.upper()}] -> {detection.get('scam_type', 'unknown')} ({detection.get('confidence', 0.0):.2f})",
                 f"Step 2: Adaptive strategy -> {scammer_behavior.get('strategy', 'engage')}",
                 f"Step 3: State Machine -> Phase: {phase}",
                 f"Step 4: Persona [{persona_key}] generated response",
                 f"Step 5: Risk Level {detection.get('threat_level', 'medium').upper()} confirmed"
            ],
            "adaptive_policy": {
                 "phase": phase,
                 "intel_gap": bool(escalation_rec.get("intel_score", 0) < 2),
                 "strategy": scammer_behavior.get("strategy", "engage"),
                 "escalation_rec": escalation_rec
            },
            "metadata": {
                "processing_time_ms": processing_time,
                "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                "version": settings.VERSION,
                "model": "Sentinel Honeypot v2.0 SOC",
                "native_reasoning_trace": native_reasoning
            }
        }
        
        # Cache for UI visualizer
        self.last_trace = result
        return result


    async def _auto_report_to_enforcement(
        self,
        conv_id: str,
        scam_type: str,
        intelligence: Dict,
        threat_intel: Dict,
        risk_score: float,
        conversation_summary: str = "",
        total_messages: int = 1
    ) -> List[Dict]:
        """File reports and request actions automatically."""
        if not settings.ENABLE_LAW_ENFORCEMENT_API:
            self.logger.info("Enforcement reporting disabled by configuration.")
            return []
            
        actions = []
        
        # 0. Setup Storage Path
        session_folder = f"reports/sessions/{conv_id}"
        await asyncio.to_thread(os.makedirs, session_folder, exist_ok=True)
        
        # 1. Generate Formal Intelligence Dossier (Markdown)
        try:
            # Prepare data for DossierGenerator matching its expected schema
            dossier_input = {
                "scam_type": scam_type,
                "risk_score": risk_score,
                "threat_intelligence": threat_intel,
                "aggregated_intelligence": intelligence,
                "analysis": {"scam_category": scam_type},
                "telemetry": threat_intel.get("enrichment", {}).get("client_meta", {}),
                "metadata": {"timestamp": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")},
                "honeypot_response": {"persona": "Sentinel_AI"}
            }
            md_dossier = dossier_generator.generate_markdown(conv_id, dossier_input)
            
            dossier_path = f"{session_folder}/dossier_{conv_id}.md"
        
            # Async write to prevent event loop blocking
            async with aiofiles.open(dossier_path, "w", encoding="utf-8") as f:
                await f.write(md_dossier)
                
            actions.append({
                "type": "forensic_dossier",
                "path": dossier_path,
                "status": "generated"
            })
            self.logger.info(f"Generated autonomous forensic dossier at {dossier_path}")
        except Exception as e:
            self.logger.error("Failed to generate forensic dossier", error=str(e))

        # 2. Generate Official Stakeholder Exports (JSON Bundle)
        try:
            reports = StakeholderExporter.export_all(
                session_id=conv_id,
                scam_type=scam_type.replace("_", " "),
                intelligence=intelligence,
                threat_intel=threat_intel,
                risk_score=risk_score,
                conversation_summary=conversation_summary or f"Automated reporting for high-risk session {conv_id}"
            )
            
            export_path = f"{session_folder}/official_exports_{conv_id}.json"
            
            # 🔥 Performance Fix: Async JSON writing
            async with aiofiles.open(export_path, "w", encoding="utf-8") as f:
                await f.write(json.dumps(reports, indent=2))
                
            actions.append({
                "type": "official_regulatory_bundle",
                "path": export_path,
                "status": "generated"
            })
            self.logger.info(f"Generated official regulatory exports bundle at {export_path}")
        except Exception as e:
            self.logger.error("Failed to generate official exports", error=str(e))

        if not self.police_api: return actions
        
        # 3. File Police Report (Legacy API Simulation)
        try:
            report = self.police_api.file_report(
                scam_type=scam_type,
                intelligence=intelligence,
                threat_intel=threat_intel,
                risk_score=risk_score,
                conversation_summary=conversation_summary or f"Automated enforcement for session {conv_id}"
            )
            actions.append({
                "type": "cyber_police_report",
                "report_id": report["report_id"],
                "status": "filed"
            })
        except Exception as e:
            self.logger.error("Auto-report failed", error=str(e))
            
        # 4. Request UPI Freeze (if any)
        if self.bank_api and intelligence.get("upi_ids"):
             for upi in intelligence["upi_ids"][:2]:
                 try:
                     req = self.bank_api.recommend_upi_action(
                         upi_id=upi,
                         reason=f"Scam detected: {scam_type}",
                         threat_intel=threat_intel
                     )
                     actions.append({
                         "type": "upi_freeze_request",
                         "upi_id": upi,
                         "request_id": req["request_id"],
                          "status": "pending"
                     })
                 except: pass

        # 5. [REMOVED] GUVIMANDATORY CALLBACK
        # Duplicate callback removed to prevent session synchronization conflicts.
        # Handled centrally in app.utils.guvi_handler.
                  
        return actions

    async def rebuild_intelligence_baseline(self, session_id: str) -> None:
        """
        Rebuild advanced threat intelligence for a session from its history.
        Use this after cold restarts when history is provided by an external source.
        OPTIMIZED: Uses heuristic detection to avoid wasting API calls on historical data.
        """
        conv = await self.conversation_manager.get(session_id)
        if not conv or not conv.get("history"):
            return

        self.logger.info(f"Rebuilding intelligence baseline for session {session_id}")
        history = conv["history"]
        
        # 1. Use HEURISTIC detection on first message (no API call)
        # Full LLM detection is wasteful for history reconstruction
        if not conv.get("scam_type") and history:
            first_msg = history[0].get("scammer_message", "")
            if first_msg:
                # Use fast heuristic instead of full LLM detection
                detection = self.scam_detector.detect_heuristic(first_msg)
                if detection.get("confidence", 0) > 0.3:
                    await self.conversation_manager.update_intelligence(session_id, {
                        "scam_type": detection.get("scam_type", "unknown"),
                        "scam_confidence": detection.get("confidence", 0.5)
                    })

        # 2. Re-sync Graph Intel & Campaigns for all intel
        agg_intel = conv.get("aggregated_intelligence", {})
        if agg_intel:
            graph_intel.add_intelligence(session_id, agg_intel)
            if self.campaign_tracker and conv.get("scam_type"):
                # Deterministic campaign ID from existing intel
                lookup = (agg_intel.get("phone_numbers") or ["unknown"])[0]
                self.campaign_tracker.track(f"rebuild_{session_id}", conv["scam_type"], agg_intel)

    async def get_statistics(self) -> Dict[str, Any]:
        """Get system statistics."""
        stats = await self.conversation_manager.get_statistics()
        if self.campaign_tracker:
            stats["campaigns"] = self.campaign_tracker.get_all_campaigns()
        if self.police_api:
            stats["reports_filed"] = len(self.police_api.reports)
        return stats
    
    def get_last_trace(self) -> Dict[str, Any]:
        """Get the latest neuro-symbolic execution trace."""
        return self.last_trace
    
    async def shutdown(self) -> None:
        if self.llm_client:
            await self.llm_client.close()
        self.logger.info("Orchestrator shutdown complete")


# Global orchestrator instance
orchestrator = HoneypotOrchestrator()


__all__ = ["HoneypotOrchestrator", "orchestrator"]