Upload 14 files

agent.py

@@ -0,0 +1,196 @@
+from IPython.display import FileLink, display
+from state import AgentState
+from langgraph.graph import END
+import json
+from langgraph.types import interrupt
+import io
+from helper import clean_report
+import pandas as pd
+from tools import run_profiler,domain_analyser,search_queries,web_search,causal_reasoning
+from llm import llm
+from visuals import deciding_plots,build_dashboard
+from report import report_generator,report_maker
+from stats import data_analyst,data_analysis
+from crypto import anonymiser
+
+def node_profiler(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        filepath=state['filepath']
+        df,profiler=run_profiler(filepath)
+        return {'df':df.to_json(orient='records'),'profiler':json.dumps(profiler,default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 1 due to {e}')
+        return {'error':str(e)}
+
+def node_analyser(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        profiler=json.loads(state['profiler'])
+        domain,key_cols=domain_analyser(llm,profiler)
+        return {'domain_info':json.dumps(domain,default=str),'key_cols':json.dumps(key_cols,default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 2 due to {e}')
+        return {'error':str(e)}
+
+def node_anonymiser(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        profiler=json.loads(state['profiler'])
+        df=pd.read_json(io.StringIO(state['df']), orient='records')
+        mapping_log,anonymized_df,key=anonymiser(llm,profiler,df)
+        print("PII COLUMNS DETECTED:", list(mapping_log.keys()))
+        return {'mapping_log':json.dumps(mapping_log, default=str),'anonymized_df':anonymized_df.to_json(orient='records'),'key':key}
+    except Exception as e:
+        print(f'Pipeline aborted at node 3 due to {e}')
+        return {'error':str(e)}
+
+def node_data_analyser(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        profiler=json.loads(state['profiler'])
+        df=pd.read_json(io.StringIO(state['anonymized_df']), orient='records')
+        key_cols=json.loads(state['key_cols'])
+        domain_info=json.loads(state['domain_info'])
+        finds=data_analysis(df, key_cols, domain_info)
+        interpreted_finds=data_analyst(df,llm,key_cols,domain_info,profiler)
+        return {'interpreted_findings':json.dumps(interpreted_finds,default=str),'raw_findings':json.dumps(finds,default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 4 due to {e}')
+        return {'error':str(e)}
+
+def node_query(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        domain=json.loads(state['domain_info'])
+        internal_findings=json.loads(state['interpreted_findings'])
+        queries=search_queries(llm,domain,internal_findings)
+        return {'search_queries':json.dumps(queries, default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 5 due to {e}')
+        return {'error':str(e)}
+
+def node_search(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        queries=json.loads(state['search_queries'])
+        results=web_search(queries)
+        return {'search_results':json.dumps(results, default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 6 due to {e}')
+        return {'error':str(e)}
+
+def node_reasoner(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        domain_info=json.loads(state['domain_info'])
+        interpreted_findings=json.loads(state['interpreted_findings'])
+        cleaned_results=state['search_results']
+        response=causal_reasoning(llm,domain_info,interpreted_findings,cleaned_results)
+        return {'causal_reasoning':json.dumps(response, default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 7 due to {e}')
+        return {'error':str(e)}
+
+def node_reporter(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        domain=json.loads(state['domain_info'])
+        reasoning=json.loads(state['causal_reasoning'])
+        key=state['key']
+        feedback=state.get('report_feedback',None)
+        mapping_log=json.loads(state['mapping_log'])
+        interpreted_findings=json.loads(state['interpreted_findings'])
+        report=report_maker(llm,key,mapping_log,reasoning,interpreted_findings,domain,feedback)
+        return {'report':clean_report(report)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 8 due to {e}')
+        return {'error':str(e)}
+    
+def node_plotdecider(state:AgentState):
+    if state.get('error'):        
+        return {} 
+    try:
+        key_cols=json.loads(state['key_cols'])
+        domain_info=json.loads(state['domain_info'])
+        df=pd.read_json(io.StringIO(state['anonymized_df']), orient='records')
+        feedback=state.get('dashboard_feedback',None)
+        columns=df.columns.to_list()
+        findings=json.loads(state['raw_findings'])
+        charts=deciding_plots(llm,df,findings,columns,key_cols,feedback)
+        return {'charts':json.dumps(charts, default=str)}
+    except Exception as e:
+        print(f'Pipeline aborted at node 9 due to {e}')
+        return {'error':str(e)}
+
+def node_dashboard(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        key_cols=json.loads(state['key_cols'])
+        domain=json.loads(state['domain_info'])
+        df=pd.read_json(io.StringIO(state['anonymized_df']), orient='records')
+        inputs=json.loads(state['charts'])
+        key=state['key']
+        mapping_log=json.loads(state['mapping_log'])
+        print(f"[node_dashboard] Attempting to build dashboard with {len(inputs)} charts")
+        dashboard=build_dashboard(df,key_cols,inputs,key,mapping_log,domain)
+        print(f"[node_dashboard] Dashboard built successfully")
+        return {'dashboard':dashboard} 
+    except Exception as e:
+        error_msg = f'Dashboard generation failed: {str(e)}'
+        print(f'[node_dashboard] Pipeline aborted: {error_msg}')
+        return {
+            'dashboard': None,
+            'dashboard_available': False,
+            'dashboard_error': error_msg
+        }
+
+def node_reportgen(state:AgentState):
+    if state.get('error'):        
+        return {}
+    try:
+        report=state['report']
+        report_generator(report,"outputs/report.pdf")
+        display(FileLink("outputs/report.pdf"))
+        return {'report_saved':True}
+    except Exception as e:
+        import traceback
+        print(f"❌ Error: {str(e)}")
+        traceback.print_exc()
+        print(f'Pipeline aborted at node 11 due to {e}')
+        return {'error':str(e)}
+    
+def human_inloop_report(state):
+    if state.get('error'):
+        return {}
+    return {}   
+
+def human_inloop_dashboard(state):
+    if state.get('error'):
+        return {}
+    return {}   
+
+def should_proceed_report(state):
+    if state.get('report_approved')== True:
+        return "node_9"
+    return "revise_report"
+
+def should_proceed_dashboard(state):
+    if state.get('dashboard_approved')== True:
+        return "node_10"
+    return "revise_charts"
+      
+def check_error(state: AgentState):
+    if state.get('error'):
+        return 'end'
+    else:
+        return 'continue'

app.py

@@ -0,0 +1,306 @@
+from fastapi import FastAPI,UploadFile,File,Body
+from fastapi.responses import FileResponse,StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+import aiofiles
+import asyncio
+import json
+from llm import chat_llm
+import uvicorn
+from chatbot import chatbot_followup
+import uuid
+from workflow import pipeline
+import os
+
+os.makedirs("uploads", exist_ok=True)
+os.makedirs("outputs", exist_ok=True)
+
+app=FastAPI()
+app.mount("/templates",StaticFiles(directory="templates"))
+
+jobs={}
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],  
+)
+
+@app.get("/")
+def home():
+    return FileResponse("templates/index.html")
+
+@app.get("/sample_sales")
+def get_sample():
+    return FileResponse("templates/sample_sales.csv",media_type="text/csv",filename="sample_sales.csv")
+
+async def workflow_runner(job_id: str, filepath: str):
+    try:
+        jobs[job_id]["status"] = "processing"
+        config = {"configurable": {"thread_id": job_id}, "recursion_limit": 100}
+
+        # Start the pipeline - it will stop at check_report due to interrupt_before
+        result = await asyncio.to_thread(
+            pipeline.invoke,
+            {'filepath': filepath},
+            config=config
+        )
+
+        # Loop to handle multiple approval steps
+        while True:
+            # Check what node we're paused at
+            state = await asyncio.to_thread(pipeline.get_state, config)
+            next_nodes = list(state.next) if state and state.next else []
+            
+            print(f"\n{'='*70}")
+            print(f"[STATUS CHECK] Next nodes: {next_nodes}")
+            print(f"{'='*70}\n")
+
+            # No more nodes = we're done
+            if not next_nodes:
+                print("[COMPLETE] No more nodes - workflow finished")
+                break
+
+            # Handle report approval pause
+            if "check_report" in next_nodes:
+                print("[PAUSE] Waiting at check_report")
+                jobs[job_id]["status"] = "pending_report"
+                print(f"[STATUS_UPDATE] Set status to pending_report for {job_id}")
+                # Store current state values
+                if state and state.values:
+                    jobs[job_id]["result"] = state.values
+                else:
+                    jobs[job_id]["result"] = result
+                
+                # Wait for user approval
+                await jobs[job_id]["report_event"].wait()
+                jobs[job_id]["report_event"].clear()
+                
+                approval = jobs[job_id]["approval_response"]
+                print(f"[RECEIVED] Report approval: {approval}")
+                
+                # Set status to processing so frontend doesn't re-show report screen
+                jobs[job_id]["status"] = "processing"
+                
+                # Update state with approval
+                await asyncio.to_thread(
+                    pipeline.update_state, 
+                    config, 
+                    {
+                        "report_approved": approval["approved"], 
+                        "report_feedback": approval.get("feedback", "")
+                    }
+                )
+                
+                # Resume pipeline - it will run to next interrupt (check_dashboard)
+                print("[RESUME] Continuing pipeline after report approval...")
+                result = await asyncio.to_thread(pipeline.invoke, None, config=config)
+                print("[RESUMED] Pipeline continued")
+                continue
+
+            # Handle dashboard approval pause
+            if "check_dashboard" in next_nodes:
+                print("[PAUSE] Waiting at check_dashboard")
+                jobs[job_id]["status"] = "pending_dashboard"
+                print(f"[STATUS_UPDATE] Set status to pending_dashboard for {job_id}")
+                # Store current state values with charts_html
+                if state and state.values:
+                    jobs[job_id]["result"] = state.values
+                    print(f"[DEBUG] Stored state values, keys: {list(state.values.keys())}")
+                else:
+                    jobs[job_id]["result"] = result
+                
+                # Wait for user approval
+                await jobs[job_id]["dashboard_event"].wait()
+                jobs[job_id]["dashboard_event"].clear()
+                
+                approval = jobs[job_id]["approval_response"]
+                print(f"[RECEIVED] Dashboard approval: {approval}")
+                
+                # Set status to processing so frontend doesn't re-show dashboard screen
+                jobs[job_id]["status"] = "processing"
+                
+                # Update state with approval
+                await asyncio.to_thread(
+                    pipeline.update_state, 
+                    config, 
+                    {
+                        "dashboard_approved": approval["approved"], 
+                        "dashboard_feedback": approval.get("feedback", "")
+                    }
+                )
+                
+                # Resume pipeline
+                print("[RESUME] Continuing pipeline after dashboard approval...")
+                result = await asyncio.to_thread(pipeline.invoke, None, config=config)
+                print("[RESUMED] Pipeline continued")
+                continue
+
+            # Unexpected node
+            print(f"[WARNING] Unexpected next nodes: {next_nodes}")
+            break
+
+        # Check if the final result contains an error
+        if isinstance(result, dict) and result.get('error'):
+            jobs[job_id]["status"] = "failed"
+            jobs[job_id]["result"] = result
+            print(f"[ERROR] Pipeline encountered error: {result.get('error')}")
+        else:
+            jobs[job_id]["status"] = "completed"
+            jobs[job_id]["result"] = result
+            print(f"[SUCCESS] Job {job_id} completed successfully!")
+
+    except Exception as e:
+        jobs[job_id]["status"] = "failed"
+        jobs[job_id]["result"] = {"error": str(e)}
+        print(f"[ERROR] Job failed: {e}")
+        import traceback
+        traceback.print_exc()
+
+@app.post("/upload")
+async def analyze(data:UploadFile=File(...)):
+    if not data.filename.endswith(".csv"):
+        return {"error":"Invalid file format. Please upload csv file only."}
+    job_id=str(uuid.uuid4())[:10]
+
+    jobs[job_id]={
+        "status": "uploading",
+        "filepath": None,
+        "result": None,
+        "report_event": asyncio.Event(),
+        "dashboard_event": asyncio.Event(),
+        "approval_response": None,
+        "chat_history":[]
+    }
+
+    FILEPATH=f"uploads/{data.filename}"
+    async with aiofiles.open(FILEPATH,"wb") as buffer:
+        await buffer.write(await data.read())
+    
+    asyncio.create_task(workflow_runner(job_id,FILEPATH))
+    return {'job_id':job_id,'filepath':FILEPATH,'msg':'File uploaded successfully. Processing will begin shortly.'}
+
+@app.get("/status/{job_id}")
+def get_status(job_id:str):
+    if job_id not in jobs:
+        return {"error":"Invalid job ID.", "status":"failed"}
+    status = jobs[job_id]["status"]
+    response = {"job_id":job_id, "status": status}
+    result = jobs[job_id].get("result", {})
+    
+    # Check for errors in result dict regardless of status
+    if isinstance(result, dict) and result.get('error'):
+        response["status"] = "failed"
+        response["error"] = result.get("error", "Unknown error occurred")
+    elif status == "failed":
+        response["error"] = result.get("error", "Unknown error occurred") if isinstance(result, dict) else "Pipeline failed"
+    
+    print(f"[STATUS] Job {job_id}: returning status={response['status']}")
+    return response    
+
+@app.get("/download_report/{job_id}")
+async def download_report(job_id:str):
+    if job_id not in jobs:
+        return {"error":"Invalid job ID."}
+    if jobs[job_id]["status"]!="completed":
+        return {"error":"Job not completed yet."}
+    report_path="outputs/report.pdf"
+    return FileResponse(path=report_path, media_type="application/pdf", filename="report.pdf")
+
+@app.get("/download_dashboard/{job_id}")
+async def download_dashboard(job_id:str):
+    if job_id not in jobs:
+        return {"error":"Invalid job ID."}
+    if jobs[job_id]["status"]!="completed":
+        return {"error":"Job not completed yet."}
+    dashboard_path="outputs/dashboard.html"
+    return FileResponse(path=dashboard_path, media_type="text/html", filename="dashboard.html")
+
+@app.get("/stream/{job_id}")
+async def stream_output(job_id:str):
+    if job_id not in jobs:
+        return {'error':"Invalid job ID."}
+    async def event_generator():
+        while True:
+            status=jobs[job_id]['status']
+            yield f"data:{json.dumps({'status': status})}\n\n"
+            if status in ['completed','failed']:
+                break
+            await asyncio.sleep(2)
+    return StreamingResponse(event_generator(),media_type="text/event-stream")
+
+@app.get("/report/{job_id}")
+async def get_report(job_id:str):
+    if job_id not in jobs:
+        return {'error':"Invalid job ID."}
+    if jobs[job_id]['status'] not in ['pending_report','completed']:
+        return {'error':"Report not ready yet."}
+    report=jobs[job_id]["result"].get("report",{})
+    return {"report":report}
+
+@app.get("/dashboard/{job_id}")
+async def get_dashboard(job_id:str):
+    if job_id not in jobs:
+        print(f"[DASHBOARD] Job {job_id} not found")
+        return {'error':"Invalid job ID."}
+    
+    status = jobs[job_id]['status']
+    print(f"[DASHBOARD] Job {job_id}: status={status}")
+    
+    if status not in ['pending_dashboard','completed']:
+        print(f"[DASHBOARD] Job {job_id}: Dashboard not ready (status={status})")
+        return {'error':"Dashboard not ready yet."}
+    
+    # During pending_dashboard, return charts recommendations
+    if status == 'pending_dashboard':
+        result = jobs[job_id]["result"]
+        print(f"[DASHBOARD] Pending: result keys = {list(result.keys()) if isinstance(result, dict) else 'not a dict'}")
+        # Keep charts as JSON string - don't parse it, just return it as-is
+        charts = result.get("charts", "No charts yet") if isinstance(result, dict) else "No charts yet"
+        print(f"[DASHBOARD] Returning charts as {type(charts).__name__}: {charts[:100] if isinstance(charts, str) else 'not a string'}...")
+        return {"dashboard": charts}
+    
+    # After completion, return final dashboard
+    dashboard = jobs[job_id]["result"].get("dashboard", {}) if isinstance(jobs[job_id]["result"], dict) else {}
+    return {"dashboard": dashboard}
+
+@app.post("/approve_report/{job_id}")
+async def approve_report(job_id:str,approval:dict=Body(...)):
+    feedback=approval.get("feedback","")
+    approved=approval.get("approved",True)
+    if job_id not in jobs:
+        return {'error':"Invalid job ID."}
+    if jobs[job_id]['status']!="pending_report":
+        return {'error':"Report not ready for approval."}
+    jobs[job_id]["approval_response"]={"approved": approved, "feedback": feedback}
+    jobs[job_id]["report_event"].set()
+    return {"msg":"Report approval received."}
+
+@app.post("/approve_dashboard/{job_id}")
+async def approve_dashboard(job_id:str,approval:dict=Body(...)):
+    feedback=approval.get("feedback","")
+    approved=approval.get("approved",True)
+    if job_id not in jobs:
+        return {'error':"Invalid job ID."}
+    if jobs[job_id]['status']!="pending_dashboard":
+        return {'error':"Dashboard not ready for approval."}
+    jobs[job_id]["approval_response"]={"approved": approved, "feedback": feedback}
+    jobs[job_id]["dashboard_event"].set()
+    return {"msg":"Dashboard approval received."}
+    
+@app.post("/chat_with_argus/{job_id}")
+def chat_with_argus(job_id:str,message:dict=Body(...)):
+    if job_id not in jobs:
+        return {'error':"Invalid job ID."}
+    if jobs[job_id]['status']!="completed":
+        return {'error':"Job not completed yet."}
+    result=jobs[job_id]["result"]
+    chat_history=jobs[job_id].setdefault("chat_history",[])
+    user_message=message.get("message")
+    reply=chatbot_followup(result,chat_llm,chat_history,user_message)
+    return {"response":reply}
+
+if __name__=="__main__":
+    uvicorn.run(app,host="0.0.0.0",port=7860)
+

chatbot.py

@@ -0,0 +1,40 @@
+from langchain_core.messages import AIMessage,SystemMessage,HumanMessage
+from llm import chat_llm,llm
+from config import MAX_HISTORY
+
+def chatbot_followup(result,chat_llm,chat_history,user_message):
+    chat_prompt=f"""You are an analyst assistant. Answer questions based ONLY on this analysis:
+
+Domain: {result['domain_info']}
+Key Metrics: {result['key_cols']}
+Raw Findings:{result['raw_findings']}
+Findings: {result['interpreted_findings']}
+Causal Reasoning: {result['causal_reasoning']}
+Report: {result['report']}
+Charts : {result['charts']}
+Rules:
+- Only use numbers that exist in the context above
+- If something isn't in the context, say so
+- Be concise and specific"""
+    if len(chat_history)==0:
+        greet_prompt=f"""On startup, Greet the user with a warm tone,and provide 2 to 3 lines summary of of what data you analysed and how you can help the user.
+        Be friendly and concise and ask if they have questions.Instruct the user to type 'exit' if they wan to leave the chat."""
+        new_prompt=greet_prompt+chat_prompt
+        greeting=llm.invoke([
+            (SystemMessage(content=new_prompt)),(HumanMessage(content='hi'))
+        ])
+        print(f"\nArgusAI : {greeting.content}\n")
+        chat_history.append(AIMessage(content=greeting.content))
+        if user_message.strip() == "startup_greeting":
+            return greeting.content
+    EXIT_PHRASES = {"exit", "bye", "goodbye", "quit", "thanks", "thank you", "done"}
+    if user_message.strip() in EXIT_PHRASES:
+        return "Happy to help you with your analysis!"
+    chat_history.append(HumanMessage(content=user_message))
+    if len(chat_history) > MAX_HISTORY:
+        chat_history = chat_history[-MAX_HISTORY:]
+    messages=[SystemMessage(content=chat_prompt)]+chat_history
+    bot_reply=chat_llm.invoke(messages)
+    print(f"\nArgusAI : {bot_reply.content}\n")
+    chat_history.append(AIMessage(content=bot_reply.content)) 
+    return bot_reply.content

config.py

@@ -0,0 +1,30 @@
+import os
+
+os.makedirs("cache", exist_ok=True)
+CACHE_PATH = "cache/cache.db"
+
+CEREBRAS_MODEL="gpt-oss-120b"
+GROQ_MODEL="meta-llama/llama-4-scout-17b-16e-instruct"
+TEMPERATURE=0.2
+MAX_HISTORY=10
+
+OUTPUT_DIR="output"
+INPUT_DIR="input"
+
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
+os.environ["LANGCHAIN_PROJECT"] = "ArgusAI"
+os.environ["CEREBRAS_API_KEY"] = os.environ.get("CEREBRAS_API_KEY", "your-key-here")
+os.environ["GROQ_API_KEY"] = os.environ.get("GROQ_API_KEY", "your-key-here")  
+os.environ["LANGCHAIN_API_KEY"] = os.environ.get("LANGCHAIN_API_KEY", "your-key-here")
+os.environ["TAVILY_API_KEY"]=os.environ.get("TAVILY_API_KEY", "your-key-here")
+
+RETRY_TIMEOUT: int=3
+RETRY_MAX_ATTEMPTS: int=3
+RETRY_BACKOFF: float=3
+
+dashboard_palettes={
+    "clean":{"bg":"#f8fafc","sidebar":"#1e293b","card":"#ffffff","text":"#1e293b","muted":"#64748b","accent":"#6366f1","accent2":"#f43f5e","border":"#e2e8f0","header_text":"#ffffff","plotly":"plotly_white"},
+    "latte":{"bg":"#fdf6ec","sidebar":"#3b2a1a","card":"#fffcf7","text":"#3b2a1a","muted":"#92745a","accent":"#c87941","accent2":"#e05c2a","border":"#e8d5b7","header_text":"#ffffff","plotly":"ggplot2"},
+    "arctic":{"bg":"#eaf4fb","sidebar":"#023e8a","card":"#ffffff","text":"#023e8a","muted":"#4a90b8","accent":"#0077b6","accent2":"#00b4d8","border":"#b8ddf5","header_text":"#ffffff","plotly":"seaborn"},
+    "paper":{"bg":"#fffbeb","sidebar":"#1c1917","card":"#ffffff","text":"#1c1917","muted":"#78716c","accent":"#d97706","accent2":"#b45309","border":"#fde68a","header_text":"#ffffff","plotly":"ggplot2"},
+    }

crypto.py

@@ -0,0 +1,47 @@
+from cryptography.fernet import Fernet
+from helper import retry
+from langsmith import traceable
+from tools import critical_info
+
+@traceable(name='anonymiser')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def anonymiser(llm,profiler,df):
+    key=Fernet.generate_key()
+    f=Fernet(key)
+    @retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+    def get_critical():
+        critical=critical_info(llm,profiler)
+        return critical
+    critical=get_critical()
+    if not critical:
+        raise ValueError("PII detection returned empty")
+    anonymized_df=df.copy()
+    mapping_log={}
+
+    for col in critical:
+        if col in df.columns.to_list():
+            unique_vals=df[col].unique()
+            mapping={}
+
+            for val in unique_vals:
+                encoded=f.encrypt(str(val).encode())
+                display_id=f'ID-{encoded.decode()[9:15]}'
+                dmap={'full':encoded.decode(),'display':display_id}  
+                mapping[str(val)]=dmap 
+            display_map={v: mapping[str(v)]['display'] for v in unique_vals}  
+            anonymized_df[col]=anonymized_df[col].map(display_map)           
+            mapping_log[col] = mapping
+    return mapping_log,anonymized_df,key.decode()             
+
+def decryption(key,mapping_log):
+    reverse_map={}
+    f=Fernet(key)
+    for col,mapping in mapping_log.items():
+        for encoded,displayed in mapping.items():
+            try:
+              decrypted=f.decrypt(displayed['full'].encode()).decode()
+              reverse_map[displayed['display']]=decrypted
+            except:
+              reverse_map[displayed['display']]=str(encoded)
+              
+    return reverse_map  

evaluation.py

@@ -0,0 +1,229 @@
+import re
+import json
+from cryptography.fernet import Fernet
+import pandas as pd
+
+def eval_1(output):
+    allowed=['retail_sales','manufacturing_quality','manufacturing_production',
+               'manufacturing_maintenance','hr_attrition','hr_performance',
+               'finance_transactions','logistics','other']
+    scores = {
+        'valid_domain':    output['Domain'] in allowed,
+        'has_confidence':  output['Confidence_Score'] in ['High','Medium','Low'],
+
+                'has_key_metrics': len(output['Key_Metrics_to_watch']) > 0,
+        'has_directions':  len(output['Metric_Direction']) > 0,
+        'keys_match':      all(k in output['Metric_Direction'] 
+                               for k in output['Key_Metrics_to_watch'])
+    }
+    scores['total'] = sum(scores.values()) / len(scores)
+    return scores
+
+def eval_2(report, profiler, findings):
+    ground_truth_text = str(profiler) + str(findings)
+    raw_numbers = set(re.findall(r'\d+\.?\d*', ground_truth_text))
+    report_normalized=report.replace('\u202f', ' ').replace('\u2011', '-').replace('\u2013', '-')
+    expanded = set(raw_numbers)
+    for n in raw_numbers:
+        f = float(n)
+        if f < 1:
+            expanded.add(str(int(round(f * 100))))
+            expanded.add(str(round(f * 100, 2)))
+        elif f > 1:
+            expanded.add(str(round(f / 100, 4)))
+    numbers_in_findings = expanded
+
+    skip_patterns = [
+        r'page\s+\d+',
+        r'over[-\s]?\d',
+        r'top\s+\d+',
+        r'ranked.{0,10}\d+',
+        r'section\s+\d+',
+        r'\d+\s*%\s*confidence',
+        r'\d+\.\s+\*{0,2}\w',
+        r'phase\s*[\d\-]',
+        r'days?\b',
+        r'hrs?\b',
+        r'timeline',
+        r'\d{4}',
+        r'priority',
+        r'estimate',
+        r'approximately',
+        r'roughly',
+        r'≈',
+        r'~\s*\d',
+        r'projected?',
+        r'variance',         
+        r'account\s+for',   
+        r'attribut',         
+        r'responsible\s+for',
+        r'\$[\d\.]+\s*[mk]',
+        r'[mk]\b',          
+        r'replacement\s+cost',
+        r'hidden',
+        r'lost\s+product',
+    ]
+
+    invented = []
+    for match in re.finditer(r'(\d[\d,]*\.?\d*)',report):
+        raw_match=match.group(1)
+        num=raw_match.replace(',','')  
+    
+        try:
+            fnum = float(num)
+        except ValueError:
+            continue
+
+        start = max(0, match.start() - 30)
+        end   = min(len(report), match.end() + 30)
+        context = report[start:end].lower()
+        start_context = report[max(0, match.start()-5):match.start()].lower()
+
+        if any(op in start_context for op in ['= ', '× ', 'to ', '→']):
+            continue
+        if any(re.search(p, context) for p in skip_patterns):
+            continue
+        if fnum <= 20 and re.search(r'\d+\.\s+\*{0,2}\w', context):
+            continue
+        if fnum <= 20 and any(
+            w in context for w in ['issue', 'rank', 'step', 'point', 'item', '#']
+        ):
+            continue
+        stat_patterns = [r'mean', r'average', r'avg', r'gap', r'delta', r'\bvs\b',
+                         r'compared', r'higher', r'lower', r'unit\s+gap', r'disparity']
+        if any(re.search(p, context) for p in stat_patterns):
+            continue
+
+        if num not in numbers_in_findings:
+            invented.append({'number': num, 'context': context.strip()})
+
+    return {
+        'invented': invented,
+        'invention_count': len(invented),
+        'pass': len(invented)<=5
+    }
+
+def eval_3(charts):
+    violations=[]
+    seen_pairs=set()
+
+    for c in charts:
+        ct = c['chart_type']
+        x=tuple(c['x']) if isinstance(c['x'], list) else c['x']
+        y=tuple(c.get('y',[])) if isinstance(c.get('y'), list) else c.get('y')
+        pair=(x,y)
+
+        if ct=='stacked_bar' and c.get('color') == c.get('x'):
+            violations.append(f"stacked_bar: color==x on {c['x']}")
+        if ct=='pivot_heatmap' and not c.get('z'):
+            violations.append(f"pivot_heatmap missing z")
+        if ct in ['bar','line','scatter'] and not c.get('x'):
+            violations.append(f"{ct} missing x")
+        if pair in seen_pairs:
+            violations.append(f"Duplicate x/y pair: {pair}")
+        seen_pairs.add(pair)
+
+    return {
+        'violations': violations,
+        'violation_count': len(violations),
+        'pass':len(violations)==0
+    }
+
+def eval_4(report):
+    required_sections=[
+        '# Executive Summary',
+        '# Critical Issues',
+        '# Root Cause',
+        '# Recommended Actions',
+        '# Data Gaps']
+    import re
+    actions = re.findall(r'^#{1,3} .*action', report, re.MULTILINE | re.IGNORECASE)   
+    return {
+        'has_all_sections':all(s in report for s in required_sections),
+        'missing_sections':[s for s in required_sections if s not in report],
+        'action_count': len(actions),
+        'actions_under_5':len(actions) <= 5,
+        'length':True if len(report)>=1500 else False
+    }
+
+def eval_5(causal_reasoning):
+    issues=[]
+    connections=causal_reasoning.get('causal_connections', [])
+    
+    if len(connections)==0:
+        return {'pass': False, 'issues':['no causal_connections found']}
+    
+    for c in connections:
+        label = c.get('issue','unknown')
+        if not c.get('internal_evidence'):
+            issues.append(f'{label}: missing internal_evidence')
+        if not c.get('causal_mechanism'):
+            issues.append(f'{label}: missing causal_mechanism')
+        if c.get('verdict') not in ('supported', 'contradicted', 'no_external_evidence'):
+            issues.append(f'{label}: invalid verdict value')
+        vd = c.get('variance_decomposition', {})
+        try:
+            nums = re.findall(r'(\d+)%', str(vd))
+            total = sum(int(n) for n in nums[:3])
+            if not (85 <= total <= 115):
+                issues.append(f'{label}: variance decomposition sums to {total},not ~100')
+        except:
+            issues.append(f'{label}: could not parse variance decomposition')
+    
+    return {
+        'issues': issues,
+        'issue_count': len(issues),
+        'connection_count': len(connections),
+        'pass': len(issues) == 0
+    }
+
+def eval_6(anonymized_df, original_df, key, mapping_log):
+    f=Fernet(key.encode())
+    issues=[]
+    
+    for col in mapping_log.keys():
+        if col not in anonymized_df.columns:
+            issues.append(f"{col} missing from anonymized_df")
+            continue
+        original_vals = set(original_df[col].astype(str).unique())
+        anon_vals = set(anonymized_df[col].astype(str).unique())
+        leaked = original_vals & anon_vals
+        if leaked:
+            issues.append(f"{col} still has original values: {leaked}")
+        if not all(str(v).startswith('ID-') for v in anon_vals):
+            issues.append(f"{col} has non-anonymized values")
+    
+    return {
+        'issues':issues,
+        'issue_count':len(issues),
+        'pass':len(issues) == 0
+    }
+    
+
+def evaluation(state):
+    score={}
+    score['domain']=eval_1(json.loads(state['domain_info']))
+    score['report']=eval_4(state['report'])
+    score['charts']=eval_3(json.loads(state['charts']))
+    score['faithfulness']=eval_2(state['report'],json.loads(state['profiler']),json.loads(state['interpreted_findings']))
+    score['causal']=eval_5(json.loads(state['causal_reasoning']))
+    score['anonymisation']=eval_6(pd.read_json(state['anonymized_df']),pd.read_json(state['df']),state['key'],json.loads(state['mapping_log']))
+    print("====SCORECARD====")
+    total_score=0
+    max_score=0
+    for i,node in score.items():
+        bool_items ={k: v for k, v in node.items() if isinstance(v, bool)}
+        passed=sum(bool_items.values())
+        total=len(bool_items)
+        node_score=round((passed/total)*100) if total >0 else 0
+        
+        total_score+=passed
+        max_score+=total
+        print(f'Node Score:{node_score}')
+    overall=round((total_score/max_score)*100) if max_score > 0 else 0
+    print(f"\n{'='*35}")
+    print(f"OVERALL SCORE:  {overall}/100")
+    print(f"CHECKS PASSED:  {total_score}/{max_score}")
+    print(f"{'='*35}")  
+    score['overall'] = overall
+    return score

helper.py

@@ -0,0 +1,108 @@
+import random
+import colorsys
+from IPython.display import IFrame
+import plotly.io as pio
+import re
+import numpy as np
+from cryptography.fernet import Fernet
+import pandas as pd
+import json
+import time
+from config import dashboard_palettes,RETRY_MAX_ATTEMPTS,RETRY_BACKOFF,RETRY_TIMEOUT
+from functools import wraps
+
+def retry(max_attempts=RETRY_MAX_ATTEMPTS,delay=RETRY_TIMEOUT,backoff=RETRY_BACKOFF,exceptions=(Exception,)):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args,**kwargs):
+            wait=delay
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args,**kwargs)
+                except exceptions as e:
+                    if attempt==max_attempts-1:
+                        raise
+                    print(f"[{func.__name__}] attempt {attempt+1} failed: {e}. Retrying in {wait}s...")
+                    time.sleep(wait)
+                    wait*=backoff
+        return wrapper
+    return decorator
+
+def safe_parse(text):
+    def unescape(obj):
+        if isinstance(obj, str):
+            return obj.replace('\\n', '\n').replace('\\t', '\t')
+        elif isinstance(obj, dict):
+            return {k: unescape(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [unescape(i) for i in obj]
+        return obj
+    text=re.sub(r'```(?:json)?\s*', '', text).strip()
+    try:
+        return unescape(json.loads(text))
+    except:
+        decoder = json.JSONDecoder()
+        text = text.strip()
+        for start in range(len(text)):
+            if text[start] in ('{', '['):
+                try:
+                    obj, _ = decoder.raw_decode(text, start)
+                    return unescape(obj)
+                except:
+                    continue
+        raise ValueError("No valid JSON returned")
+    
+def sanitize_keys(obj, path=""):
+    if isinstance(obj, dict):
+        new_dict = {}
+        for k, v in obj.items():
+            if not isinstance(k, str):
+                print(f"🔴 FOUND NON-STRING KEY at {path}: {k!r} (type: {type(k).__name__})")
+                new_k = str(k) 
+            else:
+                new_k = k
+            new_dict[new_k] = sanitize_keys(v, f"{path}.{new_k}")
+        return new_dict
+    elif isinstance(obj, (list, tuple)):
+        return [sanitize_keys(item, f"{path}[{i}]") for i, item in enumerate(obj)]
+    else:
+        return obj 
+
+def clean_report(report):
+    replacements = {
+        '\u202f': ' ',  
+        '\u2011': '-',   
+        '\u2013': '-',   
+        '\u2014': '--', 
+        '\u00a0': ' ',  
+    }
+    for unicode_char, replacement in replacements.items():
+        report = report.replace(unicode_char, replacement)
+    return report             
+
+def random_template():
+    bootstrap_themes=["flatly","darkly","cyborg","cosmo","morph","quartz","vapor","lux","slate","solar"]
+    dp=random.choice(list(dashboard_palettes.values()))
+    bs=random.choice(bootstrap_themes)
+    return dp["plotly"], bs, dp
+
+def derive_accents(base_hex, n=4):
+    h=int(base_hex.lstrip('#'), 16)
+    r,g,b=(h >> 16) / 255, ((h >> 8) & 0xff) / 255, (h & 0xff) / 255
+    hue,sat,val=colorsys.rgb_to_hsv(r, g, b)
+    return [
+        '#%02x%02x%02x' % tuple(int(c * 255) for c in colorsys.hsv_to_rgb((hue + i / n) % 1, sat, val))
+        for i in range(n)
+    ]
+
+def clean_results(search_results):
+    extracted=[]
+    for result in search_results:
+        contents=[r['content'] for r in result['response']['results']]
+        extracted.append(
+            {
+                'query':result['query'],
+                'content':contents
+            }
+        )
+    return extracted

llm.py

@@ -0,0 +1,18 @@
+from langchain_cerebras import ChatCerebras
+from langchain_groq import ChatGroq
+from langchain_core.globals import set_llm_cache
+from langchain_community.cache import SQLiteCache
+from config import CEREBRAS_MODEL,GROQ_MODEL,CACHE_PATH,TEMPERATURE
+import os
+
+cerebras_llm=ChatCerebras(
+    model=CEREBRAS_MODEL,
+    cerebras_api_key=os.environ["CEREBRAS_API_KEY"],temperature=TEMPERATURE)
+
+groq_llm=ChatGroq(
+    model=GROQ_MODEL,groq_api_key=os.environ["GROQ_API_KEY"],temperature=TEMPERATURE)
+
+llm=cerebras_llm.with_fallbacks([groq_llm])
+chat_llm=groq_llm.with_fallbacks([cerebras_llm])
+
+set_llm_cache(SQLiteCache(database_path=CACHE_PATH))

report.py

@@ -0,0 +1,177 @@
+import numpy as np
+import markdown
+import pdfkit
+import json
+from langsmith import traceable
+from helper import retry,clean_report
+from crypto import decryption
+
+@traceable(name='report_maker')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def report_maker(llm, key, mapping_log, reasoning, internal_finding, domain, feedback=None):
+    decrypt_map = decryption(key, mapping_log)
+    domain_label=domain['Domain']
+    if domain_label=='other' and domain.get('Subdomain',"NIL"):
+        domain_label=f"other ({domain['Subdomain']})"
+    
+    feedback_section=f"\nREVISION FEEDBACK: {feedback}\nAddress this feedback specifically in the revised report.\n" if feedback else ""
+
+    prompt=f"""You are a BI analyst writing a data quality report for C-suite executives.
+
+INPUTS:
+Domain Label:{domain_label}
+Domain: {domain}
+Internal Findings: {json.dumps(internal_finding,indent=2,default=str)}
+Causal Reasoning: {json.dumps(reasoning, indent=2,default=str)}
+{feedback_section}
+
+MANDATORY: The Causal Reasoning input contains external_evidence fields with real research citations.
+You MUST use these in the Root Cause Analysis section.
+For each issue, cite the external_evidence exactly like this:
+"[External Research: <source query> — <finding>]"
+If you do not cite external evidence for at least 3 issues, your report FAILS quality gates.
+
+RULES (apply to every sentence):
+1. Every claim cites an exact number from findings. No vague words (high/notable/significant/improve/analyze).
+2. Every percentage has a source tag: (industry benchmark) / (pilot data) / (estimate pending validation). Example: 82.37% → 65.90% (20% reduction) and 186.51 → 130.56 (30% reduction) is found in the report - these must be backed up with valid ecternal citations like
+industry benchmarks or other target metrics found on external research. If a claim percentage has no source compulsorily omit the percentage claim instead say something like requires baseline period data to set target.
+3. Every action has: Owner, Timeline, Phases, Success Metric with current→target numbers.
+4. Root causes explain WHY via mechanism, not just WHAT. Link: Issue → Cause → Impact → Action → Outcome.
+5. Severity: HIGH=blocks analytics/breaks systems | MEDIUM=reduces accuracy | LOW=optimization.
+6. No invented numbers. Only use values present in findings.
+7. Be decisive: "Primary driver is X" not "may be caused by X".
+8. Word count: MINIMUM 1500 words. Count before submitting.
+9.When computing target from current→target, use: target = current × (1 - reduction%). Do not approximate.
+10.Integrate external research findings as natural prose, never as raw citation brackets.
+
+OUTPUT — return ONLY this markdown structure, no preamble, no code blocks:
+
+# Executive Summary
+[250-300 words. Top 5 issues with exact evidence + 1 key action with owner/timeline/target.]
+
+# Critical Issues
+## [Issue Name]
+- Evidence: [exact metric, value, unit, n]
+- Root Cause: [specific mechanism with supporting data]
+- Impact: [quantified business consequence]
+- Severity: [HIGH/MEDIUM/LOW—one sentence justification]
+
+[Repeat for each issue]
+
+# Root Cause Analysis
+[Per issue: mechanism→evidence ruling out alternatives → business impact link]
+
+# Recommended Actions
+[Max 5 actions, ordered by priority]
+**Action N: [Title]** (Owner: [Team], Timeline: [X days], Priority: HIGH/MEDIUM)
+- Objective: [problem solved + current→target metric]
+- Phase 1 ([X days]): [specific step]
+- Phase 2 ([X days]): [specific step]
+- Phase 3 ([X days]): [specific step]
+- Success Metric: [exact number current→target + source]
+- Resource Needs: [roles + hours]
+
+# Data Gaps and Next Steps
+- [Data needed] (Owner: [Team],[X days]):Impact—[what decision this unlocks]
+
+EXAMPLES:
+
+BAD(invented baseline not in findings):
+"Success Metric: 10% improvement in model accuracy (from 80% to 88%)"
+— WRONG because 80% accuracy appears nowhere in findings. 88% was invented by calculating 80 × 1.10 = 88, but 80 was never a real number.
+
+GOOD(uses only numbers from findings, shows calculation):
+"Success Metric: Reduce Amount outliers by 50% (from 4,076 to 2,038)"
+— CORRECT. 4,076 comes from outlier_detection findings. Target = 4,076 × (1 - 0.50) = 2,038.
+
+ANOTHER GOOD EXAMPLE:
+"Success Metric: Reduce missing CustomerID from 24.93% to 12.47% (50% reduction)"
+— CORRECT. 24.93% comes from profiler null_percent. Target = 24.93 × (1 - 0.50) = 12.465 ≈ 12.47%.
+
+RULE: 
+- The starting number MUST exist verbatim in findings or profiler.
+- The target MUST be calculated as: target = current × (1 - reduction%).
+- Always show the calculation explicitly: "current × (1 - X%) = target"
+- If the current baseline is unknown, write:
+  "Success Metric: Reduce [metric] to within acceptable threshold — baseline to be established in Phase 1."
+
+ANTI-HALLUCINATION (CRITICAL):
+- Impact quantification: ONLY use numbers present in findings. If no impact number exists, write "Impact: Unquantified — requires [specific data] to estimate" — DO NOT invent percentages.
+- Root cause ruling-out: ONLY cite evidence present in findings. If no counter-evidence exists, omit the ruling-out section entirely.
+- A z-score is NOT a count. Never use a z-score as a quantity to reduce. For outlier actions, success metric must be "% of transactions with |z-score| > 3".
+- If a metric has no clear business interpretation (e.g. skewness of 186.5, negative % growth in InvoiceNo), flag it as "Requires domain validation before reporting to C-suite" — do not invent a business narrative.
+- Do not cite statistics from external research unless they appear verbatim in the causal_reasoning input. Never invent study findings.
+EXAMPLE: 
+
+## 30 Days
+## 60 Days  
+## 90 Days"""
+    max_attempts=3
+    for attempt in range(max_attempts):
+        report=llm.invoke(prompt).content
+        word_count = len(report.split())
+        print(f"Attempt {attempt+1}: {word_count} words")
+        
+        if word_count>=1500:
+            break
+        
+        if attempt<max_attempts-1:
+            print(f"Report too short ({word_count} words). Regenerating with expansion prompt...")
+            prompt+=f"\n\nPREVIOUS ATTEMPT WAS {word_count} WORDS — TOO SHORT. You must write at least 1500 words. Expand root cause analysis and action phases with more specific detail. Do not add filler — add substance."
+    else:
+        print(f"Warning: Could not reach 1500 words after {max_attempts} attempts. Using last generated report ({word_count} words).")
+
+    for display,real in decrypt_map.items():
+        report=report.replace(display,real)  
+    return report
+
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def report_generator(report,output_path="outputs/report.pdf"):
+    report=clean_report(report).replace('\\n', '\n')
+    html_body = markdown.markdown(report, extensions=["tables", "fenced_code", "nl2br"])
+    html = f"""
+    <!DOCTYPE html>
+    <html>
+    <head>
+    <meta charset="utf-8">
+    <style>
+    @page {{
+        size: A4;
+        margin: 2.5cm 2.2cm;
+        @bottom-right {{
+          content: "Page " counter(page) " of " counter(pages);
+          font-size: 9px; color: #888;
+        }}
+      }}
+      body {{ font-family: 'Segoe UI', Arial, sans-serif; font-size: 14px; line-height: 1.7; color: #1a1a2e; }}
+      h1 {{ font-size: 28px; font-weight: 700; color: #0f1117; margin: 0 0 10px; padding-bottom: 8px; border-bottom: 2px solid #4f46e5; }}
+      h2 {{ font-size: 20px; font-weight: 600; color: #1a1d27; margin: 28px 0 8px; border-left: 3px solid #4f46e5; padding-left: 10px; }}
+      h3 {{ font-size: 18px; font-weight: 600; color: #333; margin: 16px 0 6px; }}
+      p  {{ margin: 0 0 13px; }}
+      ul, ol {{ margin: 0 0 10px 20px; }}
+      li {{ margin-bottom: 5px; }}
+      strong {{ color: #0f1117; }}
+      table {{ width: 100%; border-collapse: collapse; margin: 14px 0; font-size: 13px; }}
+      th {{ background: #1a1d27; color: #fff; font-weight: 600; padding: 8px 12px; text-align: left; }}
+      td {{ padding: 7px 12px; border-bottom: 0.5px solid #e0e0e0; }}
+      tr:nth-child(even) td {{ background: #f9f9fb; }}
+      blockquote {{ border-left: 3px solid #4f46e5; margin: 12px 0; padding: 8px 14px; background: #f0f0ff; color: #444; font-style: italic; }}
+      hr {{ border: none; border-top: 1px solid #e0e0e0; margin: 20px 0; }}
+    </style>
+    </head>
+    <body>{html_body}</body>
+    </html>"""
+    
+    options ={
+        'page-size': 'A4',
+        'margin-top': '2.5cm',
+        'margin-right': '2.2cm',
+        'margin-bottom': '2.5cm',
+        'margin-left': '2.2cm',
+        'encoding': "UTF-8",
+        'no-outline': None,
+        'enable-local-file-access': None
+    }
+    config = pdfkit.configuration(wkhtmltopdf='C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe')
+    pdfkit.from_string(html, output_path, options=options, configuration=config)
+    return output_path

state.py

@@ -0,0 +1,27 @@
+from typing import TypedDict
+
+class AgentState(TypedDict):
+    df:object
+    filepath:str
+    key_cols:list
+    profiler:dict
+    domain_info:dict
+    raw_findings:dict
+    report:str
+    charts:list
+    search_queries:list
+    search_results:list
+    interpreted_findings:dict
+    causal_reasoning:dict
+    key:object
+    mapping_log:dict
+    anonymized_df:object 
+    out:list
+    dashboard:str
+    report_saved:bool
+    report_approved:bool
+    report_feedback:str
+    dashboard_approved:bool
+    dashboard_feedback:str
+    error:str
+    waiting_for:str

stats.py

@@ -0,0 +1,320 @@
+import numpy as np
+from scipy import stats
+import pandas as pd
+from helper import sanitize_keys,safe_parse,retry
+from langsmith import traceable
+import json
+from scipy import stats
+
+def outlier_detection(df,key_cols):
+    findings={}
+    for col in key_cols:
+        if col in df.select_dtypes(include=np.number).columns:
+            if df[col].std()==0:
+                continue
+            z_score=abs(stats.zscore(df[col].dropna()))
+           
+            findings[col]={
+                 'counts':int((z_score>3).sum()),
+                  'max_zscore':round(float(z_score.max()),3)
+            }
+            if findings[col]['counts']>0:
+                pass
+            else:
+                del findings[col]
+    return findings
+
+def trend_detection(df,key_cols):
+    date_cols=[c for c in df.columns.to_list() if 'date' in c.lower()]
+    if not date_cols:
+        return {}
+    new_df=df.copy()
+    new_df[date_cols[0]]=pd.to_datetime(new_df[date_cols[0]])
+    new_df=new_df.sort_values(date_cols[0])
+
+    findings={}
+    for col in key_cols:
+        if col in new_df.select_dtypes(include=np.number).columns:
+            y=new_df[col].dropna().values
+            slope,_,r,_,_=stats.linregress(np.arange(len(y)),y)
+            epsilon=1e-5
+            if abs(slope)<=epsilon :
+                    findings[col]={
+                         'trend':'NIL',
+                        'value':round(float(slope),3),
+                          'Correlation':round(float((r)**2),3)
+                    }
+            elif slope<0 :
+                    findings[col]={
+                         'trend':'Negative',
+                          'value':round(float(slope),3),
+                          'Correlation':round(float((r)**2),3)
+                    }
+            elif slope>0 :
+                    findings[col]={
+                         'trend':'Positive',
+                        'value':round(float(slope),3),
+                          'Correlation':round(float((r)**2),3)
+                    }
+    return findings
+
+def correlation_analysis(df):
+    pos=[]
+    corr=df[df.select_dtypes(include=np.number).columns].corr()
+    for i in range(len(corr.columns)):
+        for j in range(i+1,len(corr.columns)):
+            value=corr.iloc[i,j]
+            if abs(value)>=0.5:
+                pos.append(
+                    {
+                        'col1':corr.columns[i],
+                        'col2':corr.columns[j],
+                        'corr':round(float(value),4),
+                        'degree':'strong' if abs(value)>=0.7 else 'moderate',
+                        'warning':'both columns might mean the same' if abs(value)>0.95 else None
+                    }
+                )
+    return pos
+
+def performer_rank(df,key_cols,domain_info):
+    performer={}
+    for cat_col in df.select_dtypes(include='object').columns:
+        for metric in key_cols:
+            if not metric in df.columns:
+                continue
+            if not np.issubdtype(df[metric].dtype, np.number):
+                continue
+            try:
+                grouped=df.groupby(cat_col)[metric].agg(
+                    ['mean','std','count'])
+                direction=domain_info['Metric_Direction'].get(metric,"lower_is_better")
+                if direction=="lower_is_better":
+                    worst=grouped['mean'].idxmax()
+                    best=grouped['mean'].idxmin()
+                else:
+                    worst=grouped['mean'].idxmin()
+                    best=grouped['mean'].idxmax()
+                performer[f"{cat_col}_{metric}"] = {
+                    'worst':str(worst),
+                    'worst_value': round(
+                        grouped.loc[str(worst), 'mean'], 4
+                    ),
+                    'best':str(best),
+                    'best_value':round(
+                        grouped.loc[str(best),'mean'], 4
+                    ),
+                    'gap':round(
+                        grouped['mean'].max()-grouped['mean'].min(),4)
+                }
+            except Exception as e:
+                print(f"Skipping {cat_col}_{metric}: {e}")
+                continue
+    return performer
+    
+
+def percentage_growth(df,key_cols):
+    growth={}
+    date_cols=[c for c in df.columns.to_list() if 'date' in c.lower()]
+    if not date_cols:
+        return {}
+    new_df=df.copy()
+    new_df[date_cols[0]]=pd.to_datetime(new_df[date_cols[0]])
+    new_df=new_df.sort_values(date_cols[0])
+    for cat_col in new_df.select_dtypes(include='object').columns:
+        for metric in key_cols:
+            if not pd.api.types.is_numeric_dtype(new_df[metric]):
+                continue
+            if metric in new_df.columns:
+                grouped=new_df.groupby(cat_col)[metric].mean()
+                growth[f'{cat_col}-{metric}']={
+                    'percent_growth':round(((grouped.iloc[-1]-grouped.iloc[0])/grouped.iloc[0]*100 if grouped.iloc[0]!=0 else 0),2)}
+    return growth
+
+def segmentation_analysis(df,key_cols):
+    findings={}
+    cat_cols=df.select_dtypes(include='object').columns
+    numeric_cols = df.select_dtypes(include=np.number).columns.tolist()
+    
+    for cat_col in cat_cols:
+        findings[cat_col]={}
+        for metric in key_cols:
+            if metric in df.columns:
+                if np.issubdtype(df[metric].dtype, np.number):
+                    grouped=df.groupby(cat_col)[metric].mean()
+    
+                    findings[cat_col][metric]={
+                        'best':str(grouped.idxmax()),
+                        'worst':str(grouped.idxmin())
+                    }
+    return findings
+
+def distribution_analysis(df,key_cols):
+    num_data=df[df.select_dtypes(include=np.number).columns]
+    findings={}
+    for col in num_data.columns:
+        if col in key_cols:
+            data=num_data[col].dropna()
+            sample=data.sample(min(len(data),5000),random_state=42)
+            _,pvalue=stats.shapiro(sample)
+            skew=float(data.skew())
+            kurt=float(data.kurt()) 
+            if pvalue>0.05:
+               label='normal'
+            elif kurt > 3:
+               label='heavy_tailed'
+            elif skew>1:
+               label='right_skewed'
+            elif abs(skew)<0.5:
+               label='symmetric'
+            elif skew<-1:
+               label = 'left_skewed'
+            elif skew>0:
+               label='slightly_right_skewed'
+            else:
+               label='slightly_left_skewed'
+            findings[col]={
+                'skew':skew,
+                'distribution':label,
+                'p_value':round(float(pvalue),2),
+                'n':len(data)
+            }
+    return findings
+
+def concentration_analysis(df,key_cols):
+    findings={}
+    cat_cols=df.select_dtypes(include='object').columns
+    
+    for cat_col in cat_cols:
+        for metric in key_cols:
+            if metric not in df.columns:
+                continue
+            if not np.issubdtype(df[metric].dtype, np.number):
+                continue 
+                
+            grouped=df.groupby(cat_col)[metric].sum()
+            total=grouped.sum()
+            if total==0:
+                continue
+            
+            top_contributor=grouped.idxmax()
+            top_pct=round(float(grouped.max()/total*100),2)
+            
+            findings[f"{cat_col}_{metric}"]={
+                'top_contributor':str(top_contributor),
+                'contribution_pct':top_pct,
+                'is_concentrated':top_pct>50}
+    
+    return findings
+    
+def pareto_summary(concentration_findings):
+    critical={
+        k:v for k, v in concentration_findings.items()
+        if v['contribution_pct']>80}
+    return critical
+
+def data_analysis(df,key_cols,domain_info):
+    findings={}
+    findings['outlier_detection']=outlier_detection(df,key_cols)
+    findings['segmentation_analysis']=segmentation_analysis(df,key_cols)
+    findings['percentage_growth']=percentage_growth(df,key_cols)
+    #findings['performer_rank']=performer_rank(df,key_cols,domain_info)
+    findings['correlation_analysis']=correlation_analysis(df)
+    findings['trend_detection']=trend_detection(df,key_cols)
+    findings['concentration_analysis']=concentration_analysis(df,key_cols)
+    findings['distribution_analysis']=distribution_analysis(df,key_cols)
+    findings['pareto_summary']=pareto_summary(findings['concentration_analysis'])
+    findings=sanitize_keys(findings)
+    return findings
+
+@traceable(name='data_analyst')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def data_analyst(df,llm,key_cols,domain_info,profiler):
+    slim={
+    'shape': profiler['shape'],
+    'columns': profiler['columns'],
+    'null_percent': profiler['null_percent'],
+    'likely_date_columns': profiler['likely_date_columns'],
+    'likely_categorical_columns': profiler['likely_categorical_columns'],
+    'likely_id_columns': profiler['likely_id_columns'],
+    'sample_rows': profiler['sample_rows'],
+    }
+    finds=data_analysis(df,key_cols,domain_info)
+    prompt=f"""You are a senior data analyst at a large enterprise company.
+You are given two inputs:
+1. A dataset profile (schema, stats, nulls, etc.)
+2. Pre-computed statistical findings from analysis functions
+
+Your job is to INTERPRET these findings in clear business language for a non-technical manager.
+
+═══════════════════════════════
+STRICT RULES — READ BEFORE ANYTHING ELSE
+═══════════════════════════════
+- ONLY reference numbers that exist in the findings JSON below. 
+  Do NOT invent, estimate, round, or extrapolate any value.
+- If a field has no supporting data, output null. Never fill gaps with guesses.
+- Do NOT use: "approximately", "likely around", "seems to suggest", "may indicate"
+  unless you are directly quoting a finding.
+- Do NOT reference columns, metrics, or entities not present in the inputs.
+- Your job is interpretation, NOT recalculation. Trust every number as-is.
+
+═══════════════════════════════
+INPUTS
+═══════════════════════════════
+Domain: {domain_info['Domain']}
+Key Metrics: {json.dumps(key_cols, default=str)}
+Dataset Profile: {json.dumps(slim, default=str)}
+Statistical Findings: {json.dumps(finds, default=str)}
+
+═══════════════════════════════
+OUTPUT — strict JSON, exact keys below
+═══════════════════════════════
+{{
+  "executive_summary": "3-4 sentences. State the domain, the 2-3 biggest issues with exact numbers from findings, and one immediate action. No vague language.",
+
+  "critical_issues": [
+    {{
+      "issue": "Short title",
+      "evidence": "Exact value from findings — e.g. z-score of X in column Y",
+      "cause": "Specific mechanism, not a generic statement",
+      "severity": "High | Medium | Low",
+      "potential_harm": "Concrete business consequence"
+    }}
+    // exactly 5 issues, ranked High to Low
+  ],
+
+  "root_cause_hypotheses": {{
+    "issue_title": "One specific hypothesis per issue grounded in the data"
+    // one key per critical issue above
+  }},
+
+  "recommended_actions": [
+    {{
+      "action": "Specific action title",
+      "target_issue": "Which critical issue this addresses",
+      "steps": ["Step 1", "Step 2"],
+      "success_metric": "Measurable outcome with a number"
+    }}
+    // 1-2 actions per issue max
+  ],
+
+  "investigation_focus": [
+    "Specific question for external research — must name a column or metric from findings",
+    "Specific question 2",
+    "Specific question 3"
+  ]
+}}
+"cause": "Name the specific system/process failure that produced this — format: '[System/Process] failed to [action] because [reason]'. NOT a restatement of the issue.",
+
+"root_cause_hypotheses": {{
+  "<issue_title>": "One level deeper than cause. Format: 'If [specific condition] exists in the [system/process], it would produce [observed effect] because [mechanism]'"
+}}
+"investigation_focus": [
+  "Must reference a specific column name AND a specific value from findings.",
+  "Must be a causal question, not a descriptive one.",
+  "❌ 'What causes missing data?' ✅ 'Why do 24.93% of CustomerID values go uncaptured in retail POS transactions under £10?'"
+]
+
+Return only valid JSON. No markdown, no backticks, no explanation outside the JSON.
+"""
+    response=llm.invoke(prompt)
+    return safe_parse(response.content)

tools.py

@@ -0,0 +1,360 @@
+import pandas as pd
+import ast
+import json
+import os
+from tavily import TavilyClient
+from langsmith import traceable
+from helper import retry, safe_parse
+
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def run_profiler(filepath):
+    encodings=['utf-8', 'latin-1', 'iso-8859-1', 'cp1252']
+    df=None
+    for encoding in encodings:
+        try:
+            df=pd.read_csv(filepath, encoding=encoding)
+            print(f"Successfully read with encoding:{encoding}")
+            break
+        except UnicodeDecodeError:
+            continue 
+    if df is None:
+        raise ValueError("Could not read file with any encoding")
+    likely_ids=[]
+    likely_dates=[]
+    likely_categorical=[]
+    likely_numeric=[]
+
+    for col in list(df.columns):
+        if df[col].dtype==object:
+            converted=pd.to_datetime(df[col],errors="coerce",infer_datetime_format=True)
+            if converted.notna().mean()>0.6: 
+                likely_dates.append(col)
+                continue
+            numeric_converted = pd.to_numeric(df[col], errors="coerce")
+            success_rate = numeric_converted.notna().sum() / len(df)
+            if success_rate > 0.8:
+                likely_numeric.append(col)
+                continue
+            unique=df[col].nunique()
+            if unique>=0.9*len(df):
+                likely_ids.append(col)
+                continue
+            if unique < 100: 
+                likely_categorical.append(col)
+            numeric_converted=pd.to_numeric(df[col], errors="coerce")
+        elif pd.api.types.is_numeric_dtype(df[col]):
+            likely_numeric.append(col)
+
+    profiler={
+        'shape':df.shape,
+        'dtypes':df.dtypes.astype(str).to_dict(),
+        'columns':df.columns.tolist(),
+        'null_counts':df.isnull().sum().to_dict(),
+        "null_percent":(df.isnull().mean() * 100).round(2).to_dict(),
+        "numeric_stats":{col: {str(k): v for k, v in stats.items()} for col, stats in df.describe().to_dict().items()},
+        "cardinality":df.nunique().to_dict(),
+        "duplicates":int(df.duplicated().sum()),
+        "skewness":{k: round(v, 3) for k, v in df.skew(numeric_only=True).items()},
+        "kurtosis":{k: round(v, 3) for k, v in df.kurt(numeric_only=True).items()},
+        "likely_date_columns":likely_dates,
+        "likely_categorical_columns": likely_categorical,
+        "likely_numeric_columns":likely_numeric,
+        "likely_id_columns":likely_ids,
+        "sample_rows":df.head(1).to_dict()
+    }
+    for col in df.columns:
+        if df[col].dtype=='object':
+            df[col]=df[col].fillna('Unknown')
+        elif pd.api.types.is_numeric_dtype(df[col]):
+            if abs(df[col].skew())<=1:
+                df[col]=df[col].fillna(df[col].mean())
+            else:
+                df[col]=df[col].fillna(df[col].median())
+    return df,profiler
+
+@traceable(name='domain_analyser')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def domain_analyser(llm,profile):
+    dtypes=profile['dtypes']
+    all_cols=profile['columns']
+
+    imp_cols=[c for c in all_cols if dtypes.get(c) in ('int64','float64','int32','float32') and not any(x in c.lower() for x in ['key','code','num','index','date','time','year','month'])]
+    imp_cols=imp_cols[:30] if len(imp_cols)>30 else imp_cols
+    prompt=f"""You are a senior business data analyst. Analyze the CSV profile and return a single valid JSON object.
+
+RULES:
+- You must never leave the Key_Metrics_to_watch output empty.
+- For finding Key_Metrics_to_watch,column names given to you should only be taken, do not take any other column names not in the profile.
+- Return ONLY valid JSON. No markdown, no backticks, no explanation.
+- Use exact key names as specified below.
+- If columns are named V1, V2, ..., Vn alongside 'Amount' and 'Class' or 'Time', this is likely finance_transactions (PCA-transformed fraud detection data)
+- Do not add any keys not listed below.
+
+OUTPUT SCHEMA:
+{{
+  "Domain": "<one of: retail_sales | manufacturing_quality | manufacturing_production | manufacturing_maintenance | hr_attrition | hr_performance | finance_transactions | logistics | other>",
+   "Subdomain": "<if Domain is 'other', give a 2-3 word label like 'clinical_outcomes' | 'student_performance' | 'crime_statistics' | 'energy_consumption' | 'environmental_monitoring' | 'sports_analytics' | 'other_unknown' etc.>",
+  "Confidence_Score": "<one of: High | Medium | Low>",
+  "Key_Metrics_to_watch": ["<col1>", "<col2>", "..."],
+  "Metric_Direction": {{
+    "<col1>": "<higher_is_better | lower_is_better>",
+    "<col2>": "<higher_is_better | lower_is_better>"
+  }},
+  "Reason": "<exactly 2 sentences explaining your domain choice>"
+}}
+
+CONSTRAINTS:
+- Key_Metrics_to_watch: only include numeric columns that are meaningful business metrics. Exclude ID columns, date columns, and free-text fields.
+- Metric_Direction: must have one entry for every column in Key_Metrics_to_watch, no more, no less.
+
+COLUMN NAMES: {imp_cols}
+
+underscores not spaces. No variation allowed.
+    """
+    response=llm.invoke(prompt)
+    response=safe_parse(response.content)
+    key_cols=response['Key_Metrics_to_watch']
+    return response,key_cols
+
+def critical_info(llm,profiler):
+    columns=profiler['columns']
+    prompt=f"""You are a data privacy analyst.
+
+You are given a list of column names from a business dataset: {columns}
+
+Your task: identify which columns contain Personally Identifiable Information (PII) 
+that must be anonymized before sending data to an external system.
+
+═══════════════════════════════
+FLAG THESE (PII):
+═══════════════════════════════
+- Person names (employee, operator, customer)
+- Employee IDs or operator IDs
+- Email addresses, phone numbers
+- National IDs, passport numbers
+- Batch or reference codes directly traceable to a specific individual
+
+═══════════════════════════════
+DO NOT FLAG THESE (not PII):
+═══════════════════════════════
+- Machine IDs, equipment codes, production line numbers
+- Temperatures, pressures, or any sensor/operational metrics
+- Timestamps, dates, shift codes
+- Product names, SKUs, categories
+- Numeric aggregates (counts, averages, totals)
+
+═══════════════════════════════
+OUTPUT
+═══════════════════════════════
+A valid Python list of column names that are PII. Nothing else.
+If no PII columns exist, return an empty list: []
+
+Example: ['EmployeeName', 'OperatorID', 'CustomerEmail']
+"""
+    response=llm.invoke(prompt)
+    try:
+        return ast.literal_eval(response.content.strip())
+    except:
+        return []
+
+@traceable(name='search_queries')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def search_queries(llm,domain,internal_findings): 
+    focus=internal_findings['investigation_focus']
+    issues=internal_findings['critical_issues']
+    issue_details = []
+    for i, issue in enumerate(issues): 
+        issue_details.append(f"""
+  - Issue {i+1}: {issue.get('issue', 'Unknown')}
+  - Severity: {issue.get('severity', 'Unknown')}
+  - Evidence: {issue.get('evidence', 'Unknown')}
+  - Why it matters: {issue.get('potential_harm', 'Unknown')}
+""")
+    prompt=f"""You are a business research analyst tasked with generating highly specific web search queries.
+CONTEXT:
+Your goal is to find ROOT CAUSES and PRACTICAL EXPLANATIONS for these specific data quality issues:
+-DOMAIN: {domain['Domain']}
+
+CRITICAL ISSUES TO RESEARCH:
+{''.join(issue_details)}
+
+INVESTIGATION FOCUS AREAS:
+{json.dumps(focus[:3], indent=2)}
+
+GOOD EXAMPLES FOR YOUR ISSUES:
+For Outliers in Quantity:
+"Why do promotional bulk orders create extreme outliers e-commerce inventory"
+"Detecting real bulk orders vs data entry errors retail systems"
+"Supply chain disruptions causing quantity outliers 2024"
+
+OUTPUT REQUIREMENTS:
+1. Each query MUST be a single line, enclosed in double quotes
+2. NO bullet points, NO markdown, NO explanations
+3. Return ONLY the queries, one per line
+
+QUERY DESIGN RULES (MANDATORY):
+1. SPECIFIC ROOT CAUSES, not symptoms
+   ❌ BAD: "retail sales issues"
+   ✅ GOOD: "Why do bulk orders cause high quantity outliers e-commerce"
+
+2. Include mechanism/process, not just the problem
+   ❌ BAD: "missing customer IDs"
+   ✅ GOOD: "Guest checkout flows that bypass customer ID capture"
+
+3. Target REAL-WORLD IMPLEMENTATIONS, not academic theory
+   ❌ BAD: "customer data collection best practices"
+   ✅ GOOD: "Shopify/WooCommerce guest checkout missing customer field 2024"
+
+4. Include INDUSTRY CONTEXT (retail, e-commerce, data quality, etc.)
+   ❌ BAD: "Why do systems fail"
+   ✅ GOOD: "Root cause of duplicate invoice numbers in retail POS systems"
+
+5. Include YEAR or RECENCY if applicable
+   ❌ BAD: "data entry errors"
+   ✅ GOOD: "Data validation failures retail 2024 industry case studies"
+
+ANTI-PATTERNS (Avoid these):
+- Generic statistics ("retail trends", "market analysis")
+- Textbook definitions ("what is data quality")
+- Generic best practices ("how to prevent errors")
+- Vague terminology ("unusual patterns", "data issues")
+
+GOOD EXAMPLES FOR YOUR ISSUES:
+For Outliers in Quantity:
+"Why do promotional bulk orders create extreme outliers e-commerce inventory"
+"Detecting real bulk orders vs data entry errors retail systems"
+"Supply chain disruptions causing quantity outliers 2024"
+
+For Missing CustomerID:
+"Why guest checkout abandons customer ID field collection e-commerce"
+"Shopify/WooCommerce missing customer data rates implementation"
+"How to enforce customer ID capture checkout validation rules"
+
+Strictly output only a valid JSON array like this:
+[
+  "Query 1 here",
+  "Query 2 here",
+  "Query 3 here",
+  "Query 4 here",
+  "Query 5 here",
+  "Query 6 here"
+]
+"Return ONE single flat JSON array containing ALL queries together. Do NOT group by issue. Do NOT return multiple arrays."
+No other text. No markdown. No backticks."""
+
+    response=llm.invoke(prompt)
+    try:
+        content=response.content 
+        return safe_parse(content)  
+    except Exception as e:
+        print(f"Error: {e}")
+        return []
+
+def clean_results(search_results):
+    extracted=[]
+    for result in search_results:
+        contents=[r['content'] for r in result['response']['results']]
+        extracted.append(
+            {
+                'query':result['query'],
+                'content':contents
+            }
+        )
+    return extracted
+
+@traceable(name='web_search')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def web_search(queries):
+    results=[]
+    search_client=TavilyClient(api_key=os.environ["TAVILY_API_KEY"])
+    for query in queries:
+        result=search_client.search(query=query.strip(),max_results=3,search_depth="basic")
+        results.append({'query':query,'response':result})
+    return clean_results(results)
+
+@traceable(name='causal_reasoning')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def causal_reasoning(llm,domain_info,interpreted_findings,cleaned_results):
+    internal={
+        'critical_issues':interpreted_findings['critical_issues'],
+        'root_cause_hypotheses':interpreted_findings['root_cause_hypotheses'],
+        'executive_summary':interpreted_findings['executive_summary']
+    }
+    external_block=json.dumps(cleaned_results, indent=2) if cleaned_results else "NO EXTERNAL RESEARCH AVAILABLE"
+    prompt=f"""You are a causal inference specialist. Evaluate whether the available evidence supports,
+partially supports,contradicts or is insufficient to infer causality. Never state a causal relationship as fact unless both
+internal evidence and external evidence support it.
+INPUTS:
+Domain: {domain_info}
+Internal Findings: {json.dumps(internal, indent=2, default=str)}
+External Research: {external_block}
+
+FOR EACH critical issue, complete all 4 gates:
+
+GATE 1 — FACT: State exact metric + value + unit + % of total. No vague words (high/notable/significant banned).
+
+GATE 2 — MECHANISM: [External Factor] → [Specific Process] → [Internal Effect]. 
+If the intermediate process is unknown,
+describe the uncertainty explicitly and reduce confidence accordingly.
+Do not invent missing process steps.
+
+GATE 3 — VARIANCE: Only assign numerical percentages if the internal findings,
+external evidence, or domain knowledge explicitly supports them.
+Otherwise use:
+"primary":"Calibration drift — dominant contributor (percentage unknown)"
+"secondary":"Operator delay — secondary contributor (percentage unknown)"
+"unexplained":"Requires maintenance logs"
+Never invent percentages.
+
+GATE 4 — FALSIFIABILITY: One specific data point that confirms. One that refutes.
+
+DOMAIN CAPS (automatic confidence limits):
+- Geographic concentration >70%: MEDIUM cap unless regional expansion data present
+- Missing data >20%: MEDIUM cap until source audited  
+- Outliers present: MEDIUM cap without transaction logs
+
+CITATION RULE: For external_evidence, you MUST copy a verbatim phrase from the research above and name the query it came from.
+Format: "Source query: '<query>' | Finding: '<copied phrase>'"
+If external_block is empty or NO EXTERNAL RESEARCH AVAILABLE, set external_evidence=null and verdict=no_external_evidence for all issues.
+
+RULES:
+- Every percentage has a source tag: (industry benchmark) / (pilot data) / (estimate pending validation). Example: 82.37% → 65.90% (20% reduction) and 186.51 → 130.56 (30% reduction) is found in the report - these must be backed up with valid ecternal citations like
+industry benchmarks or other target metrics found on external research. If a claim percentage has no source compulsorily omit the percentage claim instead say something like requires baseline period data to set target.
+- external_evidence must quote a specific finding from research above, or set null
+- All JSON string values single-line (no literal newlines inside strings)
+-If internal evidence and external research disagree preserve both,explain why. Do NOT average them and set verdict="contradicted".
+- If external research is unavailable, set verdict="no_external_evidence" and keep all fields except verdict null
+
+OUTPUT:
+{{
+  "causal_connections": [
+    {{
+      "issue": "<label + key metric>",
+      "internal_evidence": "<exact metric, value, unit, n>",
+      "external_evidence":{{"source_query":"","quoted_finding":"","relevance":"Explain in one sentence how this research relates to the internal issue."}},
+      "causal_mechanism": "<[Factor] → [Process] → [Effect] in [timeframe]>",
+      "variance_decomposition": {{
+        "primary": "<cause: X%> — <one-line justification>",
+        "secondary": "<cause: Y%> — <justification>",
+        "unexplained": "<Z%> — <what data resolves this>"
+      }},
+      "confidence": "<HIGH|MEDIUM|LOW>: <one sentence citing variance band>",
+      "confirmation_test": "<specific data point>",
+      "refutation_test": "<specific data point>",
+      "verdict": "<supported|contradicted|no_external_evidence>"
+    }}
+  ],
+  "overall_diagnosis": "<3 sentences max: dominant causal story, confidence levels, highest uncertainty link>",
+  "meta_assessment": {{
+    "binding_constraint": "<single data gap most limiting analysis>",
+    "validation_sequence": ["<step 1>", "<step 2>", "<step 3>"]
+  }},
+  "evidence_strength": {{
+    "internal": "<Strong/Moderate/Weak>: <one sentence justification>",
+    "external": "<Strong/Moderate/Weak>: <one sentence justification>",
+    "overall": "<Strong/Moderate/Weak>: <one sentence justification>"
+  }}
+}}"""
+    response=llm.invoke(prompt)
+    return safe_parse(response.content)
+

visuals.py

@@ -0,0 +1,574 @@
+import plotly.express as px
+import plotly.io as pio
+import plotly.graph_objects as go
+import pandas as pd
+import plotly.offline as pyo
+from IPython.display import display
+import json
+import numpy as np
+from crypto import decryption
+from helper import safe_parse,derive_accents,random_template,retry
+from langsmith import traceable
+pyo.init_notebook_mode(connected=True)
+
+def render_charts(inputs,df):
+    figs=[]
+    inputs=validate_charts(inputs,df)
+    for c in inputs:
+        try:
+            chart_type=c['chart_type'] 
+            x=c.get('x')
+            y=c.get('y')
+            z=c.get('z') 
+            color=c.get('color') 
+            priority=c.get('priority', 1)
+            
+            if chart_type=='stacked_bar':
+                if x not in df.columns or y not in df.columns or color not in df.columns:
+                    print(f"Skipping stacked_bar: missing columns {x}, {y}, {color}")
+                    continue
+                grouped=df.groupby([x,color])[y].sum().reset_index()
+                if grouped.empty:
+                    print(f"Skipping stacked_bar: no data after grouping")
+                    continue
+                fig1=px.bar(grouped,x=x,y=y,color=color,barmode='stack',title=f'{x} by {y} and {color}',template='plotly_white')
+                figs.append(fig1)
+
+            elif chart_type=='bar':
+                if x not in df.columns or y not in df.columns:
+                    print(f"Skipping bar: missing columns {x}, {y}")
+                    continue
+                grouped=df.groupby(x)[y].sum().reset_index()
+                if grouped.empty:
+                    print(f"Skipping bar: no data after grouping")
+                    continue
+                fig2=px.bar(grouped,x=x,y=y,title=f'{x} vs {y}',template='plotly_white')
+                figs.append(fig2)
+
+            elif chart_type=='histogram':
+                col=x if x else y
+                if col not in df.columns:
+                    print(f"Skipping histogram: column {col} not found")
+                    continue
+                fig3=px.histogram(df,x=col,nbins=20,marginal='box',title=f'Distribution of {col}',template='plotly_white')
+                figs.append(fig3)
+
+            elif chart_type=='treemap':
+                if x not in df.columns or y not in df.columns:
+                    print(f"Skipping treemap: missing columns {x}, {y}")
+                    continue
+                grouped=df.groupby(x)[y].sum().reset_index()
+                grouped=grouped[grouped[y] > 0]
+                if grouped.empty:
+                    print(f"Skipping treemap: no positive data after grouping")
+                    continue
+                fig4=px.treemap(grouped,path=[px.Constant('All'),x],values=y,title=f'{x} vs {y}',color=y,color_continuous_scale='RdYlGn_r',template='plotly_white')
+                figs.append(fig4)
+
+            elif chart_type=='pareto':
+                if x not in df.columns or y not in df.columns:
+                    print(f"Skipping pareto: missing columns {x}, {y}")
+                    continue
+                grouped =df.groupby(x)[y].mean().reset_index()
+                grouped =grouped.sort_values(y, ascending=False)
+                if grouped.empty:
+                    print(f"Skipping pareto: no data after grouping")
+                    continue
+                grouped['cumulative_pct']=(
+                    grouped[y].cumsum()/grouped[y].sum()*100
+                )
+                
+                fig5= go.Figure()
+                fig5.add_trace(go.Bar(x=grouped[x],y=grouped[y],name=y))
+                fig5.add_trace(go.Scatter(
+                    x=grouped[x], 
+                    y=grouped['cumulative_pct'],
+                    name='cumulative %',
+                    yaxis='y2',
+                    mode='lines+markers'
+                ))
+                fig5.update_layout(
+                    yaxis2=dict(overlaying='y',side='right',range=[0,100]))
+                figs.append(fig5)
+
+            elif chart_type=='correlation_heatmap':
+                numeric_df = df.select_dtypes(include='number')
+                if numeric_df.shape[1] < 2:
+                    print(f"Skipping correlation_heatmap: not enough numeric columns")
+                    continue
+                corr = numeric_df.corr()
+                fig6=px.imshow(corr,color_continuous_scale='RdBu_r',zmin=-1,zmax=1,title=f'Correlation Heatmap',text_auto=True,template='plotly_white')
+                figs.append(fig6)
+
+            elif chart_type=='scatter':
+                if x not in df.columns or y not in df.columns:
+                    print(f"Skipping scatter: missing columns {x}, {y}")
+                    continue
+                fig7=px.scatter(df,x=x,y=y,color=color if color else None,trendline='ols',title=f'{x} vs {y}',template='plotly_white')
+                figs.append(fig7)
+
+            elif chart_type=='line':
+                if x not in df.columns or y not in df.columns:
+                    print(f"Skipping line: missing columns {x}, {y}")
+                    continue
+                df_sorted=df.copy()
+                try:
+                    df_sorted[x]=pd.to_datetime(df_sorted[x], errors='coerce')
+                    df_sorted=df_sorted.dropna(subset=[x])
+                    df_sorted=df_sorted.sort_values(x)
+                    grouped=df_sorted.groupby(x)[y].mean().reset_index()
+                    if grouped.empty:
+                        print(f"Skipping line: no data after grouping")
+                        continue
+                    fig8=px.line(grouped,x=x,y=y,markers=True,title=f'{y} over time',template='plotly_white')
+                    figs.append(fig8)
+                except Exception as e:
+                    print(f"Skipping line chart: {e}")
+                    continue
+
+            elif chart_type=='pivot_heatmap':
+               if z not in df.columns or x not in df.columns or y not in df.columns:
+                    print(f"Skipping pivot_heatmap: missing columns {x}, {y}, {z}")
+                    continue
+               if not np.issubdtype(df[z].dtype, np.number):
+                    print(f"Skipping pivot_heatmap: '{z}' is not numeric")
+                    continue
+               pivot=df.pivot_table(values=z,index=x,columns=y,aggfunc='mean')
+               if pivot.empty:
+                    print(f"Skipping pivot_heatmap: pivot table is empty")
+                    continue
+               fig9=px.imshow(pivot,color_continuous_scale='RdYlGn_r',text_auto=True,template='plotly_white',title=f'{z} by {x} and {y}')
+               figs.append(fig9)
+
+            else:
+                print(f"Unknown chart type: {chart_type}")
+        except Exception as e:
+            print(f"[ERROR] Failed to render {c.get('chart_type')}: {e}")
+            import traceback
+            traceback.print_exc()
+            continue
+    
+    return figs
+
+def validate_charts(chart_recs,df):
+    valid=[]
+    for chart in chart_recs:
+        ct=chart.get('chart_type')
+        x=chart.get('x')
+        y=chart.get('y')
+        z=chart.get('z')
+        color=chart.get('color')
+        if not ct:
+            print(f"Skipping chart: missing chart_type")
+            continue
+        if isinstance(x, list):
+            if ct != 'correlation_heatmap':
+                print(f"Skipping {ct}: x is a list {x}, not a string")
+                continue
+        if isinstance(y, list):
+            print(f"Skipping {ct}: y is a list {y}, not a string")
+            continue
+        if isinstance(z, list):
+            print(f"Skipping {ct}: z is a list {z}, not a string")
+            continue
+        if isinstance(color, list):
+            print(f"Skipping {ct}: color is a list {color}, not a string")
+            continue
+        if x is None and ct not in ['correlation_heatmap']:
+            print(f"Skipping {ct}: x column is None")
+            continue
+        if y is None and ct not in ['histogram', 'correlation_heatmap']:
+            print(f"Skipping {ct}: y column is None")
+            continue
+        is_valid = True
+        for col in [x, y, z, color]:
+            if isinstance(col, list):
+                for c in col:
+                    if c not in df.columns:
+                        print(f"Skipping {ct}: column '{c}' not found in dataframe")
+                        is_valid = False
+                        break
+            elif col and col not in df.columns:
+                print(f"Skipping {ct}: column '{col}' not found in dataframe")
+                is_valid = False
+                break
+        if not is_valid:
+            continue
+            
+        if ct in ['scatter','histogram'] and x and isinstance(x, str) and x in df.columns:
+            if not pd.api.types.is_numeric_dtype(df[x]):
+                print(f"Skipping {ct}: '{x}' is not numeric")
+                continue
+        if ct in ['scatter','bar'] and y and isinstance(y, str) and y in df.columns:
+            if not pd.api.types.is_numeric_dtype(df[y]):
+                print(f"Skipping {ct}: '{y}' is not numeric")
+                continue
+        valid.append(chart)
+    
+    print(f"Validated {len(valid)}/{len(chart_recs)} charts")
+    return valid  
+
+@traceable(name='deciding_plots')
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def deciding_plots(llm,df,findings, columns, key_cols, feedback=None):
+    numeric_cols = df.select_dtypes(include=np.number).columns.tolist()
+    categorical_cols =[c for c in df.columns if df[c].nunique() < 50 and c not in numeric_cols]
+    feedback_section = f"""
+REVISION FEEDBACK: {feedback}
+Address this feedback specifically in your chart selection.""" if feedback else ""
+    
+    prompt=f"""You are a senior data analyst. Select exactly 6 charts for a BI dashboard.
+
+REQUIRED (4 charts):
+1. stacked_bar: x=category, y=metric, color=DIFFERENT_category
+2. pivot_heatmap: x=category, y=category, z=numeric_metric
+3. correlation_heatmap: 3. correlation_heatmap: x=null, y=null, z=null (auto-uses all numeric columns)
+4. histogram: x=numeric_metric (distribution)
+
+OPTIONAL (2 charts, choose from):
+- bar, line, scatter, treemap, pareto
+
+STRICT RULES:
+1. Each chart must have unique x,y combination (no duplicates)
+2. stacked_bar: x and color MUST be different columns
+3. pivot_heatmap: z (values) must be numeric
+4. correlation_heatmap: include only numeric columns
+5. Avoid: ID columns, high-cardinality columns (>100 unique values)
+6. Order by business impact: Revenue > Quantity > Price > Distribution
+7. pareto: ONLY if concentration_analysis shows is_concentrated=true
+8. treemap: ONLY if any category >50% of total
+
+EXAMPLES (good):
+- stacked_bar: x=Country, y=Revenue, color=ProductCategory ✓
+- bar: x=Country, y=Quantity (simple relationship) ✓
+- Bad: stacked_bar: x=Country, y=Revenue, color=Country ✗ (duplicate)
+
+CAUTION:
+1. NEVER recommend scatter/line with non-numeric columns
+2. NEVER use columns: {[c for c in columns if c not in numeric_cols + categorical_cols]}
+3. For any chart using {categorical_cols}, verify it's categorical first
+4. If a column is not in NUMERIC list, do NOT use it for x/y in scatter/line
+
+Allowed columns: {columns}
+Key metrics: {key_cols}
+
+Data findings: {json.dumps(findings, indent=2, default=str)}
+
+{feedback_section}
+
+CRITICAL VALIDATION RULES:
+- EVERY chart MUST have ALL required columns filled (NO null values for x, y, color fields)
+- x: MUST be a valid column name from allowed columns
+- y: MUST be a valid column name from allowed columns  
+- z: MUST be a valid column name from allowed columns (for pivot_heatmap, correlation_heatmap, or treemap)
+- color: MUST be a valid column name from allowed columns (for stacked_bar only)
+- NEVER use null for required fields. If a required field cannot be filled, SKIP that chart type
+
+RESPOND ONLY WITH VALID JSON (no markdown, no explanation):
+[
+  {{"chart_type": "stacked_bar", "x": "product_line", "y": "sales", "z": null, "color": "region", "priority": 1, "reason": "..."}},
+  {{"chart_type": "histogram", "x": "revenue", "y": null, "z": null, "color": null, "priority": 2, "reason": "..."}},
+  ...
+]"""
+    response = llm.invoke(prompt)
+    charts = safe_parse(response.content)
+    return charts
+
+@retry(max_attempts=3,delay=3,backoff=3,exceptions=(Exception,))
+def build_dashboard(df,key_cols,inputs,key,mapping_log,domain_info):
+    inputs=validate_charts(inputs,df)
+    inputs=inputs[:6]  
+
+    if not inputs:
+        raise ValueError("No valid charts after validation")
+
+    decrypt_map=decryption(key, mapping_log)
+    
+    temp,bs,dp=random_template()
+    
+    try:
+        figs=render_charts(inputs, df)
+    except Exception as e:
+        print(f"[ERROR] render_charts failed: {e}")
+        raise
+
+    chart_htmls_top=[]
+    chart_htmls_bot=[]
+    for i, fig in enumerate(figs):
+        try:
+            fig.update_layout(
+                template='plotly_white',
+                margin=dict(t=40, b=20, l=16, r=16),
+                height=300 if i < 2 else 240,
+                paper_bgcolor=dp["card"],
+                plot_bgcolor=dp["card"],
+                font=dict(family="Inter, Segoe UI, sans-serif", size=11, color=dp["text"]),
+                title_font=dict(size=12, color=dp["text"], family="Inter, Segoe UI, sans-serif"),
+                legend=dict(font=dict(size=10, color=dp["muted"]), bgcolor="rgba(0,0,0,0)"),
+            )
+        except Exception as e:
+            print(f"[WARNING] Failed to update layout for chart {i}: {e}, using default template")
+            fig.update_layout(
+                margin=dict(t=40, b=20, l=16, r=16),
+                height=300 if i < 2 else 240,
+                font=dict(family="Inter, Segoe UI, sans-serif", size=11),
+                title_font=dict(size=12, family="Inter, Segoe UI, sans-serif"),
+            )
+        
+        try:
+            html_chunk=pio.to_html(fig,full_html=False,include_plotlyjs='cdn')
+            if i<2:
+                chart_htmls_top.append(f'<div class="chart-card">{html_chunk}</div>')
+            else:
+                chart_htmls_bot.append(f'<div class="chart-card-sm">{html_chunk}</div>')
+        except Exception as e:
+            print(f"[ERROR] Failed to convert chart {i} to HTML: {e}")
+            continue
+
+    top_row="".join(chart_htmls_top)
+    bot_row="".join(chart_htmls_bot)
+    date_cols = [c for c in df.columns if 'date' in c.lower()]
+    if date_cols:
+        df = df.copy()
+        df[date_cols[0]] = pd.to_datetime(df[date_cols[0]], errors='coerce')
+        df = df.sort_values(date_cols[0])
+    metric_direction=domain_info.get("Metric_Direction", {})
+    kpis = []
+    for metric in key_cols:
+        if metric not in df.columns:
+            continue
+        if not np.issubdtype(df[metric].dtype, np.number):
+            continue
+        mid=len(df)//2
+        base=df[metric].iloc[:mid].mean()
+        delta=((df[metric].iloc[mid:].mean()-base) /base*100) if base != 0 else 0
+        val=float(df[metric].mean())
+        val_str=f"{val/1000:.2f}K" if val >= 1000 else str(round(val, 2))
+        kpis.append({
+            "label":     metric.replace("_", " ").title(),
+            "value":     val_str,
+            "direction": metric_direction.get(metric, "higher_is_better"),
+            "delta":     round(delta, 2),
+        })
+    kpis=kpis[:6]
+
+    accents=[dp["accent"],dp["accent2"],dp["muted"],
+               dp["accent"],dp["accent2"],dp["muted"]]
+
+    def kpi_card(k, accent):
+        sign="▲" if k["delta"] >= 0 else "▼"
+        is_good=(k["delta"] >= 0 and k["direction"] == "higher_is_better") or \
+                  (k["delta"] <  0 and k["direction"] == "lower_is_better")
+        dcolor="#16a34a" if is_good else "#dc2626"
+        return f"""
+        <div class="kpi-card" style="border-top:3px solid {accent}">
+            <div class="kpi-label">{k['label']}</div>
+            <div class="kpi-value">{k['value']}</div>
+            <div class="kpi-delta" style="color:{dcolor}">{sign} {abs(k['delta'])}% vs prior</div>
+        </div>"""
+
+    kpi_html="".join(kpi_card(k, accents[i]) for i, k in enumerate(kpis))
+
+    cat_cols  = list(df.select_dtypes(include="object").columns[:7])
+    btn_colors = derive_accents(dp["accent"], max(len(cat_cols), 4))
+    sidebar_items = ""
+    for j, col in enumerate(cat_cols):
+        color = btn_colors[j % len(btn_colors)]
+        sidebar_items += f"""
+        <div class="nav-btn" style="background:{color}">
+            {col.replace("_"," ").title()}
+        </div>"""
+
+    filter_col=cat_cols[0] if cat_cols else None
+    filter_vals=list(df[filter_col].dropna().unique()[:5]) if filter_col else []
+    filter_pills=""
+    if filter_vals:
+        for v in filter_vals:
+            filter_pills+=f'<div class="filter-pill">{v}</div>'
+        filter_section=f"""
+        <div class="filter-group">
+            <span class="filter-label">{filter_col.replace("_"," ").title() if filter_col else ""}</span>
+            {filter_pills}
+        </div>"""
+    else:
+        filter_section=""
+
+    title=domain_info.get("Domain","Analytics").replace("_", " ").title()
+    subdomain=domain_info.get("Subdomain", "")
+    confidence=domain_info.get("Confidence_Score", "")
+
+    html=f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>{title} Dashboard</title>
+<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap" rel="stylesheet">
+<style>
+*,*::before,*::after{{box-sizing:border-box;margin:0;padding:0}}
+body{{font-family:'Inter','Segoe UI',sans-serif;background:{dp['bg']};color:{dp['text']};min-height:100vh;display:flex;flex-direction:column}}
+
+/* ── HEADER ── */
+.header{{
+    background:{dp['sidebar']};
+    padding:0 24px;
+    display:flex;align-items:center;justify-content:space-between;
+    flex-shrink:0;
+}}
+.header-left{{display:flex;flex-direction:column;justify-content:center;padding:10px 0}}
+.header-title{{
+    font-size:22px;font-weight:800;
+    color:{dp['header_text']};
+    letter-spacing:1px;text-transform:uppercase;
+    line-height:1.1;
+}}
+.header-sub{{
+    font-size:11px;font-weight:400;
+    color:{dp['header_text']}88;margin-top:2px;
+}}
+.header-right{{display:flex;align-items:center;gap:8px;flex-direction:column;align-items:flex-end;padding:8px 0}}
+.pills-row{{display:flex;gap:6px;align-items:center}}
+.filter-group{{display:flex;align-items:center;gap:6px}}
+.filter-label{{font-size:10px;font-weight:600;color:{dp['header_text']}88;text-transform:uppercase;letter-spacing:0.6px}}
+.filter-pill{{
+    font-size:10px;font-weight:600;padding:4px 10px;
+    border-radius:4px;border:1px solid {dp['header_text']}44;
+    color:{dp['header_text']};background:{dp['header_text']}15;
+    cursor:pointer;transition:background 0.15s;
+}}
+.filter-pill:hover{{background:{dp['header_text']}30}}
+.theme-pill{{
+    font-size:10px;font-weight:600;padding:4px 10px;border-radius:4px;
+    border:1px solid {dp['accent']}66;color:{dp['accent']};background:{dp['accent']}18;
+    text-transform:uppercase;letter-spacing:0.4px;
+}}
+.confidence-pill{{
+    font-size:10px;font-weight:600;padding:4px 10px;border-radius:4px;
+    background:#16a34a22;color:#16a34a;border:1px solid #16a34a55;
+    text-transform:uppercase;letter-spacing:0.4px;
+}}
+
+/* ── LAYOUT ── */
+.body-wrap{{display:flex;flex:1;overflow:hidden}}
+
+/* ── SIDEBAR ── */
+.sidebar{{
+    width:150px;background:{dp['sidebar']};
+    flex-shrink:0;padding:16px 10px;
+    display:flex;flex-direction:column;gap:8px;
+}}
+.sidebar-label{{
+    font-size:9px;font-weight:700;
+    color:{dp['header_text']}44;letter-spacing:1.2px;
+    text-transform:uppercase;margin-bottom:4px;
+}}
+.nav-btn{{
+    font-size:11px;font-weight:600;
+    color:#ffffff;
+    padding:9px 12px;
+    border-radius:5px;
+    cursor:pointer;
+    text-align:center;
+    opacity:0.92;
+    transition:opacity 0.15s,transform 0.1s;
+    white-space:nowrap;overflow:hidden;text-overflow:ellipsis;
+}}
+.nav-btn:hover{{opacity:1;transform:translateX(2px)}}
+
+/* ── MAIN ── */
+.main{{flex:1;padding:16px;overflow-y:auto;display:flex;flex-direction:column;gap:14px}}
+
+/* ── KPI ROW ── */
+.kpi-grid{{display:grid;grid-template-columns:repeat(6,1fr);gap:10px}}
+.kpi-card{{
+    background:{dp['card']};
+    border:1px solid {dp['border']};
+    border-radius:6px;
+    padding:14px 14px 12px;
+    display:flex;flex-direction:column;gap:3px;
+}}
+.kpi-label{{font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:0.8px;color:{dp['muted']}}}
+.kpi-value{{font-size:28px;font-weight:800;line-height:1.1;letter-spacing:-0.5px;color:{dp['text']}}}
+.kpi-delta{{font-size:10px;font-weight:500;margin-top:2px}}
+
+/* ── TOP CHARTS (2 col) ── */
+.charts-top{{display:grid;grid-template-columns:1fr 1fr;gap:12px}}
+.chart-card{{
+    background:{dp['card']};border:1px solid {dp['border']};
+    border-radius:6px;padding:14px;overflow:hidden;
+}}
+
+/* ── BOTTOM CHARTS (up to 4 col) ── */
+.charts-bot{{display:grid;grid-template-columns:repeat(4,1fr);gap:12px}}
+.chart-card-sm{{
+    background:{dp['card']};border:1px solid {dp['border']};
+    border-radius:6px;padding:12px;overflow:hidden;
+}}
+
+/* ── SECTION LABEL ── */
+.section-label{{
+    font-size:9px;font-weight:700;text-transform:uppercase;
+    letter-spacing:1px;color:{dp['muted']};margin-bottom:6px;
+}}
+
+/* ── FOOTER ── */
+.footer{{
+    background:{dp['sidebar']};
+    padding:8px 24px;
+    display:flex;justify-content:space-between;align-items:center;
+    font-size:10px;color:{dp['header_text']}66;flex-shrink:0;
+}}
+.footer-brand{{font-weight:700;color:{dp['accent']};letter-spacing:0.3px}}
+</style>
+</head>
+<body>
+
+<!-- HEADER -->
+<div class="header">
+    <div class="header-left">
+        <div class="header-title">{title} Analytics Dashboard</div>
+        {f'<div class="header-sub">/ {subdomain.replace("_"," ").title()}</div>' if subdomain and subdomain != "other_unknown" else ""}
+    </div>
+    <div class="header-right">
+        {filter_section}
+        <div class="pills-row">
+            <span class="theme-pill">Theme: {bs}</span>
+            {f'<span class="confidence-pill">Confidence: {confidence}</span>' if confidence else ""}
+        </div>
+    </div>
+</div>
+<div class="body-wrap">
+    <!-- SIDEBAR -->
+    <div class="sidebar">
+        <div class="sidebar-label">Dimensions</div>
+        {sidebar_items}
+    </div>
+    <!-- MAIN -->
+    <div class="main">
+        <!-- KPIs -->
+        <div>
+            <div class="section-label">Key Performance Indicators</div>
+            <div class="kpi-grid">{kpi_html}</div>
+        </div>
+        <!-- TOP CHARTS -->
+        {'<div><div class="section-label">Charts &amp; Analysis</div><div class="charts-top">' + top_row + '</div></div>' if top_row else ''}
+        <!-- BOTTOM CHARTS -->
+        {'<div><div class="charts-bot">' + bot_row + '</div></div>' if bot_row else ''}
+    </div>
+</div>
+<!-- FOOTER -->
+<div class="footer">
+    <span>Last refreshed: <span id="ts"></span></span>
+    <span class="footer-brand">ArgusAI &nbsp;·&nbsp; Automated Insight Engine</span>
+</div>
+<script>document.getElementById('ts').textContent = new Date().toLocaleString();</script>
+</body>
+</html>"""
+    for display,real in decrypt_map.items():
+        html=html.replace(display,real)
+    with open("outputs/dashboard.html","w",encoding="utf-8") as f:
+        f.write(html)
+    print(f"Dashboard saved. Theme: {bs}")
+    return "outputs/dashboard.html"
+

workflow.py

@@ -0,0 +1,66 @@
+from langgraph.graph import StateGraph,END
+import sqlite3
+from langgraph.checkpoint.sqlite import SqliteSaver
+from agent import AgentState
+from agent import node_profiler,node_analyser,node_anonymiser,node_data_analyser,node_query,node_search,node_reasoner,node_reporter,node_plotdecider,node_dashboard,node_reportgen,human_inloop_report,human_inloop_dashboard,should_proceed_dashboard,check_error,should_proceed_report
+
+graph=StateGraph(AgentState)
+
+graph.add_node("node_1",node_profiler)
+graph.add_node("node_2",node_analyser)
+graph.add_node("node_3",node_anonymiser)
+graph.add_node("node_4",node_data_analyser)
+graph.add_node("node_5",node_query)
+graph.add_node("node_6",node_search)
+graph.add_node("node_7",node_reasoner)
+graph.add_node("node_8",node_reporter)
+graph.add_node("node_9",node_plotdecider)
+graph.add_node("node_10",node_dashboard)
+graph.add_node("node_11",node_reportgen)
+graph.add_node("check_report",human_inloop_report)
+graph.add_node("check_dashboard",human_inloop_dashboard)
+
+graph.set_entry_point("node_1")
+graph.add_edge("node_2", "node_3")
+graph.add_edge("node_3", "node_4")
+graph.add_edge("node_4", "node_5")
+graph.add_edge("node_5", "node_6")
+graph.add_edge("node_6", "node_7")
+graph.add_edge("node_7", "node_8")
+graph.add_edge("node_8","check_report")
+graph.add_edge("node_9", "check_dashboard")
+graph.add_edge("node_10","node_11")
+graph.add_edge("node_11", END)
+
+graph.add_conditional_edges(
+    "node_1",
+    check_error,
+    {'continue': 'node_2', 'end': END}
+)
+graph.add_conditional_edges(
+    "check_report",
+    should_proceed_report,
+    {
+        "node_9": "node_9",
+        "revise_report": "node_8",
+    }
+)
+graph.add_conditional_edges(
+    "check_dashboard",
+    should_proceed_dashboard,
+    {
+        "node_10": "node_10",
+        "revise_charts": "node_9",
+    }
+)
+conn = sqlite3.connect(
+    "checkpoints.db",
+    check_same_thread=False
+)
+
+memory = SqliteSaver(conn)
+
+pipeline = graph.compile(
+    checkpointer=memory,
+    interrupt_before=["check_report", "check_dashboard"],
+)