Update app.py

app.py

@@ -3,27 +3,21 @@ import pandas as pd
 import numpy as np
 import json
 import plotly.graph_objects as go
-import plotly.express as px
-from datetime import datetime
-import hashlib
 from typing import Dict, List, Tuple, Optional
-import re
-from collections import defaultdict
-import seaborn as sns
-import matplotlib.pyplot as plt
-from io import BytesIO
-import base64
 
 # Import evaluation modules
 from evaluator_module import AetherScoreEvaluator
 from visualizer_module import EvaluationVisualizer
 from report_generator import ReportGenerator
 
-# Initialize components
+# --- Global Components & Storage ---
 evaluator = AetherScoreEvaluator()
 visualizer = EvaluationVisualizer()
 report_gen = ReportGenerator()
 
+# In-memory storage for explainability feature
+evaluation_storage: Dict[str, Dict] = {}
+
 # CSS for better styling
 custom_css = """
 .gradio-container {
@@ -36,65 +30,58 @@ custom_css = """
     color: white;
     margin: 10px 0;
 }
-.leaderboard-row {
-    transition: all 0.3s ease;
-}
-.leaderboard-row:hover {
-    transform: translateX(5px);
-    box-shadow: 0 4px 6px rgba(0,0,0,0.1);
-}
 """
 
+# --- Core Functions ---
+
 def process_single_evaluation(
     prompt: str,
     response: str,
     expected_answer: Optional[str] = None,
     agent_name: str = "Agent-1",
     task_type: str = "general"
-) -> Tuple[Dict, str, str, str]:
+) -> Tuple[Dict, go.Figure, go.Figure, str]:
     """Process a single agent response evaluation"""
-    
     if not prompt or not response:
-        return {}, "", "", "Please provide both prompt and response"
-    
+        return {}, go.Figure(), go.Figure(), "Please provide both prompt and response."
+
     # Evaluate the response
-    scores = evaluator.evaluate_single(
+    eval_result = evaluator.evaluate_single(
         prompt=prompt,
         response=response,
         expected_answer=expected_answer,
         task_type=task_type
     )
+    scores = eval_result.get("scores", {})
     
+    # Store result for explainability
+    eval_id = scores.get("eval_id", "")
+    if eval_id:
+        evaluation_storage[eval_id] = eval_result
+        print(f"Stored evaluation {eval_id}")
+
     # Generate visualizations
     spider_chart = visualizer.create_spider_chart(scores, agent_name)
     score_bars = visualizer.create_score_bars(scores, agent_name)
-    
+
     # Generate explanation
-    explanation = evaluator.generate_explanation(scores, prompt, response)
-    
+    explanation = evaluator.generate_explanation(scores)
+
     # Format scores for display
-    scores_display = {
-        "Overall Score": f"{scores.get('overall_score', 0):.2f}",
-        "Instruction Following": f"{scores.get('instruction_following', 0):.2f}",
-        "Hallucination Score": f"{scores.get('hallucination_score', 0):.2f}",
-        "Assumption Control": f"{scores.get('assumption_control', 0):.2f}",
-        "Coherence": f"{scores.get('coherence', 0):.2f}",
-        "Accuracy": f"{scores.get('accuracy', 0):.2f}"
-    }
+    scores_display = {k: f"{v:.2f}" for k, v in scores.items() if isinstance(v, float)}
+    scores_display["eval_id"] = eval_id
     
     return scores_display, spider_chart, score_bars, explanation
 
 def process_batch_evaluation(
     file_input,
     evaluation_mode: str = "comprehensive"
-) -> Tuple[str, str, str, str, pd.DataFrame]:
+) -> Tuple[go.Figure, go.Figure, go.Figure, str, pd.DataFrame]:
     """Process batch evaluation from uploaded file"""
-    
     if file_input is None:
-        return "", "", "", "", pd.DataFrame()
-    
+        return go.Figure(), go.Figure(), go.Figure(), "Please upload a file.", pd.DataFrame()
+
     try:
-        # Parse the uploaded file
         if file_input.name.endswith('.json'):
             with open(file_input.name, 'r') as f:
                 data = json.load(f)
@@ -104,55 +91,45 @@ def process_batch_evaluation(
                 for line in f:
                     data.append(json.loads(line))
         else:
-            return "", "", "", "Unsupported file format. Please upload JSON or JSONL", pd.DataFrame()
-        
+            raise ValueError("Unsupported file format. Please upload JSON or JSONL.")
+
         # Process batch evaluation
         results = evaluator.evaluate_batch(data, mode=evaluation_mode)
         
+        # Store all results for explainability
+        for res in results:
+            eval_id = res.get('scores', {}).get('eval_id')
+            if eval_id:
+                evaluation_storage[eval_id] = res
+
         # Generate visualizations
         heatmap = visualizer.create_evaluation_heatmap(results)
         distribution = visualizer.create_score_distribution(results)
         trends = visualizer.create_performance_trends(results)
-        
-        # Generate comprehensive report
         report = report_gen.generate_batch_report(results)
-        
-        # Create leaderboard
         leaderboard = create_leaderboard(results)
         
         return heatmap, distribution, trends, report, leaderboard
         
     except Exception as e:
         error_msg = f"Error processing batch file: {str(e)}"
-        return "", "", "", error_msg, pd.DataFrame()
+        return go.Figure(), go.Figure(), go.Figure(), error_msg, pd.DataFrame()
 
 def create_leaderboard(results: List[Dict]) -> pd.DataFrame:
     """Create a leaderboard from evaluation results"""
-    
-    agent_scores = defaultdict(list)
-    
-    for result in results:
-        agent_name = result.get('agent_name', 'Unknown')
-        overall_score = result.get('scores', {}).get('overall_score', 0)
-        agent_scores[agent_name].append(overall_score)
+    agent_scores = evaluator.get_agent_scores_from_results(results)
     
     leaderboard_data = []
     for agent, scores in agent_scores.items():
         leaderboard_data.append({
-            'Rank': 0,  # Will be assigned after sorting
-            'Agent': agent,
-            'Avg Score': np.mean(scores),
-            'Max Score': np.max(scores),
-            'Min Score': np.min(scores),
-            'Std Dev': np.std(scores),
-            'Evaluations': len(scores)
+            'Rank': 0, 'Agent': agent, 'Avg Score': np.mean(scores),
+            'Max Score': np.max(scores), 'Min Score': np.min(scores),
+            'Std Dev': np.std(scores), 'Evaluations': len(scores)
         })
     
-    df = pd.DataFrame(leaderboard_data)
-    df = df.sort_values('Avg Score', ascending=False)
+    df = pd.DataFrame(leaderboard_data).sort_values('Avg Score', ascending=False)
     df['Rank'] = range(1, len(df) + 1)
     
-    # Format scores
     for col in ['Avg Score', 'Max Score', 'Min Score', 'Std Dev']:
         df[col] = df[col].apply(lambda x: f"{x:.3f}")
     
@@ -161,366 +138,136 @@ def create_leaderboard(results: List[Dict]) -> pd.DataFrame:
 def compare_agents(
     agent1_file,
     agent2_file,
-    comparison_metric: str = "all"
-) -> Tuple[str, str, str, str]:
+) -> Tuple[go.Figure, go.Figure, go.Figure, str]:
     """Compare two agents' performance"""
-    
     if not agent1_file or not agent2_file:
-        return "", "", "", "Please upload files for both agents"
-    
+        return go.Figure(), go.Figure(), go.Figure(), "Please upload files for both agents."
+
     try:
-        # Load agent data
-        agent1_data = load_agent_data(agent1_file)
-        agent2_data = load_agent_data(agent2_file)
-        
-        # Evaluate both agents
-        agent1_results = evaluator.evaluate_batch(agent1_data)
-        agent2_results = evaluator.evaluate_batch(agent2_data)
-        
-        # Generate comparison visualizations
-        comparison_chart = visualizer.create_agent_comparison(
-            agent1_results, 
-            agent2_results,
-            agent1_name="Agent 1",
-            agent2_name="Agent 2"
-        )
-        
-        radar_comparison = visualizer.create_radar_comparison(
-            agent1_results,
-            agent2_results
-        )
-        
-        performance_delta = visualizer.create_performance_delta(
-            agent1_results,
-            agent2_results
-        )
+        def load_agent_data(file):
+            if file.name.endswith('.json'):
+                with open(file.name, 'r') as f: return json.load(f)
+            elif file.name.endswith('.jsonl'):
+                data = [];
+                with open(file.name, 'r') as f: 
+                    for line in f: data.append(json.loads(line))
+                return data
+            raise ValueError("Unsupported file format")
+
+        agent1_results = evaluator.evaluate_batch(load_agent_data(agent1_file))
+        agent2_results = evaluator.evaluate_batch(load_agent_data(agent2_file))
         
-        # Generate comparison report
-        comparison_report = report_gen.generate_comparison_report(
-            agent1_results,
-            agent2_results
-        )
+        comparison_chart = visualizer.create_agent_comparison(agent1_results, agent2_results)
+        radar_comparison = visualizer.create_radar_comparison(agent1_results, agent2_results)
+        performance_delta = visualizer.create_performance_delta(agent1_results, agent2_results)
+        comparison_report = report_gen.generate_comparison_report(agent1_results, agent2_results)
         
         return comparison_chart, radar_comparison, performance_delta, comparison_report
         
     except Exception as e:
         error_msg = f"Error comparing agents: {str(e)}"
-        return "", "", "", error_msg
+        return go.Figure(), go.Figure(), go.Figure(), error_msg
 
-def load_agent_data(file_input):
-    """Load agent data from file"""
-    if file_input.name.endswith('.json'):
-        with open(file_input.name, 'r') as f:
-            return json.load(f)
-    elif file_input.name.endswith('.jsonl'):
-        data = []
-        with open(file_input.name, 'r') as f:
-            for line in f:
-                data.append(json.loads(line))
-        return data
-    else:
-        raise ValueError("Unsupported file format")
+def get_detailed_explanation(eval_id: str) -> Tuple[Dict, str]:
+    """Retrieve and format the detailed explanation for an evaluation ID."""
+    if not eval_id:
+        return {}, "Please enter an Evaluation ID."
+        
+    eval_result = evaluation_storage.get(eval_id)
 
-def export_results(results_data: Dict, export_format: str = "json") -> str:
-    """Export evaluation results in various formats"""
-    
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    
-    if export_format == "json":
-        filename = f"aetherscore_results_{timestamp}.json"
-        with open(filename, 'w') as f:
-            json.dump(results_data, f, indent=2)
-    elif export_format == "csv":
-        filename = f"aetherscore_results_{timestamp}.csv"
-        df = pd.DataFrame(results_data)
-        df.to_csv(filename, index=False)
-    else:
-        filename = f"aetherscore_results_{timestamp}.html"
-        # Generate HTML report
-        html_content = report_gen.generate_html_report(results_data)
-        with open(filename, 'w') as f:
-            f.write(html_content)
-    
-    return filename
+    if not eval_result:
+        return {}, f"Error: Evaluation ID '{eval_id}' not found in memory."
 
-# Create Gradio Interface
+    scores = eval_result.get("scores", {})
+    reasons = eval_result.get("reasons", {})
+
+    # Format details for JSON display
+    details_to_display = {
+        "evaluation_id": scores.get("eval_id"),
+        "timestamp": scores.get("timestamp"),
+        "task_type": scores.get("task_type"),
+        "scores": {k: v for k, v in scores.items() if k not in ["eval_id", "timestamp", "task_type"]}
+    }
+
+    # Format breakdown for text display
+    breakdown = []
+    breakdown.append(f"### Detailed Breakdown for Evaluation: {eval_id}\n")
+    for metric, reason in reasons.items():
+        metric_name = metric.replace('_', ' ').title()
+        score = scores.get(metric, 0)
+        breakdown.append(f"**{metric_name}**: {score:.2f}\n"
+                         f"> *Reasoning*: {reason}\n")
+    
+    return details_to_display, "\n".join(breakdown)
+
+# --- Gradio Interface ---
 with gr.Blocks(title="AetherScore - AI Agent Evaluation Framework", css=custom_css) as demo:
-    
-    gr.Markdown("""
-    # 🌟 AetherScore - Advanced AI Agent Evaluation Framework
-    
-    **Comprehensive evaluation system for AI agents with multi-dimensional scoring, explainability, and advanced visualizations**
-    
-    ---
-    """)
+    gr.Markdown("# 🌟 AetherScore - Advanced AI Agent Evaluation Framework")
     
     with gr.Tabs():
         # Tab 1: Single Evaluation
         with gr.TabItem("🎯 Single Evaluation"):
-            gr.Markdown("### Evaluate a single agent response")
-            
             with gr.Row():
                 with gr.Column(scale=1):
-                    prompt_input = gr.Textbox(
-                        label="Prompt",
-                        placeholder="Enter the prompt given to the agent...",
-                        lines=3
-                    )
-                    response_input = gr.Textbox(
-                        label="Agent Response",
-                        placeholder="Enter the agent's response...",
-                        lines=3
-                    )
-                    expected_answer = gr.Textbox(
-                        label="Expected Answer (Optional)",
-                        placeholder="Enter the expected/correct answer for accuracy scoring...",
-                        lines=2
-                    )
-                    
-                    with gr.Row():
-                        agent_name_input = gr.Textbox(
-                            label="Agent Name",
-                            value="Agent-1",
-                            scale=1
-                        )
-                        task_type = gr.Dropdown(
-                            label="Task Type",
-                            choices=["general", "QA", "summarization", "reasoning", "code"],
-                            value="general",
-                            scale=1
-                        )
-                    
+                    prompt_input = gr.Textbox(label="Prompt", lines=3)
+                    response_input = gr.Textbox(label="Agent Response", lines=3)
+                    expected_answer = gr.Textbox(label="Expected Answer (Optional)", lines=2)
+                    agent_name_input = gr.Textbox(label="Agent Name", value="Agent-1")
+                    task_type = gr.Dropdown(label="Task Type", choices=["general", "QA", "summarization", "reasoning", "code"], value="general")
                     evaluate_btn = gr.Button("🔍 Evaluate", variant="primary")
-                
                 with gr.Column(scale=1):
                     scores_output = gr.JSON(label="📊 Evaluation Scores")
-                    explanation_output = gr.Textbox(
-                        label="📝 Explanation",
-                        lines=6,
-                        interactive=False
-                    )
-            
+                    explanation_output = gr.Textbox(label="📝 Explanation", lines=6, interactive=False)
             with gr.Row():
-                spider_chart_output = gr.HTML(label="🕸️ Spider Chart")
-                score_bars_output = gr.HTML(label="📊 Score Breakdown")
+                spider_chart_output = gr.Plot(label="🕸️ Spider Chart")
+                score_bars_output = gr.Plot(label="📊 Score Breakdown")
             
-            evaluate_btn.click(
-                fn=process_single_evaluation,
-                inputs=[prompt_input, response_input, expected_answer, agent_name_input, task_type],
-                outputs=[scores_output, spider_chart_output, score_bars_output, explanation_output]
-            )
-        
+            evaluate_btn.click(fn=process_single_evaluation, inputs=[prompt_input, response_input, expected_answer, agent_name_input, task_type], outputs=[scores_output, spider_chart_output, score_bars_output, explanation_output])
+
         # Tab 2: Batch Evaluation
         with gr.TabItem("📦 Batch Evaluation"):
-            gr.Markdown("### Evaluate multiple agents and responses at scale")
-            
             with gr.Row():
                 with gr.Column(scale=1):
-                    batch_file = gr.File(
-                        label="Upload Evaluation File (JSON/JSONL)",
-                        file_types=[".json", ".jsonl"]
-                    )
-                    evaluation_mode = gr.Radio(
-                        label="Evaluation Mode",
-                        choices=["comprehensive", "fast", "detailed"],
-                        value="comprehensive"
-                    )
+                    batch_file = gr.File(label="Upload Evaluation File (JSON/JSONL)", file_types=[".json", ".jsonl"])
+                    evaluation_mode = gr.Radio(label="Evaluation Mode", choices=["comprehensive", "fast", "detailed"], value="comprehensive")
                     batch_evaluate_btn = gr.Button("🚀 Run Batch Evaluation", variant="primary")
-                    
-                    gr.Markdown("""
-                    **Expected Format:**
-                    ```json
-                    {
-                        "task_id": "task_001",
-                        "agent_name": "GPT-4",
-                        "prompt": "...",
-                        "response": "...",
-                        "expected_answer": "..." (optional)
-                    }
-                    ```
-                    """)
-                
                 with gr.Column(scale=2):
-                    batch_report = gr.Textbox(
-                        label="📄 Evaluation Report",
-                        lines=10,
-                        interactive=False
-                    )
-            
+                    batch_report = gr.Textbox(label="📄 Evaluation Report", lines=10, interactive=False)
             with gr.Row():
-                heatmap_output = gr.HTML(label="🗺️ Evaluation Heatmap")
-                distribution_output = gr.HTML(label="📈 Score Distribution")
+                heatmap_output = gr.Plot(label="🗺️ Evaluation Heatmap")
+                distribution_output = gr.Plot(label="📈 Score Distribution")
+            trends_output = gr.Plot(label="📊 Performance Trends")
+            leaderboard_output = gr.Dataframe(label="🏆 Agent Leaderboard", interactive=False)
             
-            trends_output = gr.HTML(label="📊 Performance Trends")
-            leaderboard_output = gr.Dataframe(
-                label="🏆 Agent Leaderboard",
-                interactive=False
-            )
-            
-            batch_evaluate_btn.click(
-                fn=process_batch_evaluation,
-                inputs=[batch_file, evaluation_mode],
-                outputs=[heatmap_output, distribution_output, trends_output, batch_report, leaderboard_output]
-            )
-        
+            batch_evaluate_btn.click(fn=process_batch_evaluation, inputs=[batch_file, evaluation_mode], outputs=[heatmap_output, distribution_output, trends_output, batch_report, leaderboard_output])
+
         # Tab 3: Agent Comparison
         with gr.TabItem("⚔️ Agent Comparison"):
-            gr.Markdown("### Compare performance between two agents")
-            
             with gr.Row():
                 with gr.Column():
-                    agent1_file = gr.File(
-                        label="Agent 1 Results (JSON/JSONL)",
-                        file_types=[".json", ".jsonl"]
-                    )
-                    agent2_file = gr.File(
-                        label="Agent 2 Results (JSON/JSONL)",
-                        file_types=[".json", ".jsonl"]
-                    )
-                    comparison_metric = gr.Dropdown(
-                        label="Comparison Focus",
-                        choices=["all", "accuracy", "hallucination", "instruction_following"],
-                        value="all"
-                    )
+                    agent1_file = gr.File(label="Agent 1 Results (JSON/JSONL)", file_types=[".json", ".jsonl"])
+                    agent2_file = gr.File(label="Agent 2 Results (JSON/JSONL)", file_types=[".json", ".jsonl"])
                     compare_btn = gr.Button("⚖️ Compare Agents", variant="primary")
-                
                 with gr.Column():
-                    comparison_report = gr.Textbox(
-                        label="📊 Comparison Analysis",
-                        lines=8,
-                        interactive=False
-                    )
-            
+                    comparison_report = gr.Textbox(label="📊 Comparison Analysis", lines=8, interactive=False)
             with gr.Row():
-                comparison_chart = gr.HTML(label="📊 Performance Comparison")
-                radar_comparison = gr.HTML(label="🎯 Radar Comparison")
+                comparison_chart = gr.Plot(label="📊 Performance Comparison")
+                radar_comparison = gr.Plot(label="🎯 Radar Comparison")
+            performance_delta = gr.Plot(label="📈 Performance Delta Analysis")
             
-            performance_delta = gr.HTML(label="📈 Performance Delta Analysis")
-            
-            compare_btn.click(
-                fn=compare_agents,
-                inputs=[agent1_file, agent2_file, comparison_metric],
-                outputs=[comparison_chart, radar_comparison, performance_delta, comparison_report]
-            )
-        
+            compare_btn.click(fn=compare_agents, inputs=[agent1_file, agent2_file], outputs=[comparison_chart, radar_comparison, performance_delta, comparison_report])
+
         # Tab 4: Explainability Dashboard
         with gr.TabItem("🔍 Explainability"):
-            gr.Markdown("### Deep dive into evaluation decisions")
-            
+            gr.Markdown("### Deep dive into evaluation decisions. Enter an `eval_id` from a previous run.")
             with gr.Row():
-                with gr.Column():
-                    eval_id_input = gr.Textbox(
-                        label="Evaluation ID or Task ID",
-                        placeholder="Enter evaluation ID to analyze..."
-                    )
-                    load_eval_btn = gr.Button("Load Evaluation", variant="secondary")
-                
-                with gr.Column():
-                    eval_details = gr.JSON(label="Evaluation Details")
-            
+                eval_id_input = gr.Textbox(label="Evaluation ID", placeholder="Enter evaluation ID to analyze...")
+                load_eval_btn = gr.Button("Load Evaluation", variant="secondary")
             with gr.Row():
-                decision_tree = gr.HTML(label="Decision Tree")
-                feature_importance = gr.HTML(label="Feature Importance")
-            
-            detailed_breakdown = gr.Textbox(
-                label="Detailed Score Breakdown",
-                lines=10,
-                interactive=False
-            )
-        
-        # Tab 5: Settings & Export
-        with gr.TabItem("⚙️ Settings & Export"):
-            gr.Markdown("### Configuration and Export Options")
-            
-            with gr.Row():
-                with gr.Column():
-                    gr.Markdown("#### Evaluation Settings")
-                    
-                    hallucination_threshold = gr.Slider(
-                        label="Hallucination Detection Threshold",
-                        minimum=0.0,
-                        maximum=1.0,
-                        value=0.7,
-                        step=0.1
-                    )
-                    
-                    coherence_weight = gr.Slider(
-                        label="Coherence Weight",
-                        minimum=0.0,
-                        maximum=1.0,
-                        value=0.25,
-                        step=0.05
-                    )
-                    
-                    accuracy_weight = gr.Slider(
-                        label="Accuracy Weight",
-                        minimum=0.0,
-                        maximum=1.0,
-                        value=0.25,
-                        step=0.05
-                    )
-                    
-                    save_settings_btn = gr.Button("💾 Save Settings", variant="secondary")
-                
-                with gr.Column():
-                    gr.Markdown("#### Export Options")
-                    
-                    export_format = gr.Radio(
-                        label="Export Format",
-                        choices=["JSON", "CSV", "HTML Report"],
-                        value="JSON"
-                    )
-                    
-                    include_visualizations = gr.Checkbox(
-                        label="Include Visualizations in Export",
-                        value=True
-                    )
-                    
-                    export_btn = gr.Button("📥 Export Results", variant="primary")
-                    export_status = gr.Textbox(
-                        label="Export Status",
-                        interactive=False
-                    )
-    
-    gr.Markdown("""
-    ---
-    
-    ### 📚 About AetherScore
-    
-    AetherScore is a comprehensive evaluation framework designed to assess AI agents across multiple dimensions:
-    
-    - **Instruction Following**: How well the agent adheres to given instructions
-    - **Hallucination Detection**: Identifies fabricated or unsupported information
-    - **Assumption Control**: Measures unwarranted assumptions made by the agent
-    - **Coherence**: Evaluates logical consistency and flow
-    - **Accuracy**: Compares responses against ground truth when available
-    
-    ### 🎯 Key Features
-    
-    - **Multi-dimensional Scoring**: Comprehensive evaluation across 5+ metrics
-    - **Batch Processing**: Evaluate thousands of responses efficiently
-    - **Advanced Visualizations**: Spider charts, heatmaps, trend analysis
-    - **Explainable AI**: Understand why scores were assigned
-    - **Agent Comparison**: Head-to-head performance analysis
-    - **Customizable Weights**: Adjust importance of different metrics
-    
-    ### 🔧 API Integration
-    
-    ```python
-    # Example API usage
-    from aetherscore import evaluate
-    
-    result = evaluate(
-        prompt="Your prompt here",
-        response="Agent response",
-        metrics=["all"]
-    )
-    ```
-    
-    ---
-    
-    **Version**: 1.0.0 | **License**: MIT | **GitHub**: [AetherScore](https://github.com/aetherscore)
-    """)
+                eval_details = gr.JSON(label="Evaluation Details")
+                detailed_breakdown = gr.Markdown(label="Detailed Score Breakdown")
+
+            load_eval_btn.click(fn=get_detailed_explanation, inputs=[eval_id_input], outputs=[eval_details, detailed_breakdown])
 
 if __name__ == "__main__":
     demo.launch(share=True)