Upload 4 files

app.py

@@ -0,0 +1,34 @@
+import gradio as gr
+import pandas as pd
+from evaluator import evaluate_responses
+from synthetic_data import generate_synthetic_dataset
+
+# Demo synthetic dataset
+df = generate_synthetic_dataset(num_agents=10, num_samples=50)
+
+def run_evaluation(use_llm_judge=False):
+    results = evaluate_responses(df, use_llm_judge=use_llm_judge)
+    leaderboard = results.groupby("agent")["final_score"].mean().reset_index()
+    leaderboard = leaderboard.sort_values("final_score", ascending=False)
+    return results, leaderboard
+
+with gr.Blocks(title="Agentic Evaluation Framework") as demo:
+    gr.Markdown("# 🤖 Agentic Evaluation Framework")
+    gr.Markdown("Automatically evaluate AI agents across multiple dimensions.")
+
+    with gr.Tab("Synthetic Data Preview"):
+        gr.DataFrame(df, label="Generated Dataset", interactive=False)
+
+    with gr.Tab("Run Evaluation"):
+        use_llm = gr.Checkbox(label="Use LLM Judge (Optional)", value=False)
+        run_button = gr.Button("Run Evaluation")
+        results_output = gr.DataFrame(label="Evaluation Results")
+        leaderboard_output = gr.DataFrame(label="Leaderboard")
+
+        run_button.click(
+            fn=run_evaluation,
+            inputs=[use_llm],
+            outputs=[results_output, leaderboard_output]
+        )
+
+demo.launch()

evaluator.py

@@ -0,0 +1,37 @@
+import pandas as pd
+import textstat
+import language_tool_python
+
+tool = language_tool_python.LanguageTool('en-US')
+
+def evaluate_responses(df, use_llm_judge=False):
+    scores = []
+    for _, row in df.iterrows():
+        response = row["response"]
+
+        # Rule-based metrics
+        grammar_matches = len(tool.check(response))
+        readability = textstat.flesch_reading_ease(response)
+
+        # Simple scoring
+        instruction_follow = 1 if row["instruction"].lower() in response.lower() else 0
+        coherence = 1 if readability > 40 else 0
+        grammar_score = max(0, 1 - grammar_matches/10)
+
+        final_score = (instruction_follow + coherence + grammar_score) / 3
+
+        # Optional LLM judge (stubbed for now, can hook Hugging Face API later)
+        if use_llm_judge:
+            final_score = (final_score + 0.8) / 2  # Example: trust LLM judge
+
+        scores.append({
+            "agent": row["agent"],
+            "instruction": row["instruction"],
+            "response": response,
+            "score_instruction": instruction_follow,
+            "score_coherence": coherence,
+            "score_grammar": grammar_score,
+            "final_score": final_score
+        })
+
+    return pd.DataFrame(scores)

requirements.txt

@@ -0,0 +1,6 @@
+gradio==4.44.0
+pandas
+numpy
+scikit-learn
+language-tool-python
+textstat

synthetic_data.py

@@ -0,0 +1,11 @@
+import pandas as pd
+import random
+
+def generate_synthetic_dataset(num_agents=5, num_samples=20):
+    data = []
+    for agent in range(num_agents):
+        for i in range(num_samples):
+            instr = f"Task {i}: Summarize topic {random.randint(1,10)}"
+            resp = f"This is a response to {instr}. Agent {agent} provides details."
+            data.append({"agent": f"Agent_{agent}", "instruction": instr, "response": resp})
+    return pd.DataFrame(data)