Update evaluator_module.py (#35)

- Update evaluator_module.py (ad320ec347545b90f7eba3ccdb98ae61a8b6dde6)


Co-authored-by: Manay Porwal <manayporwal07@users.noreply.huggingface.co>

evaluator_module.py

@@ -11,11 +11,6 @@ import hashlib
 from datetime import datetime
 import concurrent.futures
 import random
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
-import evaluate
-import warnings
-warnings.filterwarnings("ignore")
-
 
 class AetherScoreEvaluator:
     def __init__(self):
@@ -28,160 +23,64 @@ class AetherScoreEvaluator:
             self.nlp = spacy.load("en_core_web_sm")
 
         # LLM Judge Model
-        # self.judge_model = pipeline(
-        #     "text2text-generation", 
-        #     model="google/flan-t5-base", 
-        #     device=-1  # 0 for GPU 
-        # )
-        # Load evaluation metrics
-        self.rouge = evaluate.load('rouge')
-        self.sacrebleu = evaluate.load('sacrebleu')
-        self.bertscore = evaluate.load('bertscore')
-
-        # Load a model to generate factual reference answers
-        model_name = "google/flan-t5-base"  # small & fast
-        self.judge_tokenizer = AutoTokenizer.from_pretrained(model_name)
-        self.judge_model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-
+        self.judge_model = pipeline(
+            "text2text-generation", 
+            model="google/flan-t5-base", 
+            device=-1  # 0 for GPU 
+        )
 
         # Sentence Transformer for Sentence ----> Embedding
         self.sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
         
         # Scoring weights # Domain Specific weights can be added for better results
-        self.weights = {
-            'instruction_following': 0.20,
-            'hallucination_score': 0.15,
-            'assumption_control': 0.15,
-            'coherence': 0.15,
-            'accuracy': 0.15,
-            'llm_judge_score': 0.20
-        }        
+        self.weights = {'instruction_following': 0.25, 'hallucination_score': 0.20,
+                        'assumption_control': 0.20, 'coherence': 0.20, 'accuracy': 0.15}
+        
         # In-memory cache
         self.cache = {}
 
     # LLM Judge
     def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
+        
         print("Using HF model LLM Judge...")
     
         query = (
             f"Prompt: {prompt}\n"
             f"Response: {response}\n\n"
-            "Return only a JSON object:\n"
+            "Return ONLY a valid JSON object in this exact format:\n"
             "{\n"
-            '  "score": <float between 0 and 1>,\n'
-            '  "explanation": "<short reasoning>"\n'
-            "}\n"
+            "  \"hallucination_score\": <float between 0 and 1>,\n"
+            "  \"assumption_control\": <float between 0 and 1>,\n"
+            "  \"explanation\": \"<one or two sentences>\"\n"
+            "}"
         )
     
         try:
             result = self.judge_model(query, max_new_tokens=128, truncation=True)
             output = result[0]['generated_text']
         except Exception as e:
-            return {"llm_judge_score": (0.0, f"HF model failed: {e}")}
+            return {
+                "hallucination_score": (0.1, f"HF model failed: {e}"),
+                "assumption_control": (0.1, f"HF model failed: {e}")
+            }
     
-        # Defaults
-        score = 0.5
-        explanation = output.strip()
+        # Default values
+        halluc_score, assumption_score, explanation = 0.5, 0.5, output
     
         try:
             parsed = json.loads(output.strip())
-            score = float(parsed.get("score", score))
+            halluc_score = float(parsed.get("hallucination_score", halluc_score))
+            assumption_score = float(parsed.get("assumption_control", assumption_score))
             explanation = parsed.get("explanation", explanation)
         except Exception:
             pass  # fallback to defaults
     
-        # Hardcoded interpretation
-        if score < 0.3:
-            auto_expl = "Poor alignment with the prompt."
-        elif score < 0.7:
-            auto_expl = "Partially aligned with the prompt."
-        else:
-            auto_expl = "Well aligned with the prompt."
-    
-        explanation = f"{explanation} | {auto_expl}"
-    
-        return {"llm_judge_score": (score, explanation)}
-
-
-
-    
-    # LLM Judge
-    def _generate_reference(self, prompt: str) -> str:
-        """Generate a factual reference answer from the prompt."""
-        input_text = f"Provide a factual answer: {prompt}"
-        input_ids = self.judge_tokenizer(input_text, return_tensors="pt").input_ids
-        outputs = self.judge_model.generate(
-            input_ids,
-            max_length=256,
-            num_beams=4,
-            early_stopping=True
-        )
-        return self.judge_tokenizer.decode(outputs[0], skip_special_tokens=True)
-    
-    def detect(self, prompt: str, output: str) -> dict:
-        """Compute hallucination-related scores."""
-        reference = self._generate_reference(prompt)
-    
-        # Metric 1: ROUGE-L
-        rouge_l = self.rouge.compute(predictions=[output], references=[reference])['rougeL']
-    
-        # Metric 2: SacreBLEU (normalize 0–1)
-        sacrebleu = self.sacrebleu.compute(predictions=[output], references=[[reference]])['score'] / 100
-    
-        # Metric 3: BERTScore F1
-        bert_f1 = np.mean(
-            self.bertscore.compute(predictions=[output], references=[reference], lang='en')['f1']
-        )
-    
-        # Combine (weights adjusted to sum = 1)
-        weights = {
-            "rouge_l": 0.33,
-            "sacrebleu": 0.33,
-            "bertscore_f1": 0.34,
-        }
-        halluc_score = (
-            weights["rouge_l"] * (1 - rouge_l)
-            + weights["sacrebleu"] * (1 - sacrebleu)
-            + weights["bertscore_f1"] * (1 - bert_f1)
-        )
-    
-        return {
-            "prompt": prompt,
-            "output": output,
-            "reference": reference,
-            "scores": {
-                "rougeL": rouge_l,
-                "sacrebleu": sacrebleu,
-                "bertscore": bert_f1,
-            },
-            "hallucination_score": float(halluc_score),
-        }
-    
-    def hallucination_score(self,prompt: str, output: str) -> dict:
-        """
-        Keeps the SAME return format as _evaluate_with_llm_judge:
-        {
-            "hallucination_score": (float, explanation),
-            "assumption_control": (float, explanation)
-        }
-        """
-        result=self.detect(prompt,output)
-        
-        explanation = (
-            f"Compared output with generated reference.\n"
-            f"ROUGE={result['scores']['rougeL']:.2f}, "
-            f"BLEU={result['scores']['sacrebleu']:.2f}, "
-            f"BERTScore={result['scores']['bertscore']:.2f}."
-        )
-    
-        halluc_score = result["hallucination_score"]
-        assumption_score = 1 - halluc_score  # simple proxy
-    
         return {
             "hallucination_score": (halluc_score, explanation),
-            "assumption_control": (assumption_score, "Inverse of hallucination score (simplified)."),
+            "assumption_control": (assumption_score, explanation)
         }
 
+
     # Single Evaluation # Inputs-->> Prompt, Agent Response, Expected Answer(Optional), Agent Name and Task type( General, QA, Summarizaton)etc 
     def evaluate_single(self, prompt: str, response: str, expected_answer: Optional[str] = None, task_type: str = "general") -> Dict:
         
@@ -194,15 +93,11 @@ class AetherScoreEvaluator:
 
         scores, reasons = {}, {}
 
-        # hallucination and assumption controle
-        halluc_results = self.hallucination_score(prompt, response)
-        scores['hallucination_score'], reasons['hallucination_score'] = halluc_results['hallucination_score']
-        scores['assumption_control'], reasons['assumption_control'] = halluc_results['assumption_control']
-        
-        # llm judge score
+        # Taking Back scores and reasons of hallucination and assumption control from LLM Judge
         llm_judge_results = self._evaluate_with_llm_judge(prompt, response)
-        scores['llm_judge_score'], reasons['llm_judge_score'] = llm_judge_results['llm_judge_score']
-        
+        scores['hallucination_score'], reasons['hallucination_score'] = llm_judge_results['hallucination_score']
+        scores['assumption_control'], reasons['assumption_control'] = llm_judge_results['assumption_control']
+
         # Evaluating Instruction Following, Coherence and Accuracy
         scores['instruction_following'], reasons['instruction_following'] = self._evaluate_instruction_following(prompt, response)
         scores['coherence'], reasons['coherence'] = self._evaluate_coherence(response)