Update evaluator_module.py (#17)

- Update evaluator_module.py (bbadeb4cf08318a2bc50d34a03f55f39aadde90e)


Co-authored-by: Manay Porwal <manayporwal07@users.noreply.huggingface.co>

evaluator_module.py

@@ -13,7 +13,6 @@ import concurrent.futures
 import random
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 import evaluate
-from unieval.metric.evaluator import get_evaluator
 import warnings
 warnings.filterwarnings("ignore")
 
@@ -50,52 +49,60 @@ class AetherScoreEvaluator:
         self.sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
         
         # Scoring weights # Domain Specific weights can be added for better results
-        self.weights = {'instruction_following': 0.25, 'hallucination_score': 0.20,
-                        'assumption_control': 0.20, 'coherence': 0.20, 'accuracy': 0.15}
-        
+        self.weights = {
+            'instruction_following': 0.20,
+            'hallucination_score': 0.15,
+            'assumption_control': 0.15,
+            'coherence': 0.15,
+            'accuracy': 0.15,
+            'llm_judge_score': 0.20
+        }        
         # In-memory cache
         self.cache = {}
 
     # LLM Judge
-    # def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
-        
-    #     print("Using HF model LLM Judge...")
+    def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
+        print("Using HF model LLM Judge...")
+    
+        query = (
+            f"Prompt: {prompt}\n"
+            f"Response: {response}\n\n"
+            "Return only a JSON object:\n"
+            "{\n"
+            '  "score": <float between 0 and 1>,\n'
+            '  "explanation": "<short reasoning>"\n'
+            "}\n"
+        )
     
-    #     query = (
-    #         f"Prompt: {prompt}\n"
-    #         f"Response: {response}\n\n"
-    #         "Return ONLY a valid JSON object in this exact format:\n"
-    #         "{\n"
-    #         "  \"hallucination_score\": <float between 0 and 1>,\n"
-    #         "  \"assumption_control\": <float between 0 and 1>,\n"
-    #         "  \"explanation\": \"<one or two sentences>\"\n"
-    #         "}"
-    #     )
+        try:
+            result = self.judge_model(query, max_new_tokens=128, truncation=True)
+            output = result[0]['generated_text']
+        except Exception as e:
+            return {"llm_judge_score": (0.0, f"HF model failed: {e}")}
     
-    #     try:
-    #         result = self.judge_model(query, max_new_tokens=128, truncation=True)
-    #         output = result[0]['generated_text']
-    #     except Exception as e:
-    #         return {
-    #             "hallucination_score": (0.1, f"HF model failed: {e}"),
-    #             "assumption_control": (0.1, f"HF model failed: {e}")
-    #         }
+        # Defaults
+        score = 0.5
+        explanation = output.strip()
     
-    #     # Default values
-    #     halluc_score, assumption_score, explanation = 0.5, 0.5, output
+        try:
+            parsed = json.loads(output.strip())
+            score = float(parsed.get("score", score))
+            explanation = parsed.get("explanation", explanation)
+        except Exception:
+            pass  # fallback to defaults
     
-    #     try:
-    #         parsed = json.loads(output.strip())
-    #         halluc_score = float(parsed.get("hallucination_score", halluc_score))
-    #         assumption_score = float(parsed.get("assumption_control", assumption_score))
-    #         explanation = parsed.get("explanation", explanation)
-    #     except Exception:
-    #         pass  # fallback to defaults
+        # Hardcoded interpretation
+        if score < 0.3:
+            auto_expl = "Poor alignment with the prompt."
+        elif score < 0.7:
+            auto_expl = "Partially aligned with the prompt."
+        else:
+            auto_expl = "Well aligned with the prompt."
     
-    #     return {
-    #         "hallucination_score": (halluc_score, explanation),
-    #         "assumption_control": (assumption_score, explanation)
-    #     }
+        explanation = f"{explanation} | {auto_expl}"
+    
+        return {"llm_judge_score": (score, explanation)}
+
 
 
     
@@ -193,11 +200,15 @@ class AetherScoreEvaluator:
 
         scores, reasons = {}, {}
 
-        # Taking Back scores and reasons of hallucination and assumption control from LLM Judge
+        # hallucination and assumption controle
         halluc_results = self.hallucination_score(prompt, response)
         scores['hallucination_score'], reasons['hallucination_score'] = halluc_results['hallucination_score']
         scores['assumption_control'], reasons['assumption_control'] = halluc_results['assumption_control']
-
+        
+        # llm judge score
+        llm_judge_results = self._evaluate_with_llm_judge(prompt, response)
+        scores['llm_judge_score'], reasons['llm_judge_score'] = llm_judge_results['llm_judge_score']
+        
         # Evaluating Instruction Following, Coherence and Accuracy
         scores['instruction_following'], reasons['instruction_following'] = self._evaluate_instruction_following(prompt, response)
         scores['coherence'], reasons['coherence'] = self._evaluate_coherence(response)