Update evaluator_module.py (#39)

- Update evaluator_module.py (9d0d6c25aabcf7e27201675fcaff554759bcb474)


Co-authored-by: Manay Porwal <manayporwal07@users.noreply.huggingface.co>

evaluator_module.py

@@ -40,44 +40,79 @@ class AetherScoreEvaluator:
         self.cache = {}
 
     # LLM Judge
-    def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
+    # def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
         
-        print("Using HF model LLM Judge...")
+    #     print("Using HF model LLM Judge...")
     
-        query = (
-            f"Prompt: {prompt}\n"
-            f"Response: {response}\n\n"
-            "Return ONLY a valid JSON object in this exact format:\n"
-            "{\n"
-            "  \"hallucination_score\": <float between 0 and 1>,\n"
-            "  \"assumption_control\": <float between 0 and 1>,\n"
-            "  \"explanation\": \"<one or two sentences>\"\n"
-            "}"
-        )
+    #     query = (
+    #         f"Prompt: {prompt}\n"
+    #         f"Response: {response}\n\n"
+    #         "Return ONLY a valid JSON object in this exact format:\n"
+    #         "{\n"
+    #         "  \"hallucination_score\": <float between 0 and 1>,\n"
+    #         "  \"assumption_control\": <float between 0 and 1>,\n"
+    #         "  \"explanation\": \"<one or two sentences>\"\n"
+    #         "}"
+    #     )
     
-        try:
-            result = self.judge_model(query, max_new_tokens=128, truncation=True)
-            output = result[0]['generated_text']
-        except Exception as e:
-            return {
-                "hallucination_score": (0.1, f"HF model failed: {e}"),
-                "assumption_control": (0.1, f"HF model failed: {e}")
-            }
+    #     try:
+    #         result = self.judge_model(query, max_new_tokens=128, truncation=True)
+    #         output = result[0]['generated_text']
+    #     except Exception as e:
+    #         return {
+    #             "hallucination_score": (0.1, f"HF model failed: {e}"),
+    #             "assumption_control": (0.1, f"HF model failed: {e}")
+    #         }
     
-        # Default values
+    #     # Default values
         
-        halluc_score = random.uniform(0.3, 0.7)
-        assumption_score = random.uniform(0.3, 0.7)
-        explanation = output
+    #     halluc_score = random.uniform(0.3, 0.7)
+    #     assumption_score = random.uniform(0.3, 0.7)
+    #     explanation = output
     
-        try:
-            parsed = json.loads(output.strip())
-            halluc_score = float(parsed.get("hallucination_score", halluc_score))
-            assumption_score = float(parsed.get("assumption_control", assumption_score))
-            explanation = parsed.get("explanation", explanation)
-        except Exception:
-            pass  # fallback to defaults
+    #     try:
+    #         parsed = json.loads(output.strip())
+    #         halluc_score = float(parsed.get("hallucination_score", halluc_score))
+    #         assumption_score = float(parsed.get("assumption_control", assumption_score))
+    #         explanation = parsed.get("explanation", explanation)
+    #     except Exception:
+    #         pass  # fallback to defaults
     
+    #     return {
+    #         "hallucination_score": (halluc_score, explanation),
+    #         "assumption_control": (assumption_score, explanation)
+    #     }
+
+    import random
+    import json
+    
+    def _evaluate_with_llm_judge(self, prompt: str, response: str) -> Dict:
+        
+        print("Using rule-based evaluation instead of HF LLM...")
+
+        prompt_words = set(prompt.lower().split())
+        response_words = response.lower().split()
+        
+        # Hallucination score: fraction of words in response that are not in prompt
+        if response_words:
+            halluc_score = len([w for w in response_words if w not in prompt_words]) / len(response_words)
+        else:
+            halluc_score = 0.1
+        
+        # Assumption control: fraction of sentences starting with uncertain words
+        uncertain_starts = ("i assume", "maybe", "probably", "likely", "could be")
+        sentences = response.lower().split(".")
+        if sentences:
+            assumption_score = sum(1 for s in sentences if s.strip().startswith(uncertain_starts)) / len(sentences)
+        else:
+            assumption_score = 0.1
+        
+        # Ensure scores are between 0 and 1
+        # halluc_score = max(0, min(1, halluc_score))
+        # assumption_score = max(0, min(1, assumption_score))
+        
+        explanation = "Rule-based evaluation applied."
+        
         return {
             "hallucination_score": (halluc_score, explanation),
             "assumption_control": (assumption_score, explanation)