Update evaluator_module.py (#31)

- Update evaluator_module.py (617a05d32ef93621944bd34f378c84e6177f6afa)


Co-authored-by: Manay Porwal <manayporwal07@users.noreply.huggingface.co>

evaluator_module.py

@@ -239,71 +239,71 @@ class AetherScoreEvaluator:
         
         return result
     # Batch Evaluation # Input of JSON/CSV file
-    # def evaluate_batch(self, data: List[Dict], mode: str = "comprehensive") -> List[Dict]:
-    #     """Process a batch of evaluations in parallel."""
-        
-    #     results = []
-
-    #     # Get Item function
-    #     def process_item(item):
-    #         # Calling our Evalution function for Single prompt response pair
-    #         eval_result = self.evaluate_single(
-    #             prompt=item.get('prompt', ''),
-    #             response=item.get('response', ''),
-    #             expected_answer=item.get('expected_answer',''),
-    #             task_type=item.get('task_type', 'general'),
-    #             eval_id=f"eval_{idx}",     
-    #             agent_name=agent_name
-    #         )
-    #         # Combining with original metadata
-    #         eval_result.update({
-    #             'task_id': item.get('task_id', eval_result['scores']['eval_id']),
-    #             'agent_name': item.get('agent_name', 'Unknown'),
-    #         })
-    #         return eval_result
-        
-    #     with concurrent.futures.ThreadPoolExecutor() as executor:
-    #         future_to_item = {executor.submit(process_item, item): item for item in data}
-    #         for future in concurrent.futures.as_completed(future_to_item):
-    #             try:
-    #                 results.append(future.result())
-    #             except Exception as exc:
-    #                 print(f'An item generated an exception: {exc}')
-
-    #     return results
-
     def evaluate_batch(self, data: List[Dict], mode: str = "comprehensive") -> List[Dict]:
         """Process a batch of evaluations in parallel."""
         
         results = []
-    
-        def process_item(item, idx):
-            agent_name = item.get("agent_name", f"Agent-{idx}")
+
+        # Get Item function
+        def process_item(item):
+            # Calling our Evalution function for Single prompt response pair
             eval_result = self.evaluate_single(
-                prompt=item.get("prompt", ""),
-                response=item.get("response", ""),
-                expected_answer=item.get("expected_answer", ""),
-                task_type=item.get("task_type", "general"),
+                prompt=item.get('prompt', ''),
+                response=item.get('response', ''),
+                expected_answer=item.get('expected_answer',''),
+                task_type=item.get('task_type', 'general'),
+                eval_id=f"eval_{idx}",     
+                agent_name=agent_name
             )
+            # Combining with original metadata
             eval_result.update({
-                "task_id": item.get("task_id", eval_result["scores"]["eval_id"]),
-                "agent_name": agent_name,
+                'task_id': item.get('task_id', eval_result['scores']['eval_id']),
+                'agent_name': item.get('agent_name', 'Unknown'),
             })
             return eval_result
-    
+        
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            future_to_idx = {
-                executor.submit(process_item, item, idx): idx
-                for idx, item in enumerate(data, start=1)
-            }
-            for future in concurrent.futures.as_completed(future_to_idx):
+            future_to_item = {executor.submit(process_item, item): item for item in data}
+            for future in concurrent.futures.as_completed(future_to_item):
                 try:
                     results.append(future.result())
                 except Exception as exc:
-                    print(f"An item generated an exception: {exc}")
-    
+                    print(f'An item generated an exception: {exc}')
+
         return results
 
+    # def evaluate_batch(self, data: List[Dict], mode: str = "comprehensive") -> List[Dict]:
+    #     """Process a batch of evaluations in parallel."""
+        
+    #     results = []
+    
+    #     def process_item(item, idx):
+    #         agent_name = item.get("agent_name", f"Agent-{idx}")
+    #         eval_result = self.evaluate_single(
+    #             prompt=item.get("prompt", ""),
+    #             response=item.get("response", ""),
+    #             expected_answer=item.get("expected_answer", ""),
+    #             task_type=item.get("task_type", "general"),
+    #         )
+    #         eval_result.update({
+    #             "task_id": item.get("task_id", eval_result["scores"]["eval_id"]),
+    #             "agent_name": agent_name,
+    #         })
+    #         return eval_result
+    
+    #     with concurrent.futures.ThreadPoolExecutor() as executor:
+    #         future_to_idx = {
+    #             executor.submit(process_item, item, idx): idx
+    #             for idx, item in enumerate(data, start=1)
+    #         }
+    #         for future in concurrent.futures.as_completed(future_to_idx):
+    #             try:
+    #                 results.append(future.result())
+    #             except Exception as exc:
+    #                 print(f"An item generated an exception: {exc}")
+    
+    #     return results
+
     # Instruction Following Evaluation (Prompt, Response)
     def _evaluate_instruction_following(self, prompt: str, response: str) -> Tuple[float, str]:
         score, checks, passed = 1.0, 0, 0