fix(backend): sanitize Ellipsis and placeholders in K2 response conversion

app/services/k2_think_engine.py

@@ -21,6 +21,61 @@ from langchain_openai import ChatOpenAI
 from langchain.schema import HumanMessage
 
 
+def is_empty_or_placeholder_dict(d: dict) -> bool:
+    """Check if a dictionary only contains None, Ellipsis, or empty placeholder strings."""
+    for v in d.values():
+        if v is not None and v is not Ellipsis and v != "":
+            return False
+    return True
+
+
+def sanitize_ellipsis_and_placeholders(data: Any) -> Any:
+    """
+    Recursively remove Ellipsis, None, "...", and invalid placeholders from the data structure.
+    """
+    if data is Ellipsis or data is None:
+        return None
+    
+    if isinstance(data, str):
+        cleaned_str = data.strip()
+        if cleaned_str in ("...", "Ellipsis", "..", "."):
+            return ""
+        return data
+        
+    if isinstance(data, list):
+        cleaned_list = []
+        for item in data:
+            if item is Ellipsis or item is None:
+                continue
+            if isinstance(item, str):
+                cleaned_item_str = item.strip()
+                if cleaned_item_str in ("...", "Ellipsis", "..", "."):
+                    continue
+            cleaned_item = sanitize_ellipsis_and_placeholders(item)
+            if cleaned_item is not None:
+                if isinstance(cleaned_item, dict) and is_empty_or_placeholder_dict(cleaned_item):
+                    continue
+                cleaned_list.append(cleaned_item)
+        return cleaned_list
+        
+    if isinstance(data, dict):
+        cleaned_dict = {}
+        for k, v in data.items():
+            if v is Ellipsis or v is None:
+                continue
+            if isinstance(v, str):
+                cleaned_val_str = v.strip()
+                if cleaned_val_str in ("...", "Ellipsis", "..", "."):
+                    cleaned_dict[k] = ""
+                    continue
+            cleaned_v = sanitize_ellipsis_and_placeholders(v)
+            if cleaned_v is not None:
+                cleaned_dict[k] = cleaned_v
+        return cleaned_dict
+        
+    return data
+
+
 class K2ThinkEngine:
     """
     Moteur K2 Think - IA Principal Unique
@@ -427,6 +482,9 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
                 }
 
             # 7. Conversion en objets schemas.py
+            # Clean k2_analysis of Ellipsis and other placeholders
+            k2_analysis = sanitize_ellipsis_and_placeholders(k2_analysis)
+            
             comp_analysis = self._convert_k2_to_comparative_analysis(k2_analysis, request.documents)
             hypotheses = self._convert_k2_to_counter_hypotheses(k2_analysis)
             protocol = await self._convert_k2_to_protocol(k2_analysis)
@@ -501,15 +559,23 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
             raw_gaps = [raw_gaps]
 
         for gap in raw_gaps:
-            if not gap: continue
+            if gap is None or gap is Ellipsis:
+                continue
             if isinstance(gap, dict):
+                gap_desc = gap.get("description") or gap.get("gap_description") or gap.get("name") or "Research Gap Detected"
+                imp_score = gap.get("importance_score") or gap.get("importance")
+                try:
+                    imp_score = float(imp_score) if imp_score is not None else 0.8
+                except (ValueError, TypeError):
+                    imp_score = 0.8
+                    
                 gaps.append(ResearchGap(
-                    gap_description=gap.get("description", gap.get("gap_description", gap.get("name", "Research Gap Detected"))),
-                    importance_score=float(gap.get("importance_score", gap.get("importance", 0.8))),
-                    related_variables=gap.get("related_variables", gap.get("variables", [])),
-                    suggested_investigation=gap.get("suggested_investigation", gap.get("investigation", "Investigation required")),
+                    gap_description=gap_desc,
+                    importance_score=imp_score,
+                    related_variables=gap.get("related_variables") or gap.get("variables") or [],
+                    suggested_investigation=gap.get("suggested_investigation") or gap.get("investigation") or "Investigation required",
                     source_documents=[doc.id for doc in docs],
-                    citations=gap.get("citations", [])
+                    citations=gap.get("citations") or []
                 ))
             else:
                 gaps.append(ResearchGap(
@@ -520,13 +586,28 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
                     source_documents=[doc.id for doc in docs]
                 ))
 
+        raw_divergences = raw_comp.get("divergences")
+        divergences = raw_divergences if isinstance(raw_divergences, list) else []
+        
+        raw_contradictions = raw_comp.get("contradictions")
+        contradictions = raw_contradictions if isinstance(raw_contradictions, list) else []
+        
+        raw_common = raw_comp.get("common_findings")
+        common_findings = raw_common if isinstance(raw_common, list) else []
+
+        conf_score = raw_comp.get("confidence_score")
+        try:
+            conf_score = float(conf_score) if conf_score is not None else 0.8
+        except (ValueError, TypeError):
+            conf_score = 0.8
+
         return ComparativeAnalysis(
             document_ids=[doc.id for doc in docs],
-            divergences=raw_comp.get("divergences", []),
-            contradictions=raw_comp.get("contradictions", []),
-            common_findings=raw_comp.get("common_findings", []),
+            divergences=divergences,
+            contradictions=contradictions,
+            common_findings=common_findings,
             research_gaps=gaps,
-            confidence_score=raw_comp.get("confidence_score", 0.8)
+            confidence_score=conf_score
         )
 
     def _convert_k2_to_counter_hypotheses(
@@ -534,14 +615,26 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
         k2_result: Dict[str, Any]
     ) -> List[CounterHypothesis]:
         hypotheses = []
-        for h in k2_result.get("counter_hypotheses", []):
+        raw_hypotheses = k2_result.get("counter_hypotheses") or []
+        if not isinstance(raw_hypotheses, list):
+            raw_hypotheses = []
+            
+        for h in raw_hypotheses:
+            if not h: continue
             if isinstance(h, dict):
+                conf_against = h.get("confidence_against")
+                try:
+                    conf_against = float(conf_against) if conf_against is not None else 0.5
+                except (ValueError, TypeError):
+                    conf_against = 0.5
+                    
                 hypotheses.append(CounterHypothesis(
-                    hypothesis=h.get("hypothesis", "Hypothesis"),
-                    rationale=h.get("rationale", ""),
-                    potential_bias=h.get("potential_bias", ""),
-                    validation_experiment=h.get("validation_experiment", ""),
-                    confidence_against=h.get("confidence_against", 0.5)
+                    hypothesis=h.get("hypothesis") or "Hypothesis",
+                    rationale=h.get("rationale") or "",
+                    potential_bias=h.get("potential_bias") or "",
+                    validation_experiment=h.get("validation_experiment") or "",
+                    confidence_against=conf_against,
+                    citations=h.get("citations") or []
                 ))
         return hypotheses
 
@@ -553,23 +646,49 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
         if not isinstance(proto_data, dict): proto_data = {}
         
         steps = []
-        for i, s in enumerate(proto_data.get("steps", []), 1):
+        raw_steps = proto_data.get("steps") or []
+        if not isinstance(raw_steps, list):
+            raw_steps = []
+            
+        for i, s in enumerate(raw_steps, 1):
             if isinstance(s, dict):
+                dur_hours = s.get("duration_hours")
+                try:
+                    dur_hours = float(dur_hours) if dur_hours is not None else 1.0
+                except (ValueError, TypeError):
+                    dur_hours = 1.0
+                    
                 steps.append(ExperimentalStep(
                     step_number=i,
-                    description=s.get("description", f"Step {i}"),
-                    duration_hours=float(s.get("duration_hours", 1)),
-                    materials=s.get("materials", []),
-                    critical_parameters=s.get("critical_parameters", [])
+                    description=s.get("description") or f"Step {i}",
+                    duration_hours=dur_hours,
+                    materials=s.get("materials") or [],
+                    critical_parameters=s.get("critical_parameters") or [],
+                    validation_criteria=s.get("validation_criteria") or "Standard validation",
+                    risk_level=s.get("risk_level") or "low",
+                    contingency_plan=s.get("contingency_plan")
                 ))
 
-        raw_vars = proto_data.get("variables", [])
+        raw_vars = proto_data.get("variables") or []
+        if not isinstance(raw_vars, list):
+            raw_vars = []
+            
         valid_vars = []
         for v in raw_vars:
             if isinstance(v, dict):
-                valid_vars.append(v)
+                valid_vars.append(ExperimentalVariable(
+                    name=v.get("name") or "Variable",
+                    type=v.get("type") or "independent",
+                    measurement_unit=v.get("measurement_unit"),
+                    measurement_method=v.get("measurement_method") or "TBD",
+                    possible_values=v.get("possible_values")
+                ))
             else:
-                valid_vars.append({"name": str(v), "type": "independent", "measurement_method": "TBD"})
+                valid_vars.append(ExperimentalVariable(
+                    name=str(v),
+                    type="independent",
+                    measurement_method="TBD"
+                ))
 
         def _ensure_str(val, default="TBD"):
             if val is None: return default
@@ -577,6 +696,12 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
                 return "\n- ".join([str(x) for x in val]) if val else default
             return str(val)
 
+        est_dur = proto_data.get("estimated_duration_days")
+        try:
+            est_dur = float(est_dur) if est_dur is not None else 30.0
+        except (ValueError, TypeError):
+            est_dur = 30.0
+
         return ExperimentalProtocol(
             title=_ensure_str(proto_data.get("title"), "New Protocol"),
             objective=_ensure_str(proto_data.get("objective"), "Objective"),
@@ -585,10 +710,10 @@ DO NOT USE <think> TAGS. DO NOT CONVERSE.
             expected_outcomes=_ensure_str(proto_data.get("expected_outcomes"), "TBD"),
             variables=valid_vars,
             statistical_analysis_plan=_ensure_str(proto_data.get("statistical_analysis_plan"), "Standard descriptive statistics"),
-            success_criteria=proto_data.get("success_criteria", ["Completion of all steps"]),
-            estimated_duration_days=float(proto_data.get("estimated_duration_days", 30.0)),
-            alternative_approaches=proto_data.get("alternative_approaches", ["None specified"]),
-            risk_assessment=proto_data.get("risk_assessment", {"overall_risk": "low"})
+            success_criteria=proto_data.get("success_criteria") or ["Completion of all steps"],
+            estimated_duration_days=est_dur,
+            alternative_approaches=proto_data.get("alternative_approaches") or ["None specified"],
+            risk_assessment=proto_data.get("risk_assessment") or {"overall_risk": "low"}
         )
 
     def _log_reasoning(self, phase: str, step: str, description: str):