Add full RAG pipeline: agent, rag_engine, generator, knowledge_base, full Gradio UI

agent.py

@@ -22,7 +22,6 @@ from rag_engine import MLOpsRAGEngine
 logger = logging.getLogger(__name__)
 
 MAX_ITERATIONS = 2
-RELEVANCE_SCORE_THRESHOLD = 0.30
 
 
 @dataclass
@@ -96,21 +95,19 @@ class MLOpsRAGAgent:
             logger.error(f"Retrieval failed: {e}")
             return [], []
 
-        # Filter by similarity score threshold first (fast path)
-        score_filtered = [
-            n for n in nodes
-            if self.rag.get_node_score(n) >= RELEVANCE_SCORE_THRESHOLD
-        ]
+        # ChromaDB returns cosine *distance* (lower = more similar).
+        # A score threshold on distance would silently discard the best chunks,
+        # so we pass all retrieved nodes to the Flan-T5 relevance check instead.
+        scores = [self.rag.get_node_score(n) for n in nodes]
+        logger.info(f"Raw node distances: {[round(s, 3) for s in scores]}")
 
-        # Self-reflection relevance check on remaining nodes
         relevant = []
-        for node in score_filtered:
+        for node in nodes:
             text = self.rag.get_node_text(node)
             try:
                 if self.gen.check_relevance(query, text):
                     relevant.append(node)
             except Exception:
-                # On error, include by score only
                 relevant.append(node)
 
         logger.info(f"Nodes after relevance filtering: {len(relevant)}/{len(nodes)}")

generator.py

@@ -95,7 +95,10 @@ class FlanT5Generator:
             f"Answer:"
         )
         result = self._generate(prompt, max_new_tokens=5).lower()
-        return "yes" in result
+        # Lenient: include the chunk unless the model explicitly says "no".
+        # "yes" in result is too strict — Flan-T5 sometimes outputs synonyms or
+        # fuller sentences; requiring an explicit "no" avoids false negatives.
+        return "no" not in result
 
     def reflect_on_answer(self, query: str, answer: str) -> tuple[bool, str]:
         """Self-reflect on whether the generated answer adequately addresses the query."""

rag_engine.py

@@ -16,8 +16,9 @@ from llama_index.vector_stores.chroma import ChromaVectorStore
 logger = logging.getLogger(__name__)
 
 EMBED_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
-KNOWLEDGE_BASE_PATH = "./knowledge_base"
-CHROMA_DB_PATH = "./chroma_db"
+_HERE = Path(__file__).parent
+KNOWLEDGE_BASE_PATH = str(_HERE / "knowledge_base")
+CHROMA_DB_PATH = str(_HERE / "chroma_db")
 COLLECTION_NAME = "mlops_knowledge_base"
 CHUNK_SIZE = 512
 CHUNK_OVERLAP = 64