Add full RAG pipeline: agent, rag_engine, generator, knowledge_base, full Gradio UI

rag_engine.py

@@ -49,11 +49,13 @@ class MLOpsRAGEngine:
         Settings.llm = None  # We handle generation separately
 
     def _setup_vector_store(self):
+        print(f"[DEBUG] ChromaDB path: {self.chroma_path}", flush=True)
         self.chroma_client = chromadb.PersistentClient(path=self.chroma_path)
         self.chroma_collection = self.chroma_client.get_or_create_collection(
             name=COLLECTION_NAME,
             metadata={"hnsw:space": "cosine"},
         )
+        print(f"[DEBUG] ChromaDB collection '{COLLECTION_NAME}' size on init: {self.chroma_collection.count()}", flush=True)
         self.vector_store = ChromaVectorStore(chroma_collection=self.chroma_collection)
         self.storage_context = StorageContext.from_defaults(
             vector_store=self.vector_store
@@ -62,23 +64,30 @@ class MLOpsRAGEngine:
     def build_index(self, force_rebuild: bool = False) -> VectorStoreIndex:
         """Build or load the vector index from the knowledge base documents."""
         existing_count = self.chroma_collection.count()
+        print(f"[DEBUG] build_index called. existing ChromaDB count={existing_count}, force_rebuild={force_rebuild}", flush=True)
 
         if existing_count > 0 and not force_rebuild:
-            logger.info(f"Loading existing index ({existing_count} chunks in ChromaDB)")
+            print(f"[DEBUG] Reusing existing index ({existing_count} chunks)", flush=True)
             self.index = VectorStoreIndex.from_vector_store(
                 vector_store=self.vector_store,
                 embed_model=self.embed_model,
             )
             return self.index
 
-        logger.info(f"Building index from {self.knowledge_base_path}")
+        print(f"[DEBUG] Knowledge base path: {self.knowledge_base_path}", flush=True)
+        print(f"[DEBUG] Knowledge base path exists: {self.knowledge_base_path.exists()}", flush=True)
+
         if not self.knowledge_base_path.exists():
             raise FileNotFoundError(f"Knowledge base path not found: {self.knowledge_base_path}")
 
+        txt_files = sorted(self.knowledge_base_path.glob("*.txt"))
+        print(f"[DEBUG] .txt files found: {[f.name for f in txt_files]}", flush=True)
+
         # Load .txt files directly — avoids the llama-index-readers-file dependency
         documents = []
-        for txt_file in sorted(self.knowledge_base_path.glob("*.txt")):
+        for txt_file in txt_files:
             text = txt_file.read_text(encoding="utf-8")
+            print(f"[DEBUG]   loaded {txt_file.name} ({len(text)} chars)", flush=True)
             documents.append(Document(
                 text=text,
                 metadata={"file_name": txt_file.name},
@@ -87,7 +96,7 @@ class MLOpsRAGEngine:
 
         if not documents:
             raise FileNotFoundError(f"No .txt files found in {self.knowledge_base_path}")
-        logger.info(f"Loaded {len(documents)} documents")
+        print(f"[DEBUG] Total documents loaded: {len(documents)}", flush=True)
 
         parser = SentenceSplitter(
             chunk_size=CHUNK_SIZE,
@@ -95,7 +104,7 @@ class MLOpsRAGEngine:
             paragraph_separator="\n\n",
         )
         nodes = parser.get_nodes_from_documents(documents)
-        logger.info(f"Created {len(nodes)} chunks")
+        print(f"[DEBUG] Total chunks created: {len(nodes)}", flush=True)
 
         self.index = VectorStoreIndex(
             nodes=nodes,
@@ -103,7 +112,8 @@ class MLOpsRAGEngine:
             embed_model=self.embed_model,
             show_progress=True,
         )
-        logger.info("Index built successfully")
+        final_count = self.chroma_collection.count()
+        print(f"[DEBUG] Index built. ChromaDB collection size after indexing: {final_count}", flush=True)
         return self.index
 
     def retrieve(