implemented semantic retrieval pipeline using embeddings and chromadb

.gitignore

@@ -5,4 +5,4 @@ __pycache__/
 instance/
 *.log
 chroma_db
-rag-system/app/loade.py
+rag-system/test/loade.py

rag-system/app/chunker.py

@@ -1,11 +1,9 @@
-from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import NLTKTextSplitter
 
-def chunk_clean_text(text, chunk_size=500, chunk_overlap=100):
-    text_splitter = RecursiveCharacterTextSplitter(
+def chunk_clean_text(text, chunk_size=800, chunk_overlap=150):
+    text_splitter = NLTKTextSplitter(
         chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        length_function=len,
-        separators=["\n\n", "\n", " ", ""]
+        chunk_overlap=chunk_overlap
     )
 
     chunk_list = text_splitter.split_text(text)

rag-system/app/embeding.py

@@ -0,0 +1,4 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings
+
+def create_embedding():
+    return HuggingFaceEmbeddings(model_name= "all-MiniLM-L6-v2")

rag-system/app/loader.py

@@ -1,42 +1,26 @@
 import re
+import os
 from pypdf import PdfReader
 
-def extract_and_clean_pdf(pdf_path):
+def extract_clean_text_pdf(pdf_path):
+
     try:
-        reader = PdfReader(pdf_path)
+        loader = PdfReader(pdf_path)
     except Exception as e:
-        print(f"erro loading PDF: {e}")
+        print(f"Error loading PDF: {e}")
         return "", 0
-
-    raw_text_pieces = []
-
-    for page_number, page in enumerate(reader.pages):
+    
+    full_text = []
+    for page_num, page in enumerate(loader.pages):
         page_text = page.extract_text()
         if page_text:
-            raw_text_pieces.append(page_text)
+            full_text.append(page_text)
     
-    raw_text = "\n".join(raw_text_pieces)
-    cleaned_text = re.sub(r'[ \t]+', ' ', raw_text)
+    raw_text = "\n".join(full_text)
+    cleaned_text = re.sub(r'[\t]+', ' ', raw_text)
     cleaned_text = re.sub(r'\n{3,}', '\n\n', cleaned_text)
     cleaned_text = "\n".join([line.strip() for line in cleaned_text.splitlines()])
 
     total_length = len(cleaned_text)
-    first_500_chars = cleaned_text[:500]
-    
-    return cleaned_text, total_length
-
-# if __name__ == "__main__":
-#     pdf_path = r"C:/Users/ajmel/desktop/internship-projects/rag-system/data/ArtificiaL_.pdf"
-#     print(f"Reading and cleaning: {pdf_path}...\n")
-#     preview, length = extract_and_clean_pdf(pdf_path)
-    
-#     # PRINT REQUIRED METRICS
-#     print(f"=========================================")
-#     print(f"TOTAL TEXT LENGTH: {length} characters")
-#     print(f"=========================================\n")
-#     print("FIRST 500 CHARACTERS PREVIEW:")
-#     print("-" * 40)
-#     print(preview)
-#     print("-" * 40)
-
 
+    return cleaned_text, total_length

rag-system/app/main.py

@@ -1,13 +1,15 @@
 # 1. Bring in the tools from your other files
-from loader import extract_and_clean_pdf
+from loader import extract_clean_text_pdf
 from chunker import chunk_clean_text
-
+from embeding import create_embedding
+from vectore_store import create_vector_store
+from retriever import create_retriever
 def run_pipeline(pdf_path):
     print("=== STARTING RAG DATA PIPELINE ===")
     
     # 2. Load and Clean the PDF
     print(f"\nStep 1: Reading and cleaning text from: {pdf_path}...")
-    full_text, total_length = extract_and_clean_pdf(pdf_path)
+    full_text, total_length = extract_clean_text_pdf(pdf_path)
     
     # Safety check if the PDF failed to load
     if not full_text:
@@ -17,21 +19,29 @@ def run_pipeline(pdf_path):
     print(f"Success! Cleaned {total_length} characters from the document.")
 
     # 3. Chunk the Cleaned Text
-    print(f"\nStep 2: Splitting text into chunks (Size: 500, Overlap: 100)...")
-    chunks = chunk_clean_text(full_text, chunk_size=500, chunk_overlap=100)
+    print(f"\nStep 2: Splitting text into chunks (Size: 800, Overlap: 150)...")
+    chunks = chunk_clean_text(full_text, chunk_size=800, chunk_overlap=150)
     
     print(f"Success! Created {len(chunks)} total chunks.")
 
-    # 4. Preview the results
-    print(f"\n=========================================")
-    print(f"PIPELINE OUTPUT PREVIEW")
-    print(f"=========================================")
-    
-    # Show the first 2 chunks as a test sample
-    for index, chunk in enumerate(chunks[:2]):
-        print(f"\n--- CHUNK {index + 1} (Length: {len(chunk)} characters) ---")
-        print(chunk)
-        print("-" * 40)
+    # create embedding
+    print(f"\n step3: craete embedding for chunks...")
+    embeddings = create_embedding()
+    print("sucess! embedding craeted")
+
+    #4. stor vectore in db
+    print(f"creating vectore store")
+    vectore_store = create_vector_store(chunks, embeddings)
+    print("success! vectore store created and stored in chroma db")
+
+    #5 retrive similar chunks
+    print("retrive similar chunks from vectore store")
+    retriver = create_retriever(vectore_store)
+    print("sucessfully retrived similar chunks from vectore store")
+
+
+   
+
 
 if __name__ == "__main__":
     # The path to your actual PDF file

rag-system/app/retriever.py

@@ -0,0 +1,14 @@
+from vectore_store import create_vector_store
+
+def create_retriever(vector_store):
+    query = input("enter your query")
+    retriver = vector_store.as_retriever(
+        search_type = "mmr",
+        search_kwargs = {"k": 3}
+    )
+    similar_docs = retriver.invoke(query)
+
+    print("top 3: similar chunks recived from vector store")
+
+    for docs in similar_docs:
+        print(f"-- {docs.page_content[:100]}")

rag-system/app/tempCodeRunnerFile.py

@@ -1 +1,12 @@
-langchain_huggingface
+
+
+    # 6. Preview the results
+    # print(f"\n=========================================")
+    # print(f"PIPELINE OUTPUT PREVIEW")
+    # print(f"=========================================")
+    
+    # # Show the first 2 chunks as a test sample
+    # for index, chunk in enumerate(chunks[:2]):
+    #     print(f"\n--- CHUNK {index + 1} (Length: {len(chunk)} characters) ---")
+    #     print(chunk)
+    #     print("-" * 40)

rag-system/app/vectore_store.py

@@ -0,0 +1,13 @@
+from langchain_community.vectorstores import Chroma
+import chromadb
+
+def create_vector_store(chunk_list, embedding_model):
+    perssisten_client = chromadb.PersistentClient("./chroma_db")
+
+    vectore_store = Chroma.from_texts(
+        texts = chunk_list,
+        embedding = embedding_model,
+        collection_name = "rag_chunks",
+        client = perssisten_client
+    )
+    return vectore_store

rag-system/notebooks/chunk_inspect.ipynb

@@ -0,0 +1,24 @@
+{
+ "cells": [],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv (3.12.7)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

rag-system/requirements.txt

@@ -10,3 +10,4 @@ chromadb>=0.4.24
 streamlit>=1.32.0
 pypdf>=6.12.0
 sentence_transformers>=5.5.1
+nltk>=3.9.4

rag-system/test/inspect_chunk.py

@@ -0,0 +1,63 @@
+import os
+import sys
+# Ensure the parent `rag-system` directory is on sys.path so `app` package imports work
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from app.loader import extract_clean_text_pdf
+from app.chunker import chunk_clean_text
+
+def inspect_chunk(pdf_path):
+
+    print("--------starting debuging chunking quality--------------")
+
+    cleaned_text, total_length = extract_clean_text_pdf(pdf_path)
+    chunks = chunk_clean_text(cleaned_text, chunk_size=800, chunk_overlap=150)
+
+    too_short_count = 0
+    broken_sentence_count = 0
+    total_chars = 0
+
+
+    min_len_treshold = 40
+    valid_endings = (".", "!", "?", '"', "'", "”", "’")
+
+    for idx, chunk in enumerate(chunks):
+        chunk_len = len(chunk)
+        total_chars += chunk_len
+
+        is_too_short = chunk_len < min_len_treshold
+        is_broken_sentence = not chunk.endswith(valid_endings)
+
+        if is_too_short or is_broken_sentence:
+            print(f"issues found in chunk {idx + 1}: Length={chunk_len}, Ends with valid punctuation: {not is_broken_sentence}")
+
+            if is_too_short:
+                print(f"[Critical]: chunks fells below the minimum semanic length threshold")
+            if is_broken_sentence:
+                print(f"[Critical]: chunk ends with a broken sentence, which may cause loss of context for the LLM")
+            
+            if is_too_short: too_short_count += 1
+            if is_broken_sentence: broken_sentence_count += 1
+
+    avg_chunk = total_chars / len(chunks) if chunks else 0
+    broken_ratio = (broken_sentence_count / len(chunks)) * 100 if chunks else 0
+
+    print("\n--------chunking quality report--------------")
+    print(f"Total chunks: {len(chunks)}")
+    print(f"Too short chunks: {too_short_count}")
+    print(f"Broken sentence chunks: {broken_sentence_count}")
+    print(f"Average chunk length: {avg_chunk:.2f}")
+    print(f"Broken sentence ratio: {broken_ratio:.2f}%")
+
+    #
+    print("\n architectural recommendations:")
+
+    if broken_ratio > 30:
+        print(f"[Critical]: A high percentage of chunks end with broken sentences. Consider adjusting chunking parameters or implementing smarter sentence-aware chunking to preserve context for the LLM.")
+    elif avg_chunk < 150:
+        print(f"[Warning]: The average chunk length is quite low, which may lead to inefficient use of the LLM's context window. Consider increasing the chunk size or reducing overlap to create more meaningful chunks.")
+    else:
+        print(f"[Success]: The chunking quality appears to be good with a reasonable average length and low broken sentence ratio. You can proceed with this configuration for your RAG system.")
+if __name__ == "__main__":
+    pdf_path = r"C:/Users/ajmel/desktop/internship-projects/rag-system/data/ArtificiaL_.pdf"
+    inspect_chunk(pdf_path)

requirements.txt

@@ -9,4 +9,5 @@ langchain-google-genai>=1.0.0
 chromadb>=0.4.24
 streamlit>=1.32.0
 pypdf>=6.12.0
-sentence_transformers>=5.5.1
+sentence_transformers>=5.5.1
+nltk>=3.9.4