implemented pdf chunking pipeline for rag system

.gitattributes

@@ -0,0 +1,7 @@
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+venv
+.env
+__pycache__/
+*.pyc
+instance/
+*.log
+chroma_db

README.md

@@ -0,0 +1,49 @@
+# 🗺️ AI Engineering Roadmap
+
+Welcome to the AI Engineering learning roadmap. This repository tracks the journey from core RAG architecture to advanced multimodal systems and AI safety frameworks.
+
+---
+
+## 🚀 Phase 1: Foundation & Retrieval-Augmented Generation (RAG)
+* **Duration:** ~12 Days
+* **Core Objective:** Build a production-ready system capable of chatting with unstructured documents.
+
+### 📚 Core Concepts
+- [ ] **Text Chunking:** Strategies for splitting text without losing context.
+- [ ] **Embeddings:** Converting words and sentences into numerical vectors.
+- [ ] **Vector Databases:** Storing and querying high-dimensional vector data.
+- [ ] **Information Retrieval:** Fetching the most relevant context for a user query.
+- [ ] **Prompt Engineering:** Structuring system instructions for optimal LLM output.
+- [ ] **Hallucination Reduction:** Designing guardrails to ensure factual accuracy.
+- [ ] **API Development:** Building robust backends using **FastAPI**.
+- [ ] **UI Development:** Creating interactive web interfaces with **Streamlit**.
+
+📦 **Final Deliverable:** **Chat with PDF AI System**
+
+---
+
+## 🎬 Phase 2: Multimodal AI & Media Automation
+* **Duration:** ~8 Days
+* **Core Objective:** Process and synthesize multiple data types (audio, video, text) into automated pipelines.
+
+### 📚 Core Concepts
+- [ ] **Audio Transcription:** Deploying OpenAI's **Whisper** for high-accuracy speech-to-text.
+- [ ] **AI Summarization:** Condensing long-form media content into key insights.
+- [ ] **Video Processing:** Constructing automated workflows to slice and stitch video timelines.
+- [ ] **Automation Workflows:** Linking multiple model outputs into a seamless production pipeline.
+
+📦 **Final Deliverable:** **AI Reel Generator System**
+
+---
+
+## 🛡️ Phase 3: AI Safety, Security & Evaluation
+* **Duration:** ~6 Days
+* **Core Objective:** Stress-test LLMs against adversarial inputs and implement strict safety guardrails.
+
+### 📚 Core Concepts
+- [ ] **Prompt Attacks:** Understanding prompt injection and adversarial manipulation.
+- [ ] **Jailbreak Testing:** Red-teaming models to bypass standard system alignment.
+- [ ] **AI Evaluation:** Quantifying model performance, bias, and correctness.
+- [ ] **Safety Frameworks:** Implementing responsible AI principles and compliance guidelines.
+
+📦 **Final Deliverable:** **AI Safety Audit Report + Testing Framework**

ai-saftey-audit/README.md

@@ -0,0 +1,8 @@
+# GuardrailAI: Safety & Bias Evaluation Toolkit
+
+An alignment and auditing framework built to detect bias, mitigate toxicity, and secure LLM applications.
+
+### Key Features:
+* **Prompt Injection Defense:** Real-time filtering to block malicious user inputs.
+* **Bias & Fairness Auditing:** Automated test suites to scan model responses for stereotypes or unfair bias.
+* **Toxicity Guardrails:** An evaluation layer that ensures model outputs remain safe and compliant.

multimodal-engine/README.md

@@ -0,0 +1,8 @@
+# SynapseMedia: Multimodal Content Engine
+
+An AI engine designed to process, analyze, and generate content across multiple data types (Text, Images, and Audio).
+
+### Key Features:
+* **Cross-Modal Search:** Search image and video databases using natural language queries.
+* **Automated Asset Generation:** Generates text descriptions from visual inputs and vice versa.
+* **Unified API:** A single backend framework handling both vision and language models.

rag-system/README.md

@@ -0,0 +1,8 @@
+# ContextFlow: Advanced RAG Pipeline
+
+A production-ready Retrieval-Augmented Generation (RAG) system built to connect LLMs to custom data. 
+
+### Key Features:
+* **Hybrid Search:** Combines semantic vector search with keyword matching.
+* **Smart Re-ranking:** Optimizes context relevance before sending data to the LLM.
+* **Document Pipeline:** Handles PDF parsing, text chunking, and automated vector database ingestion.

rag-system/app/load_pdf.py

@@ -0,0 +1,33 @@
+import os
+from dotenv import load_dotenv
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+#from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+
+# Load environment variables
+load_dotenv()
+# load PDF
+pdf_path = r"C:/Users/ajmel/desktop/internship-projects/rag-system/data/ArtificiaL_.pdf"
+loader = PyPDFLoader(pdf_path)
+documents = loader.load()
+
+# split into chunks
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = text_splitter.split_documents(documents)
+
+#create embeding
+embeding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+persistence_directory = "./chroma_db"
+
+vectorstore = Chroma.from_documents(
+    documents=chunks,
+    embedding=embeding_model,
+    collection_name="pdf_chunks",
+    persist_directory=persistence_directory
+)
+vectorstore.persist()
+
+print(f"Successfully loaded and embedded {len(chunks)} chunks into ChromaDB!")
+

rag-system/app/testdb_load.py

@@ -0,0 +1,48 @@
+import chromadb
+
+# sample data
+raw_document = (
+    "The domestic cat (Felis catus) is a small carnivorous mammal. It is the only "
+    "domesticated species in the family Felidae. Cats are known for their agility, "
+    "exceptional hunting skills, and flexible bodies. On the other hand, the "
+    "domestic dog (Canis lupus familiaris) is a domesticated descendant of the wolf. "
+    "Dogs were the first species to be domesticated by humans over 15,000 years ago, "
+    "evolving to become uniquely attuned to human behavior."
+)
+
+# chunking
+def text_chunking(text, chunk_size = 20, overlap = 5):
+    words = text.split()
+    chunks = []
+    i = 0
+    while i < len(words):
+        chunk = " ".join(words[i: i + chunk_size])
+        chunks.append(chunk)
+
+        i += chunk_size - overlap
+
+    return chunks
+chunk_text = text_chunking(raw_document, chunk_size=20, overlap=5)
+
+chroma_client = chromadb.Client()
+collection = chroma_client.get_or_create_collection(name="animal_knowladge")
+unique_ids = [f"chunks_{i}" for i in range(len(chunk_text))]
+collection.add(
+    documents=chunk_text,
+    ids=unique_ids
+)
+
+query = "Tell me about feline traits"
+
+search_result = collection.query(
+    query_texts= [query],
+    n_results=1 
+)
+
+collection.get(ids=["chunks_0"])
+
+print(f"______rag pipeline______")
+print(f"user query: {query}\n")
+print(f"Retrived Context Chunk:")
+print(search_result["documents"][0][0])
+print(f"\nDistance Score (Lower means closer match): {search_result['distances'][0][0]}")

rag-system/data/ArtificiaL_.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cdae8b6aaa4b525decc36e0d80d2c16dc709aa924e21266640eb47cbe327800
+size 11745647

requirements.txt

@@ -0,0 +1,11 @@
+requests>=2.31.0
+cryptography>=42.0.0
+pyjwt>=2.8.0
+python-dotenv>=1.0.0
+langchain>=0.1.0
+langchain-community>=0.0.10
+langchain-openai>=0.1.0
+langchain-google-genai>=1.0.0
+chromadb>=0.4.24
+streamlit>=1.32.0
+pypdf>=6.12.0