Add multilingual RAG chatbot

app.py

@@ -0,0 +1,174 @@
+"""
+Gradio Blocks UI for the multilingual RAG chatbot.
+Run: python3 app.py
+"""
+
+import subprocess
+import pathlib
+import gradio as gr
+from rag_pipeline import RAGPipeline
+
+if not pathlib.Path("faiss_index").exists():
+    print("FAISS index not found — running ingest.py...")
+    subprocess.run(["python3", "ingest.py"], check=True)
+
+pipeline = RAGPipeline()
+
+DESCRIPTION = """
+# 🌐 Multilingual RAG Chatbot
+Ask questions in **English**, **العربية (Arabic)**, or **Bahasa Melayu (Malay)**.
+Answers are grounded in the knowledge base. Retrieved source passages appear below each answer.
+"""
+
+EXAMPLES = [
+    ["What is retrieval-augmented generation?"],
+    ["How does machine learning differ from deep learning?"],
+    ["What are vector databases used for?"],
+    ["ما هو الذكاء الاصطناعي؟"],
+    ["كيف يعمل التعلم الآلي؟"],
+    ["ما هي قواعد البيانات المتجهية؟"],
+    ["Apakah itu kecerdasan buatan?"],
+    ["Bagaimana pembelajaran mesin berfungsi?"],
+    ["Apakah RAG dan kegunaannya?"],
+]
+
+RTL_CSS = """
+.message-bubble p, .message-bubble span, .prose p {
+    unicode-bidi: plaintext;
+    text-align: start;
+}
+[lang="ar"], .rtl-text {
+    direction: rtl;
+    text-align: right;
+    font-family: 'Segoe UI', Tahoma, Arial, sans-serif;
+}
+.source-box {
+    border-left: 3px solid #6366f1;
+    padding-left: 0.75rem;
+    margin: 0.5rem 0;
+    font-size: 0.875rem;
+}
+.example-btn { font-size: 0.82rem !important; }
+"""
+
+JS_RTL = """
+function applyRTL() {
+    document.querySelectorAll('.message-bubble p, .prose p').forEach(el => {
+        if (/[؀-ۿ]/.test(el.innerText || '')) {
+            el.setAttribute('dir', 'rtl');
+            el.style.textAlign = 'right';
+        }
+    });
+}
+const observer = new MutationObserver(applyRTL);
+observer.observe(document.body, { childList: true, subtree: true });
+applyRTL();
+"""
+
+
+def format_sources(sources: list[dict]) -> str:
+    if not sources:
+        return "_No sources retrieved._"
+    lines = []
+    for i, s in enumerate(sources, 1):
+        snippet = s["content"].replace("\n", " ").strip()
+        if len(snippet) > 300:
+            snippet = snippet[:300] + "…"
+        file_name = s["source"].split("/")[-1]
+        lines.append(f"**[{i}] `{file_name}`**\n> {snippet}")
+    return "\n\n".join(lines)
+
+
+def respond(message: str, history: list):
+    if not message.strip():
+        yield history, "_Please enter a question._"
+        return
+
+    result = pipeline.ask(message)
+    answer = result["answer"]
+    sources_md = format_sources(result["sources"])
+
+    history = history + [
+        {"role": "user", "content": message},
+        {"role": "assistant", "content": answer},
+    ]
+    yield history, sources_md
+
+
+def clear_all():
+    pipeline.chain.memory.clear()
+    return [], "_Sources will appear here after your first question._"
+
+
+with gr.Blocks(title="Multilingual RAG Chatbot") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            chatbot = gr.Chatbot(
+                elem_id="chatbot",
+                label="Chat",
+                height=480,
+            )
+
+            with gr.Row():
+                msg_box = gr.Textbox(
+                    placeholder="Type your question in English, العربية, or Bahasa Melayu…",
+                    show_label=False,
+                    scale=8,
+                    autofocus=True,
+                )
+                send_btn = gr.Button("Send ➤", variant="primary", scale=1)
+
+            with gr.Row():
+                clear_btn = gr.Button("🗑 Clear Chat", variant="secondary", size="sm")
+
+        with gr.Column(scale=2):
+            gr.Markdown("### 📄 Retrieved Sources")
+            sources_box = gr.Markdown(
+                value="_Sources will appear here after your first question._",
+            )
+
+    with gr.Accordion("💡 Example Questions", open=True):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("**🇬🇧 English**")
+                for ex in EXAMPLES[:3]:
+                    gr.Button(ex[0], elem_classes="example-btn").click(
+                        fn=lambda q=ex[0]: q,
+                        outputs=msg_box,
+                    )
+            with gr.Column():
+                gr.Markdown("**🇸🇦 العربية**")
+                for ex in EXAMPLES[3:6]:
+                    gr.Button(ex[0], elem_classes="example-btn").click(
+                        fn=lambda q=ex[0]: q,
+                        outputs=msg_box,
+                    )
+            with gr.Column():
+                gr.Markdown("**🇲🇾 Bahasa Melayu**")
+                for ex in EXAMPLES[6:]:
+                    gr.Button(ex[0], elem_classes="example-btn").click(
+                        fn=lambda q=ex[0]: q,
+                        outputs=msg_box,
+                    )
+
+    submit_inputs = [msg_box, chatbot]
+    submit_outputs = [chatbot, sources_box]
+
+    msg_box.submit(respond, submit_inputs, submit_outputs).then(
+        fn=lambda: "", outputs=msg_box
+    )
+    send_btn.click(respond, submit_inputs, submit_outputs).then(
+        fn=lambda: "", outputs=msg_box
+    )
+    clear_btn.click(clear_all, outputs=[chatbot, sources_box])
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        show_error=True,
+        css=RTL_CSS,
+        js=JS_RTL,
+    )

data/sample_docs.txt

@@ -0,0 +1,43 @@
+===== ENGLISH =====
+
+Artificial Intelligence (AI) is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction. AI applications include expert systems, natural language processing, speech recognition, and machine vision.
+
+Machine learning is a subset of AI that provides systems the ability to automatically learn and improve from experience without being explicitly programmed. It focuses on the development of computer programs that can access data and use it to learn for themselves.
+
+Deep learning is part of a broader family of machine learning methods based on artificial neural networks with representation learning. Learning can be supervised, semi-supervised or unsupervised.
+
+Natural Language Processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language, in particular how to program computers to process and analyze large amounts of natural language data.
+
+Vector databases store data as high-dimensional vectors and are optimized for similarity search. They are widely used in AI applications for semantic search, recommendation systems, and retrieval-augmented generation (RAG).
+
+Retrieval-Augmented Generation (RAG) is an AI framework that retrieves relevant information from a knowledge base before generating a response. It combines the power of retrieval systems with generative language models to produce accurate, grounded answers.
+
+===== ARABIC =====
+
+الذكاء الاصطناعي هو محاكاة عمليات الذكاء البشري بواسطة الآلات، وخاصة أنظمة الكمبيوتر. تشمل هذه العمليات التعلم والتفكير والتصحيح الذاتي. تطبيقات الذكاء الاصطناعي تشمل الأنظمة الخبيرة ومعالجة اللغة الطبيعية والتعرف على الكلام والرؤية الآلية.
+
+التعلم الآلي هو فرع من الذكاء الاصطناعي يوفر للأنظمة القدرة على التعلم التلقائي والتحسن من الخبرة دون أن تتم برمجتها بشكل صريح. يركز على تطوير برامج الكمبيوتر التي يمكنها الوصول إلى البيانات واستخدامها للتعلم بمفردها.
+
+معالجة اللغة الطبيعية هي مجال فرعي من اللغويات وعلوم الكمبيوتر والذكاء الاصطناعي يهتم بالتفاعلات بين الحواسيب واللغة البشرية. يهدف إلى تمكين الحواسيب من فهم النصوص والكلام البشري وتحليلها.
+
+قواعد البيانات المتجهية تخزن البيانات كمتجهات عالية الأبعاد وهي محسّنة للبحث عن التشابه. تُستخدم على نطاق واسع في تطبيقات الذكاء الاصطناعي للبحث الدلالي وأنظمة التوصية والتوليد المعزز بالاسترجاع.
+
+التوليد المعزز بالاسترجاع هو إطار عمل يجمع بين استرجاع المعلومات ذات الصلة من قاعدة المعرفة وتوليد الاستجابات. يجمع قوة أنظمة الاسترجاع مع نماذج اللغة التوليدية لإنتاج إجابات دقيقة ومؤسسة.
+
+الشبكات العصبية الاصطناعية هي نماذج حوسبية مستوحاة من الشبكات العصبية البيولوجية في الدماغ البشري. تتكون من طبقات من العقد أو الخلايا العصبية الاصطناعية المترابطة التي تعالج المعلومات.
+
+===== MALAY =====
+
+Kecerdasan buatan (AI) ialah simulasi proses kecerdasan manusia oleh mesin, terutamanya sistem komputer. Proses-proses ini termasuk pembelajaran, penaakulan, dan pembetulan diri. Aplikasi AI termasuk sistem pakar, pemprosesan bahasa semula jadi, pengecaman pertuturan, dan penglihatan mesin.
+
+Pembelajaran mesin adalah subset AI yang memberikan sistem keupayaan untuk belajar secara automatik dan bertambah baik daripada pengalaman tanpa diprogramkan secara eksplisit. Ia memberi tumpuan kepada pembangunan program komputer yang boleh mengakses data dan menggunakannya untuk belajar sendiri.
+
+Pembelajaran mendalam adalah sebahagian daripada keluarga kaedah pembelajaran mesin yang lebih luas berdasarkan rangkaian neural buatan dengan pembelajaran perwakilan. Pembelajaran boleh diselia, separuh diselia atau tidak diselia.
+
+Pemprosesan Bahasa Semula Jadi (NLP) ialah subbidang linguistik, sains komputer, dan kecerdasan buatan yang berkaitan dengan interaksi antara komputer dan bahasa manusia. Ia bertujuan untuk membolehkan komputer memahami dan menganalisis data bahasa semula jadi dalam jumlah yang besar.
+
+Pangkalan data vektor menyimpan data sebagai vektor berdimensi tinggi dan dioptimumkan untuk carian persamaan. Ia digunakan secara meluas dalam aplikasi AI untuk carian semantik, sistem cadangan, dan penjanaan yang ditambah dengan pengambilan semula.
+
+Penjanaan Bertambah Pengambilan Semula (RAG) ialah rangka kerja AI yang mengambil semula maklumat yang relevan daripada pangkalan pengetahuan sebelum menjana respons. Ia menggabungkan kuasa sistem pengambilan semula dengan model bahasa generatif untuk menghasilkan jawapan yang tepat dan berasas.
+
+Model bahasa besar (LLM) ialah model kecerdasan buatan yang dilatih pada set data teks yang besar. Ia boleh menjana teks seperti manusia, menjawab soalan, meringkaskan dokumen, menterjemah bahasa, dan melaksanakan banyak tugas bahasa yang lain.

ingest.py

@@ -0,0 +1,49 @@
+"""
+Loads data/sample_docs.txt, chunks it, embeds with multilingual HuggingFace
+model, and saves a FAISS index to faiss_index/.
+"""
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+load_dotenv()
+
+EMBEDDING_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+SOURCE_FILE = Path("data/sample_docs.txt")
+INDEX_DIR = Path("faiss_index")
+CHUNK_SIZE = 500
+CHUNK_OVERLAP = 100
+
+
+def main():
+    if not SOURCE_FILE.exists():
+        raise FileNotFoundError(f"{SOURCE_FILE} not found.")
+
+    print(f"Loading {SOURCE_FILE}...")
+    loader = TextLoader(str(SOURCE_FILE), encoding="utf-8")
+    docs = loader.load()
+
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+        separators=["\n\n", "\n", ".", "،", " ", ""],
+    )
+    chunks = splitter.split_documents(docs)
+    print(f"Created {len(chunks)} chunks.")
+
+    print("Embedding chunks (this may take a moment)...")
+    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+    vectorstore = FAISS.from_documents(chunks, embeddings)
+
+    INDEX_DIR.mkdir(exist_ok=True)
+    vectorstore.save_local(str(INDEX_DIR))
+    print(f"FAISS index saved to {INDEX_DIR}/")
+
+
+if __name__ == "__main__":
+    main()

packages.txt

@@ -0,0 +1 @@
+libgomp1

rag_pipeline.py

@@ -0,0 +1,94 @@
+"""
+RAG pipeline: loads the FAISS index and builds a ConversationalRetrievalChain
+backed by Groq (llama-3.3-70b-versatile). Returns the answer and source chunks.
+"""
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_classic.chains import ConversationalRetrievalChain
+from langchain_classic.memory import ConversationBufferMemory
+from langchain_groq import ChatGroq
+
+load_dotenv()
+
+EMBEDDING_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+INDEX_DIR = Path("faiss_index")
+GROQ_MODEL = "llama-3.3-70b-versatile"
+TOP_K = 4
+
+SYSTEM_PROMPT = (
+    "You are a helpful multilingual assistant that supports English, Arabic, and Malay. "
+    "Always reply in the same language the user used. "
+    "Base your answer strictly on the provided context. "
+    "If the context does not contain enough information, say so honestly."
+)
+
+
+def load_vectorstore() -> FAISS:
+    if not INDEX_DIR.exists():
+        raise FileNotFoundError(
+            "FAISS index not found. Run `python ingest.py` first."
+        )
+    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+    return FAISS.load_local(
+        str(INDEX_DIR), embeddings, allow_dangerous_deserialization=True
+    )
+
+
+class RAGPipeline:
+    def __init__(self):
+        vectorstore = load_vectorstore()
+        retriever = vectorstore.as_retriever(search_kwargs={"k": TOP_K})
+
+        llm = ChatGroq(
+            model=GROQ_MODEL,
+            api_key=os.environ["GROQ_API_KEY"],
+            max_tokens=1024,
+        )
+
+        memory = ConversationBufferMemory(
+            memory_key="chat_history",
+            return_messages=True,
+            output_key="answer",
+        )
+
+        self.chain = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            retriever=retriever,
+            memory=memory,
+            return_source_documents=True,
+            output_key="answer",
+            combine_docs_chain_kwargs={"prompt": _build_prompt()},
+        )
+
+    def ask(self, question: str) -> dict:
+        """
+        Returns:
+            {
+                "answer": str,
+                "sources": [{"content": str, "source": str}, ...]
+            }
+        """
+        result = self.chain.invoke({"question": question})
+        sources = [
+            {
+                "content": doc.page_content,
+                "source": doc.metadata.get("source", "unknown"),
+            }
+            for doc in result.get("source_documents", [])
+        ]
+        return {"answer": result["answer"], "sources": sources}
+
+
+def _build_prompt():
+    from langchain_core.prompts import PromptTemplate
+
+    template = (
+        SYSTEM_PROMPT
+        + "\n\nContext:\n{context}\n\nQuestion: {question}\n\nAnswer:"
+    )
+    return PromptTemplate(input_variables=["context", "question"], template=template)

requirements.txt

@@ -0,0 +1,8 @@
+langchain>=0.3.0
+langchain-community>=0.3.0
+langchain-huggingface>=0.1.0
+langchain-groq>=0.2.0
+faiss-cpu>=1.8.0
+sentence-transformers>=3.0.0
+gradio>=4.44.0
+python-dotenv>=1.0.0