Create smart_library_search.py

smart_library_search.py

@@ -0,0 +1,925 @@
+import chromadb
+from chromadb.config import Settings
+from sentence_transformers import SentenceTransformer
+import fitz  #chroma_db PyMuPDF
+from transformers import AutoTokenizer
+import os
+import re
+import gradio as gr
+import os
+from flask import send_file
+import google.generativeai as genai
+import tempfile
+import shutil
+
+download_counts={}
+
+# === CLOUD STORAGE URL MAPPING ===
+# Add direct download links for each PDF file
+ARTICLE_URLS = {
+    "Davit Marikyan-2019-Computing And Informatics-unified_theory_of_acceptance_and_use_of_technology.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETDFRinNWmxIo6Sri8r0tFQB3MbD3M6CcdRv5a8T5jlY-g?e=vbCFca",
+    "enard Omallah George-2015-Computing And Informatics-Role_of_E_Resources_for_Research_Managemt.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETbZst6ntatPm-pVPsOVA0YBFUpuV40t0H7c70NzqHrdRg?e=pT6Abp", 
+    "Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/EYl1sNw6NqlEqUtZcCidDdMBkxQJEjEnv8VHh2giPHXL_A?e=azCxFs",
+    "Mahmood-2025-Education-revalence and Effects of Smartphone Use on Academic Performance of Undergraduate Student Nurses: An Analytical Cross-Sectional Study.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ERgGX0tf7jpOhfy5bDhWDzkBeaxL4x1RPWfyN0sHDGnuvA?e=WNUZst",
+    "Arana_Cedeño-2022-Social Science-Wars and other current challenges for health and life.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/Eab88y2jonpMg9aSTfgkKxIBTFSIQRUJljeeG9SYhqbMiw?e=aAz1Bv",
+    "A Arunprakash-2022-Education-Investigating Digital Education, A Study on Fostering Accessibility and Equity in Learning Environments.pdf:": "https://1drv.ms/b/c/78b29f6d9bcf3843/EZ1cpWglJ4NFhQZnd83Q4z8B2u0ihaSGaZILBdvCRBuJJg?e=S7PoZK",
+    "Anyanwu, D.-2021-Computing And Informatics-Cybersecurity in the Age of the Internet of Things A Review of Challenges.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EaXYy_YTWUFPmTJp_IQuPO0BWUVes_to3J_IXYE8ZXUywg?e=mK1con",
+    "Ashish Makanadar-2020-Computing And Informatics-Digital surveillance capitalism and citiesdata, democracy and activism.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ET4Pk0ndottEtwKUnRE18XoBqt-qG9N229QUALcjIQT0BQ?e=oW3vDN",
+    "Benedict Dellot-2017-Social Science-RSA The Age Of Automation Report.pdf" : "",
+    "Carmen G. Gonzalez-2023-Social Science-Climate Change, Race, and Migration.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EfyivsAY9XxLgnOJsK_bk5IBzakhpkyVrInyv71rdUKwZg?e=xftDHo",
+    "D Paris-2020-Education-Culturally Sustaining Pedagogy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EbcH1oNsQS9FjD0rZ4xMv8kBFdhJ5n6rMSGh-eEE-g7t6w?e=UyBpgA",
+    "Ahmad Samadi-2021-Education-How Can Support and Stability Prevent Teacher Burnout and Support (1).pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWMuLDLZEo5FgkTwfUTT0LMBiPJ2-A0_5N52RPEET_UGgQ?e=56zG3D",
+    "Lesley Bartlett-2023-Education-Debating the “Science of Reading” and its Impact on Policy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ESlCuNhXdwFBoJbmU4DbZ4IB902hJ1cg_BdE11W_CMdxmA?e=LkroLE",
+    "M. SHELLEY THOMAS-2024-Education-Trauma-Informed Practices in Schools Across Two.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ERT6CkW2gHZKlXPVcRuhTzABZ-vcpXdwdwFy89eUds6zBA?e=tysA4K",
+    "Manuel Au-Yong-Oliveira-2021-Social Science-The Role of AI and Automation on the FutURE.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EXi3c_Vib75GgygLr9U04voB8vP2jZPN5RH_66-8Sgvb0w?e=6ZehVo", 
+    "Myra Marx Fettee-2022-Social Science-Inequality, Intersectionality and the Politics of Discourse.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EUToraqEywhLgAOfKFWx_jAB5JFgUweNo3QeEBjfTptniw?e=UZFKUu", 
+    "Paula Braveman-2023-Social Science-The Social Determinants.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EeKtmqDtD-xCq-zKv56M4g4B8mvZtzV4fmh3aLZufjp4jw?e=Htlay0", 
+    "Ruhee D’Cunha-2021-Computing And Informatics-Challenges in the use of quantum computing hardware-efficient Ans¨atze.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EdCAR3vs_NxFsCLdXLTKOLkBMbTfI0yEtyjn7MozZwcU5Q?e=4FwmQq", 
+    "Samuel M. Wilson-2022-Social Science-The Anthropology of Online Communities.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/Ed8oce6arD1Cp74j88qGCh8B4nYqMPc8fEik1DdzPHBBPw?e=iQeOW9", 
+    "Ullrich K. H. Ecker-2017Social Science-Why rebuttals may not work: the psychology of misinformation.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWNyYK5cK-9Fo5qve_EU4ccB_r6z5_D6s9fvT-zaG6ByNA?e=gjo72C" 
+    
+    # Add all your files here with their direct cloud storage URLs
+}
+# Initialize tokenizer for chunking
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
+
+def extract_text_from_pdf(pdf_path):
+    doc = fitz.open(pdf_path)
+    full_text = ""
+    
+    print(f"Processing {os.path.basename(pdf_path)} - {len(doc)} pages")
+    
+    for page_num in range(len(doc)):
+        page = doc.load_page(page_num)
+        text = page.get_text()
+        
+        # Skip obviously preliminary pages (title, declarations, etc.)
+        if page_num < 3:  # First few pages are usually preliminaries
+            if any(word in text.lower() for word in ['declaration', 'dedication', 'acknowledgement', 'table of content']):
+                print(f"Skipping preliminary page {page_num + 1}")
+                continue
+        
+        full_text += text + "\n"
+    
+    full_text = re.sub(r'\s+', ' ', full_text).strip()
+    print(f"Extracted {len(full_text)} characters")
+    return full_text
+
+def enhanced_extract_text_from_pdf(pdf_path):
+    """
+    Enhanced PDF text extraction that handles various PDF structures
+    """
+    doc = fitz.open(pdf_path)
+    text = ""
+    
+    for page_num, page in enumerate(doc):
+        # Extract text with different methods if needed
+        page_text = page.get_text("text")
+        
+        # Check if this page has substantial content (not just headers/footers)
+        if len(page_text.strip()) > 100:  # Avoid empty or nearly empty pages
+            text += f"\n--- Page {page_num + 1} ---\n{page_text}\n"
+    
+    # Alternative: try different extraction methods
+    if len(text.strip()) < 500:  # If too little text extracted
+        print(f"Warning: Only {len(text)} characters extracted - trying alternative method")
+        text = ""
+        for page in doc:
+            text += page.get_text("blocks")  # Try blocks method
+    
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def parse_metadata_from_filename(filename):
+    """
+    Extracts metadata from filenames in various formats.
+    Handles: Author_Name-Year-Department-Title_Keywords.pdf
+    Also handles: Other_Formats-With-Different-Structures.pdf
+    """
+    # Remove the .pdf extension
+    name_without_ext = os.path.splitext(filename)[0]
+    
+    # Default metadata
+    metadata = {
+        "source": filename,
+        "author": "Unknown Author",
+        "year": "Unknown Year", 
+        "department": "General",
+        "title": name_without_ext.replace('_', ' ').title()  # Fallback title
+    }
+    
+    # Split by hyphens to get the components
+    parts = name_without_ext.split('-')
+    
+    # Different parsing strategies based on number of parts
+    if len(parts) >= 4:
+        # Format: Author-Year-Department-Title (most structured)
+        metadata["author"] = parts[0].replace('_', ' ').title()
+        metadata["year"] = parts[1]
+        metadata["department"] = parts[2].replace('_', ' ').title()
+        metadata["title"] = ' '.join(parts[3:]).replace('_', ' ').title()
+    
+    elif len(parts) == 3:
+        # Format: Author-Year-Title or other 3-part formats
+        # Check if the second part looks like a year (4 digits)
+        if parts[1].isdigit() and len(parts[1]) == 4:
+            metadata["author"] = parts[0].replace('_', ' ').title()
+            metadata["year"] = parts[1]
+            metadata["title"] = parts[2].replace('_', ' ').title()
+        else:
+            # Not a year, so probably Author-Department-Title
+            metadata["author"] = parts[0].replace('_', ' ').title()
+            metadata["department"] = parts[1].replace('_', ' ').title()
+            metadata["title"] = parts[2].replace('_', ' ').title()
+    
+    elif len(parts) == 2:
+        # Format: Author-Title or Year-Title
+        if parts[0].isdigit() and len(parts[0]) == 4:
+            metadata["year"] = parts[0]
+            metadata["title"] = parts[1].replace('_', ' ').title()
+        else:
+            metadata["author"] = parts[0].replace('_', ' ').title()
+            metadata["title"] = parts[1].replace('_', ' ').title()
+    
+    elif len(parts) == 1:
+        # No hyphens, just use the whole filename as title
+        metadata["title"] = name_without_ext.replace('_', ' ').title()
+    
+    # Clean up author name - remove "email" prefix if present
+    if metadata["author"].lower().startswith("email"):
+        metadata["author"] = metadata["author"][5:].strip()
+    
+    # Clean up any remaining underscores in all fields
+    for key in ["author", "department", "title"]:
+        if isinstance(metadata[key], str):
+            metadata[key] = metadata[key].replace('_', ' ')
+
+    return metadata
+
+
+
+def chunk_text(text, chunk_size=1000, overlap=100):
+    """
+    Smart chunking that prioritizes main content
+    """
+    tokens = tokenizer.encode(text)
+    chunks = []
+    
+    # Skip very short chunks (headers/footers)
+    for i in range(0, len(tokens), chunk_size - overlap):
+        chunk_tokens = tokens[i:i + chunk_size]
+        if len(chunk_tokens) < 100:  # Skip very short chunks
+            continue
+            
+        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
+        
+        # Skip chunks that are mostly preliminary content
+        if is_preliminary_content(chunk_text):
+            continue
+            
+        chunks.append(chunk_text)
+    
+    return chunks
+
+def is_preliminary_content(text):
+    """
+    Identify and skip preliminary pages content
+    """
+    preliminary_keywords = [
+        'declaration', 'dedication', 'acknowledgement', 
+        'table of content', 'abstract', 'chapter one',
+        'page', '©', 'all rights reserved'
+    ]
+    
+    text_lower = text.lower()
+    return any(keyword in text_lower for keyword in preliminary_keywords)
+
+def semantic_chunk_text(text, chunk_size=512):
+    """
+    Chunk at sentence boundaries for better coherence
+    """
+    import nltk
+    nltk.download('punkt', quiet=True)
+    from nltk.tokenize import sent_tokenize
+    
+    sentences = sent_tokenize(text)
+    chunks = []
+    current_chunk = ""
+    
+    for sentence in sentences:
+        if len(current_chunk) + len(sentence) < chunk_size:
+            current_chunk += " " + sentence
+        else:
+            if current_chunk.strip():
+                chunks.append(current_chunk.strip())
+            current_chunk = sentence
+    
+    if current_chunk.strip():
+        chunks.append(current_chunk.strip())
+    
+    return chunks
+
+def process_and_index_with_chunks(directory_path):
+    """
+    Process documents and store them as chunks in ChromaDB
+    """
+    documents = []
+    metadatas = []
+    ids = []
+    embeddings_list = []
+
+    for filename in os.listdir(directory_path):
+        if filename.endswith(".pdf"):
+            file_path = os.path.join(directory_path, filename)
+            print(f"Processing {filename}...")
+
+            # Extract full text
+            text = extract_text_from_pdf(file_path)
+            
+            # Get metadata
+            file_metadata = parse_metadata_from_filename(filename)
+            file_metadata["source_file"] = filename
+            file_metadata["full_text"] = text  # Keep full text in metadata
+
+            # Split into chunks
+            chunks = chunk_text(text)
+            print(f"  Split into {len(chunks)} chunks")
+            
+            # Create embedding for each chunk and add to collection
+            for i, chunk in enumerate(chunks):
+                chunk_id = f"{filename}_chunk_{i}"
+                embedding = model.encode(chunk).tolist()
+                
+                documents.append(chunk)
+                embeddings_list.append(embedding)
+                metadatas.append(file_metadata)  # Same metadata for all chunks
+                ids.append(chunk_id)
+
+    # Add to ChromaDB
+    collection.add(
+        documents=documents,
+        embeddings=embeddings_list,
+        metadatas=metadatas,
+        ids=ids
+    )
+    print(f"✅ Indexed {len(documents)} chunks from {len(os.listdir(directory_path))} documents")
+
+def reindex_with_larger_chunks():
+    """
+    Re-index with larger chunk sizes for more content
+    """
+    print("🔄 Re-indexing with larger chunks (1000 tokens)...")
+    
+    # Clear existing collection
+    try:
+        chroma_client.delete_collection("mk_library_doc_ceelt")
+    except:
+        pass
+    
+    global collection
+    collection = chroma_client.create_collection(name="mk_library_doc_ceelt")
+    
+    # Process with larger chunks
+    process_and_index_with_chunks("pdf_store")  # This will use the updated chunk_size
+    print("🎉 Re-indexed with larger chunks!")
+# --- RUN THIS ONCE TO POPULATE YOUR DATABASE ---
+
+def smart_chunk_text(text, chunk_size=1000, overlap=100):
+    """
+    Smarter chunking that tries to preserve complete paragraphs
+    """
+    # Split into paragraphs first
+    paragraphs = [p for p in text.split('\n\n') if p.strip()]
+    
+    chunks = []
+    current_chunk = ""
+    
+    for paragraph in paragraphs:
+        if len(current_chunk) + len(paragraph) < chunk_size:
+            current_chunk += "\n\n" + paragraph
+        else:
+            if current_chunk.strip():
+                chunks.append(current_chunk.strip())
+            current_chunk = paragraph
+    
+    if current_chunk.strip():
+        chunks.append(current_chunk.strip())
+    
+    return chunks
+
+model = SentenceTransformer('all-MiniLM-L6-v2')
+chroma_client = chromadb.PersistentClient(path="chroma_db")
+
+
+# === GEMINI API CONFIGURATION ===
+GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
+if GEMINI_API_KEY:
+    try:
+        genai.configure(api_key=GEMINI_API_KEY)
+        
+        # Choose one of these working models:
+        gemini_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')  # ✅ Best option
+        # OR
+        gemini_model = genai.GenerativeModel('models/gemini-1.5-flash-latest')  # ✅ Faster option
+        # OR  
+        gemini_model = genai.GenerativeModel('models/gemini-2.0-flash')  # ✅ Good balance
+        
+        print("✅ Gemini API configured successfully")
+        
+    except Exception as e:
+        print(f"❌ Gemini configuration failed: {e}")
+        gemini_model = None
+else:
+    gemini_model = None
+    print("🔶 Gemini API not configured - using local responses")
+
+# Create (or get) a collection. Think of it as a table for your library data.
+collection = chroma_client.create_collection(name="mk_library_mor")
+
+# 🚨 RE-INDEXING STEP (RUN ONCE - THEN COMMENT OUT)
+print("Starting re-indexing with full text...")
+#reindex_all_documents()  # This will recreate the collection with full text
+print("Re-indexing completed!")
+
+
+# Re-process just this problematic PDF
+def reprocess_specific_pdf(filename):
+    pdf_path = os.path.join("pdf_store", filename)
+    
+    if os.path.exists(pdf_path):
+        print(f"Re-processing: {filename}")
+        
+        # Remove existing entry from ChromaDB
+        try:
+            collection.delete(ids=[filename])
+            print(f"Removed old entry for {filename}")
+        except:
+            print(f"No existing entry to remove for {filename}")
+        
+        # Extract with enhanced method
+        text = extract_text_from_pdf(pdf_path)
+        print(f"Extracted text length: {len(text)}")
+        
+        if len(text) > 1000:
+            # Get metadata
+            metadata = parse_metadata_from_filename(filename)
+            metadata["full_text"] = text
+            
+            # Create embedding and add to collection
+            embedding = model.encode(text).tolist()
+            doc_snippet = text[:1000] + "..." if len(text) > 1000 else text
+            
+            collection.add(
+                documents=[doc_snippet],
+                embeddings=[embedding],
+                metadatas=[metadata],
+                ids=[filename]
+            )
+            print(f"Successfully re-indexed {filename}")
+            return True
+        else:
+            print(f"Warning: Very little text extracted ({len(text)} chars)")
+            return False
+    return False
+
+# Run this for the problematic PDF
+reprocess_specific_pdf("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf")
+
+
+
+# Add this debug function to check what text was actually extracted
+def debug_pdf_text(pdf_filename):
+    file_path = os.path.join("pdf_store", pdf_filename)
+    if os.path.exists(file_path):
+        full_text = extract_text_from_pdf(file_path)
+        print(f"=== TEXT EXTRACTED FROM {pdf_filename} ===")
+        print(full_text[:1000])  # First 1000 chars
+        return full_text
+    return None
+
+# Test with the specific PDF
+debug_pdf_text("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf")
+
+
+pdf_store = "pdf_store"
+process_and_index_with_chunks(pdf_store)
+
+
+
+def semantic_search(query, n_results=5, year_filter=None, department_filter=None, author_filter=None):
+    """Search that returns individual chunks"""
+    query_embedding = model.encode([query]).tolist()
+    
+    # Build filter
+    where_filter = {}
+    if year_filter and year_filter != "All":
+        where_filter["year"] = {"$gte": year_filter}
+    if department_filter and department_filter != "All":
+        where_filter["department"] = {"$eq": department_filter}
+    if author_filter and author_filter != "All":
+        where_filter["author"] = {"$eq": author_filter}
+    
+    # Query ChromaDB
+    results = collection.query(
+        query_embeddings=query_embedding,
+        n_results=n_results,
+        where=where_filter if where_filter else None,
+        include=['metadatas', 'documents', 'distances']
+    )
+    
+    # Format results
+    output = []
+    for meta, doc_text, distance in zip(results['metadatas'][0], results['documents'][0], results['distances'][0]):
+        similarity_score = 1 - (distance / 2)
+        
+        output.append({
+            "title": meta.get('title', 'Research Document'),
+            "author": meta.get('author', 'Unknown Author'),
+            "year": meta.get('year', ''),
+            "department": meta.get('department', 'General Studies'),
+            "source": meta.get('source_file', ''),
+            "content": doc_text,  # This is now the actual chunk content
+            "relevance": f"{similarity_score:.1%}",
+            "chunk": True  # Flag that this is a chunk
+        })
+    
+    return output
+
+def prepare_context(relevant_docs):
+    """Prepare context from relevant documents"""
+    context = "Based on the following research documents:\n\n"
+    
+    for i, doc in enumerate(relevant_docs, 1):
+        context += f"Document {i}: {doc['title']} by {doc['author']} ({doc['year']})\n"
+        context += f"Content: {doc['content'][:250]}...\n\n"
+    
+    return context
+
+
+def generate_contextual_response(question, relevant_docs):
+    """Smart response with adaptive content length"""
+    if not relevant_docs:
+        return "🔍 **I couldn't find specific research on this topic.**"
+    
+    response = "**📚 Research Findings:**\n\n"
+    
+    for i, doc in enumerate(relevant_docs[:3], 1):
+        content = doc['content']
+        
+        # Show more content for highly relevant results
+        if float(doc['relevance'].strip('%')) > 70:  # Highly relevant
+            preview = content[:800] + "..." if len(content) > 800 else content
+        else:  # Moderately relevant
+            preview = content[:400] + "..." if len(content) > 400 else content
+        
+        response += f"**{i}. {doc['title']}**\n"
+        response += f"   👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n"
+        response += f"   📖 {preview}\n"
+        response += f"   🎯 Relevance: {doc['relevance']}\n\n"
+    
+    response += "**📋 Source References:**\n"
+    for doc in relevant_docs[:3]:
+        response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n"
+    
+    return response
+
+
+def generate_local_response(question, relevant_docs):
+    """Enhanced response formatting for better readability"""
+    if not relevant_docs:
+        return "🔍 **I couldn't find specific research on this topic.**\n\n**Try:**\n• Using broader search terms\n• Adjusting the filters\n• Asking about general research areas"
+    
+    # Start with a more natural introduction
+    response = "**📚 I found some relevant research for you:**\n\n"
+    
+    for i, doc in enumerate(relevant_docs[:3], 1):
+        # Create a cleaner, more readable snippet
+        content = doc['content']
+        
+        # Remove excessive whitespace and formatting issues
+        content = re.sub(r'\s+', ' ', content).strip()
+        
+        # Create a better preview - focus on the actual content
+        if len(content) > 120:
+            # Try to find a complete sentence
+            sentences = content.split('.')
+            if len(sentences) > 1 and len(sentences[0]) > 20:
+                preview = sentences[0] + '.'
+            else:
+                preview = content[:120] + '...'
+        else:
+            preview = content
+        
+        response += f"**{i}. {doc['title']}**\n"
+        response += f"   👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n"
+        response += f"   📖 {preview}\n\n"
+    
+    # Add more natural follow-up suggestions
+    response += "**💡 You might want to ask:**\n"
+    response += "• 'Can you tell me more about the first study?'\n"
+    response += "• 'What methodology was used in this research?'\n" 
+    response += "• 'What were the main findings or conclusions?'\n"
+    response += "• 'Are there similar studies on this topic?'"
+    
+    return response
+
+def create_text_snippet(text, max_words=10, query_terms=None):
+    """
+    Creates a clean text snippet from the full text.
+    - Shows the beginning of the content (not metadata like declarations)
+    - Highlights query terms if provided
+    - Limits to a specific number of words
+    """
+    # Remove extra whitespace and make lowercase for processing
+    clean_text = ' '.join(text.split())
+    
+    # Try to find the actual content (skip declarations, acknowledgements, etc.)
+    # Look for common academic document sections to find the main content
+    content_starters = [
+        "abstract", "chapter", "introduction", "background", 
+        "this study", "research", "the purpose", "objective"
+    ]
+    
+    # Find where the actual content begins
+    content_start = 0
+    lower_text = clean_text.lower()
+    for starter in content_starters:
+        pos = lower_text.find(starter)
+        if pos != -1 and (content_start == 0 or pos < content_start):
+            content_start = pos
+    
+    # If we found a content start, use that section
+    if content_start > 0:
+        snippet_text = clean_text[content_start:]
+    else:
+        snippet_text = clean_text
+    
+    # Truncate to max_words
+    words = snippet_text.split()
+    if len(words) > max_words:
+        snippet = ' '.join(words[:max_words]) + '...'
+    else:
+        snippet = snippet_text
+    
+    # Optional: Highlight query terms if provided
+    if query_terms:
+        for term in query_terms:
+            if term.lower() in snippet.lower():
+                # Simple highlighting with HTML
+                snippet = snippet.replace(term, f"<strong>{term}</strong>")
+                snippet = snippet.replace(term.lower(), f"<strong>{term.lower()}</strong>")
+                snippet = snippet.replace(term.upper(), f"<strong>{term.upper()}</strong>")
+    
+    return snippet
+
+def call_gemini_api(question, context):
+    """More specific prompt for research objectives"""
+    prompt = f"""As a research assistant, analyze this context to find the SPECIFIC RESEARCH OBJECTIVES.
+
+QUESTION: {question}
+
+CONTEXT EXCERPTS:
+{context}
+
+Instructions:
+1. Look for sections titled: "Objectives", "Research Objectives", "Study Objectives"
+2. If no specific objectives section, look for research goals or aims mentioned in introduction
+3. If found, list the specific objectives clearly
+4. If not found, state that objectives could not be located in the provided excerpts
+
+Provide a structured response:"""
+
+    try:
+        response = gemini_model.generate_content(prompt)
+        return response.text
+    except Exception as e:
+        raise Exception(f"Gemini API call failed: {str(e)}")
+
+def chat_with_research(question, chat_history, year_filter="All", department_filter="All", author_filter="All"):
+    if not question.strip():
+        return chat_history, ""
+    
+    try:
+        # Find relevant research
+        relevant_docs = semantic_search(
+            question, 
+            n_results=3,
+            year_filter=year_filter,
+            department_filter=department_filter, 
+            author_filter=author_filter
+        )
+        
+        if not relevant_docs:
+            response = "🔍 **I couldn't find specific research on this topic.**\n\n"
+            response += "**Try:**\n• Using different keywords\n• Adjusting the filters\n• Asking about broader research areas"
+            
+            chat_history.append((question, response))
+            return chat_history, ""
+        
+        # === NEW: AI-GENERATED SUMMARY SECTION ===
+        if GEMINI_API_KEY and gemini_model:
+            try:
+                # Prepare context for AI summary
+                context = "Research Context:\n"
+                for i, doc in enumerate(relevant_docs[:3], 1):
+                    context += f"\nDocument {i}: {doc['title']} by {doc['author']} ({doc['year']})\n"
+                    context += f"Content: {doc['content'][:300]}...\n"
+                
+                # Generate AI summary
+                summary_prompt = f"""Based on the following research excerpts, provide a concise summary (about 150 words) that directly answers this question: {question}
+
+                {context}
+
+                Instructions:
+                - Provide a direct, comprehensive answer to the question
+                - short in-text citation, APA 7 format
+                - Write in a natural, conversational tone
+                - Keep it around 150 words
+                - Style: Professional academic tone, no references, direct answer only
+                - Focus on the key insights from the research"""
+                
+                ai_response = gemini_model.generate_content(summary_prompt)
+                summary = ai_response.text
+                
+                # Format the response with summary first, then references
+                response = f"**🤖 AI Research Summary:**\n\n{summary}\n\n"
+                response += "**📚 Source References:**\n"
+                for doc in relevant_docs:
+                    response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n"
+                    
+            except Exception as e:
+                print(f"AI summary failed: {e}")
+                # Fallback to regular response
+                response = generate_contextual_response(question, relevant_docs)
+        else:
+            # Local mode without AI summary
+            response = generate_contextual_response(question, relevant_docs)
+        
+    except Exception as e:
+        response = f"⚠️ **I encountered a technical issue**\n\nPlease try again or ask a different question.\n\n*Error: {str(e)}*"
+    
+    chat_history.append((question, response))
+    return chat_history, ""
+
+# Function to get unique values for dropdowns from the collection's metadata
+def get_unique_metadata_values(metadata_field):
+    # Get all metadata (be cautious with very large collections)
+    all_metadata = collection.get(include=['metadatas'])['metadatas']
+    # Extract the specific field, handling missing keys
+    values = [meta.get(metadata_field, '') for meta in all_metadata]
+    # Get unique, non-empty values and sort them
+    unique_values = sorted(list(set(filter(None, values))))
+    return ["All"] + unique_values # Add "All" option
+
+# Fetch unique values for our filters (run this once when the app starts)
+unique_departments = get_unique_metadata_values('department')
+unique_authors = get_unique_metadata_values('author')
+# For years, we might just want a list of decades or a slider. Using a dropdown for simplicity.
+unique_years = sorted(list(set(meta.get('year', '2000') for meta in collection.get(include=['metadatas'])['metadatas'])))
+unique_years = ["All"] + unique_years
+
+def run_advanced_search(query, num_results, year_filter, department_filter, author_filter):
+    results = semantic_search(
+        query, 
+        n_results=num_results,
+        year_filter=year_filter, 
+        department_filter=department_filter, 
+        author_filter=author_filter
+    )
+    
+    # Group results by document source
+    grouped_results = {}
+    for res in results:
+        source = res['source']
+        if source not in grouped_results:
+            grouped_results[source] = {
+                'title': res['title'],
+                'author': res['author'],
+                'year': res['year'],
+                'department': res['department'],
+                'source': res['source'],
+                'chunks': [],
+                'best_relevance': 0.0  # Store as float for comparison
+            }
+        
+        grouped_results[source]['chunks'].append(res['content'])
+        
+        # Convert relevance percentage to float for comparison
+        current_rel = grouped_results[source]['best_relevance']
+        new_rel = float(res['relevance'].strip('%')) / 100  # Convert "85.0%" to 0.85
+        
+        if new_rel > current_rel:
+            grouped_results[source]['best_relevance'] = new_rel
+    
+    # Convert back to percentage string for display
+    for source in grouped_results:
+        grouped_results[source]['relevance'] = f"{grouped_results[source]['best_relevance'] * 100:.1f}%"
+    
+    # Convert to list and sort by relevance
+    unique_docs = list(grouped_results.values())
+    unique_docs.sort(key=lambda x: x['best_relevance'], reverse=True)
+    
+    # Now generate HTML output
+    output_html = """
+    <div style='
+        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+        color: #000000 !important;
+        line-height: 1.6;
+    '>
+    """
+    
+    if not unique_docs:
+        output_html += "<p style='color: #000000 !important; padding: 2em; text-align: center;'>No results found matching your criteria.</p>"
+        return output_html
+    
+    for res in unique_docs:
+        # Create preview from chunks
+        if res['chunks']:
+            preview_text = " ".join([chunk[:200] for chunk in res['chunks'][:2]])
+            if len(preview_text) > 400:
+                preview_text = preview_text[:400] + "..."
+            if len(res['chunks']) > 2:
+                preview_text += f" [+{len(res['chunks'])-2} more relevant sections]"
+        else:
+            preview_text = "No content preview available"
+        
+        cloud_url = ARTICLE_URLS.get(res['source'], "")
+        
+        if cloud_url:
+            download_btn = f"""
+            <div style='text-align: center; margin: 15px 0;'>
+                <a href='{cloud_url}' target='_blank' 
+                   style='display: inline-block; padding: 12px 24px; background: #28a745; color: white; text-decoration: none; border-radius: 6px; font-weight: bold;'>
+                   📥 Download PDF
+                </a>
+            </div>
+            """
+        else:
+            download_btn = """
+            <div style='text-align: center; margin: 15px 0; padding: 10px; background: #ffe6e6; border-radius: 5px;'>
+                <p style='color: #d63031; margin: 0;'>⚠️ Download not available</p>
+            </div>
+            """
+
+        output_html += f"""
+        <div style='margin-bottom: 2em; padding: 1.5em; border: 2px solid #e0e0e0; border-radius: 10px; background: #ffffff;'>
+            <h3 style='margin-top: 0; margin-bottom: 1em; color: #000000 !important; font-size: 1.4em; padding-bottom: 0.5em; border-bottom: 3px solid #3498db;'>{res['title']}</h3>
+            
+            <div style='display: grid; grid-template-columns: auto 1fr; gap: 0.5em 1em; margin-bottom: 1.5em; padding: 1em; background: #f8f9fa; border-radius: 8px;'>
+                <span style='font-weight: bold; color: #000000 !important;'>👤 Author:</span>
+                <span style='color: #000000 !important;'>{res['author']}</span>
+                
+                <span style='font-weight: bold; color: #000000 !important;'>📅 Year:</span>
+                <span style='color: #000000 !important;'>{res['year']}</span>
+                
+                <span style='font-weight: bold; color: #000000 !important;'>🏫 Department:</span>
+                <span style='color: #000000 !important;'>{res['department']}</span>
+            </div>
+            
+            {download_btn}
+            
+            <div style='background: #e8f4fc; padding: 1em; border-radius: 8px; margin-bottom: 1em; text-align: center;'>
+                <span style='color: #e74c3c !important; font-weight: bold; font-size: 1.1em;'>🎯 Relevance: {res['relevance']}</span>
+            </div>
+            
+            <div style='background: #f8f9fa; padding: 1.2em; border-radius: 8px;'>
+                <h4 style='margin-top: 0; color: #000000 !important; margin-bottom: 0.5em;'>📖 Preview:</h4>
+                <p style='margin: 0; line-height: 1.6; color: #000000 !important;'>{preview_text}</p>
+            </div>
+        </div>
+        """
+    
+    output_html += "</div>"
+    return output_html
+
+# Create the advanced interface with dropdowns
+iface = gr.Interface(
+    fn=run_advanced_search,
+    inputs=[
+        gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."),
+        gr.Dropdown(choices=unique_years, label="Published After Year", value="All"),
+        gr.Dropdown(choices=unique_departments, label="Department", value="All"),
+        gr.Dropdown(choices=unique_authors, label="Author", value="All")
+    ],
+    outputs=gr.HTML(label="Filtered Search Results"),
+    title="🏛️ Mount Kenya University - Advanced Library Search",
+    description="Find relevant resources using semantic search. Filter by year, department, or author to narrow down results."
+)
+
+# Create the advanced interface with dropdowns AND result count control
+iface = gr.Interface(
+    fn=run_advanced_search,
+    inputs=[
+        gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."),
+        gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results"),  # Slider option
+        # gr.Number(value=10, label="Number of Results", precision=0),  # Number input option
+        gr.Dropdown(choices=unique_years, label="Published After Year", value="All"),
+        gr.Dropdown(choices=unique_departments, label="Department", value="All"),
+        gr.Dropdown(choices=unique_authors, label="Author", value="All")
+    ],
+    outputs=gr.HTML(label="Filtered Search Results"),
+    title="🏛️ Mount Kenya University - Advanced Library Search",
+    description="Find relevant resources using semantic search. Choose how many results to see and filter by year, department, or author."
+)
+
+# --- NEW: Create Tabbed Interface ---
+with gr.Blocks(title="MKU Smart Library Search", theme=gr.themes.Default()) as demo:
+    gr.Markdown("# 🏛️ Mount Kenya University - Smart Library Search")
+    gr.Markdown("Explore our academic resources through semantic search or chat with our research database.")
+
+    file_download = gr.File(visible=False, label="Download PDF")
+
+    # ====== HIDDEN COMPONENTS FOR DOCUMENT TRACKING ======
+    current_pdf_title = gr.Textbox(value="", visible=False)
+    current_pdf_filename = gr.Textbox(value="", visible=False)
+    # ====== END HIDDEN COMPONENTS ======
+
+    
+    # Tab 1: Your Existing Semantic Search
+    with gr.Tab("🔍 Semantic Search"):
+        gr.Markdown("### Search our library collection with advanced filters")
+        with gr.Row():
+            with gr.Column():
+                search_query = gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture...")
+                num_results = gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results")
+                year_filter = gr.Dropdown(choices=unique_years, label="Published After Year", value="All")
+                department_filter = gr.Dropdown(choices=unique_departments, label="Department", value="All")
+                author_filter = gr.Dropdown(choices=unique_authors, label="Author", value="All")
+                search_btn = gr.Button("Search", variant="primary")
+            
+            with gr.Column():
+                search_output = gr.HTML(label="Search Results")
+        
+        # Connect your existing function
+        search_btn.click(
+            fn=run_advanced_search,
+            inputs=[search_query, num_results, year_filter, department_filter, author_filter],
+            outputs=search_output
+        )
+        
+    #tab2  chat interface    
+    with gr.Tab("💬 Chat with Research"):
+        gr.Markdown("## 🤖 Research Discussion Assistant")
+
+        # === CHAT INTERFACE ===
+        with gr.Row():
+            with gr.Column(scale=3):
+                chatbot = gr.Chatbot(
+                    label="Research Conversation",
+                    height=400,
+                    value=[
+                        ("👋", "Hello! I can help you explore research papers.")
+                    ]
+                )
+            
+            with gr.Column(scale=1):
+                gr.Markdown("### 💡 Chat Tips")
+                gr.Markdown("""
+                **Use filters to:**
+                - Focus on recent research
+                - Explore specific departments  
+                - Find authors' work
+                - Narrow down results
+                """)
+        
+        # === MESSAGE INPUT ===
+        with gr.Row():
+            msg = gr.Textbox(
+                label="Your research question",
+                placeholder="e.g., 'What are recent findings about AI in education?'",
+                scale=4
+            )
+            submit_btn = gr.Button("Send", variant="primary", scale=1)
+        
+        clear_btn = gr.Button("🔄 Clear Conversation")
+        
+        # === EVENT HANDLERS ===
+        submit_btn.click(
+            chat_with_research,
+            inputs=[msg, chatbot, year_filter, department_filter, author_filter],
+            outputs=[chatbot, msg]
+        )
+        
+        msg.submit(
+            chat_with_research,
+            inputs=[msg, chatbot, year_filter, department_filter, author_filter],
+            outputs=[chatbot, msg]
+        )
+        
+        clear_btn.click(lambda: [], None, chatbot)
+
+
+
+if __name__ == "__main__":
+    # Display configuration status
+    if GEMINI_API_KEY:
+        print("✅ Hugging Face API enabled")
+    else:
+        print("🔶 Local mode - API token not set")
+        print("💡 Get token: https://huggingface.co/settings/tokens")
+    
+    demo.launch(
+        share=True,
+        server_name="0.0.0.0",
+        server_port=7861
+    )