app.py

import chromadb
from chromadb.config import Settings
from sentence_transformers import SentenceTransformer
import fitz  #chroma_db PyMuPDF
from transformers import AutoTokenizer
import os
import re
import gradio as gr
import os
from flask import send_file
import google.generativeai as genai
import tempfile
import shutil

download_counts={}

## === CLOUD STORAGE URL MAPPING ===
# Add direct download links for each PDF file
ARTICLE_URLS = {
    "Davit Marikyan-2019-Computing And Informatics-unified_theory_of_acceptance_and_use_of_technology.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETDFRinNWmxIo6Sri8r0tFQB3MbD3M6CcdRv5a8T5jlY-g?e=vbCFca",
    "enard Omallah George-2015-Computing And Informatics-Role_of_E_Resources_for_Research_Managemt.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETbZst6ntatPm-pVPsOVA0YBFUpuV40t0H7c70NzqHrdRg?e=pT6Abp", 
    "Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/EYl1sNw6NqlEqUtZcCidDdMBkxQJEjEnv8VHh2giPHXL_A?e=azCxFs",
    "Mahmood-2025-Education-revalence and Effects of Smartphone Use on Academic Performance of Undergraduate Student Nurses: An Analytical Cross-Sectional Study.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ERgGX0tf7jpOhfy5bDhWDzkBeaxL4x1RPWfyN0sHDGnuvA?e=WNUZst",
    "Arana_Cedeño-2022-Social Science-Wars and other current challenges for health and life.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/Eab88y2jonpMg9aSTfgkKxIBTFSIQRUJljeeG9SYhqbMiw?e=aAz1Bv",
    "A Arunprakash-2022-Education-Investigating Digital Education, A Study on Fostering Accessibility and Equity in Learning Environments.pdf:": "https://1drv.ms/b/c/78b29f6d9bcf3843/EZ1cpWglJ4NFhQZnd83Q4z8B2u0ihaSGaZILBdvCRBuJJg?e=S7PoZK",
    "Anyanwu, D.-2021-Computing And Informatics-Cybersecurity in the Age of the Internet of Things A Review of Challenges.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EaXYy_YTWUFPmTJp_IQuPO0BWUVes_to3J_IXYE8ZXUywg?e=mK1con",
    "Ashish Makanadar-2020-Computing And Informatics-Digital surveillance capitalism and citiesdata, democracy and activism.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ET4Pk0ndottEtwKUnRE18XoBqt-qG9N229QUALcjIQT0BQ?e=oW3vDN",
    "Benedict Dellot-2017-Social Science-RSA The Age Of Automation Report.pdf" : "",
    "Carmen G. Gonzalez-2023-Social Science-Climate Change, Race, and Migration.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EfyivsAY9XxLgnOJsK_bk5IBzakhpkyVrInyv71rdUKwZg?e=xftDHo",
    "D Paris-2020-Education-Culturally Sustaining Pedagogy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EbcH1oNsQS9FjD0rZ4xMv8kBFdhJ5n6rMSGh-eEE-g7t6w?e=UyBpgA",
    "Ahmad Samadi-2021-Education-How Can Support and Stability Prevent Teacher Burnout and Support (1).pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWMuLDLZEo5FgkTwfUTT0LMBiPJ2-A0_5N52RPEET_UGgQ?e=56zG3D",
    "Lesley Bartlett-2023-Education-Debating the “Science of Reading” and its Impact on Policy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ESlCuNhXdwFBoJbmU4DbZ4IB902hJ1cg_BdE11W_CMdxmA?e=LkroLE",
    "M. SHELLEY THOMAS-2024-Education-Trauma-Informed Practices in Schools Across Two.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ERT6CkW2gHZKlXPVcRuhTzABZ-vcpXdwdwFy89eUds6zBA?e=tysA4K",
    "Manuel Au-Yong-Oliveira-2021-Social Science-The Role of AI and Automation on the FutURE.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EXi3c_Vib75GgygLr9U04voB8vP2jZPN5RH_66-8Sgvb0w?e=6ZehVo", 
    "Myra Marx Fettee-2022-Social Science-Inequality, Intersectionality and the Politics of Discourse.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EUToraqEywhLgAOfKFWx_jAB5JFgUweNo3QeEBjfTptniw?e=UZFKUu", 
    "Paula Braveman-2023-Social Science-The Social Determinants.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EeKtmqDtD-xCq-zKv56M4g4B8mvZtzV4fmh3aLZufjp4jw?e=Htlay0", 
    "Ruhee D’Cunha-2021-Computing And Informatics-Challenges in the use of quantum computing hardware-efficient Ans¨atze.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EdCAR3vs_NxFsCLdXLTKOLkBMbTfI0yEtyjn7MozZwcU5Q?e=4FwmQq", 
    "Samuel M. Wilson-2022-Social Science-The Anthropology of Online Communities.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/Ed8oce6arD1Cp74j88qGCh8B4nYqMPc8fEik1DdzPHBBPw?e=iQeOW9", 
    "Ullrich K. H. Ecker-2017Social Science-Why rebuttals may not work: the psychology of misinformation.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWNyYK5cK-9Fo5qve_EU4ccB_r6z5_D6s9fvT-zaG6ByNA?e=gjo72C" 
    
    # Add all your files here with their direct cloud storage URLs
}
# Initialize tokenizer for chunking
tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")

def extract_text_from_pdf(pdf_path):
    doc = fitz.open(pdf_path)
    full_text = ""
    
    print(f"Processing {os.path.basename(pdf_path)} - {len(doc)} pages")
    
    for page_num in range(len(doc)):
        page = doc.load_page(page_num)
        text = page.get_text()
        
        # Skip obviously preliminary pages (title, declarations, etc.)
        if page_num < 3:  # First few pages are usually preliminaries
            if any(word in text.lower() for word in ['declaration', 'dedication', 'acknowledgement', 'table of content']):
                print(f"Skipping preliminary page {page_num + 1}")
                continue
        
        full_text += text + "\n"
    
    full_text = re.sub(r'\s+', ' ', full_text).strip()
    print(f"Extracted {len(full_text)} characters")
    return full_text

def enhanced_extract_text_from_pdf(pdf_path):
    """
    Enhanced PDF text extraction that handles various PDF structures
    """
    doc = fitz.open(pdf_path)
    text = ""
    
    for page_num, page in enumerate(doc):
        # Extract text with different methods if needed
        page_text = page.get_text("text")
        
        # Check if this page has substantial content (not just headers/footers)
        if len(page_text.strip()) > 100:  # Avoid empty or nearly empty pages
            text += f"\n--- Page {page_num + 1} ---\n{page_text}\n"
    
    # Alternative: try different extraction methods
    if len(text.strip()) < 500:  # If too little text extracted
        print(f"Warning: Only {len(text)} characters extracted - trying alternative method")
        text = ""
        for page in doc:
            text += page.get_text("blocks")  # Try blocks method
    
    text = re.sub(r'\s+', ' ', text).strip()
    return text

def parse_metadata_from_filename(filename):
    """
    Extracts metadata from filenames in various formats.
    Handles: Author_Name-Year-Department-Title_Keywords.pdf
    Also handles: Other_Formats-With-Different-Structures.pdf
    """
    # Remove the .pdf extension
    name_without_ext = os.path.splitext(filename)[0]
    
    # Default metadata
    metadata = {
        "source": filename,
        "author": "Unknown Author",
        "year": "Unknown Year", 
        "department": "General",
        "title": name_without_ext.replace('_', ' ').title()  # Fallback title
    }
    
    # Split by hyphens to get the components
    parts = name_without_ext.split('-')
    
    # Different parsing strategies based on number of parts
    if len(parts) >= 4:
        # Format: Author-Year-Department-Title (most structured)
        metadata["author"] = parts[0].replace('_', ' ').title()
        metadata["year"] = parts[1]
        metadata["department"] = parts[2].replace('_', ' ').title()
        metadata["title"] = ' '.join(parts[3:]).replace('_', ' ').title()
    
    elif len(parts) == 3:
        # Format: Author-Year-Title or other 3-part formats
        # Check if the second part looks like a year (4 digits)
        if parts[1].isdigit() and len(parts[1]) == 4:
            metadata["author"] = parts[0].replace('_', ' ').title()
            metadata["year"] = parts[1]
            metadata["title"] = parts[2].replace('_', ' ').title()
        else:
            # Not a year, so probably Author-Department-Title
            metadata["author"] = parts[0].replace('_', ' ').title()
            metadata["department"] = parts[1].replace('_', ' ').title()
            metadata["title"] = parts[2].replace('_', ' ').title()
    
    elif len(parts) == 2:
        # Format: Author-Title or Year-Title
        if parts[0].isdigit() and len(parts[0]) == 4:
            metadata["year"] = parts[0]
            metadata["title"] = parts[1].replace('_', ' ').title()
        else:
            metadata["author"] = parts[0].replace('_', ' ').title()
            metadata["title"] = parts[1].replace('_', ' ').title()
    
    elif len(parts) == 1:
        # No hyphens, just use the whole filename as title
        metadata["title"] = name_without_ext.replace('_', ' ').title()
    
    # Clean up author name - remove "email" prefix if present
    if metadata["author"].lower().startswith("email"):
        metadata["author"] = metadata["author"][5:].strip()
    
    # Clean up any remaining underscores in all fields
    for key in ["author", "department", "title"]:
        if isinstance(metadata[key], str):
            metadata[key] = metadata[key].replace('_', ' ')

    return metadata



def chunk_text(text, chunk_size=1000, overlap=100):
    """
    Smart chunking that prioritizes main content
    """
    tokens = tokenizer.encode(text)
    chunks = []
    
    # Skip very short chunks (headers/footers)
    for i in range(0, len(tokens), chunk_size - overlap):
        chunk_tokens = tokens[i:i + chunk_size]
        if len(chunk_tokens) < 100:  # Skip very short chunks
            continue
            
        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)
        
        # Skip chunks that are mostly preliminary content
        if is_preliminary_content(chunk_text):
            continue
            
        chunks.append(chunk_text)
    
    return chunks

def is_preliminary_content(text):
    """
    Identify and skip preliminary pages content
    """
    preliminary_keywords = [
        'declaration', 'dedication', 'acknowledgement', 
        'table of content', 'abstract', 'chapter one',
        'page', '©', 'all rights reserved'
    ]
    
    text_lower = text.lower()
    return any(keyword in text_lower for keyword in preliminary_keywords)

def semantic_chunk_text(text, chunk_size=512):
    """
    Chunk at sentence boundaries for better coherence
    """
    import nltk
    nltk.download('punkt', quiet=True)
    from nltk.tokenize import sent_tokenize
    
    sentences = sent_tokenize(text)
    chunks = []
    current_chunk = ""
    
    for sentence in sentences:
        if len(current_chunk) + len(sentence) < chunk_size:
            current_chunk += " " + sentence
        else:
            if current_chunk.strip():
                chunks.append(current_chunk.strip())
            current_chunk = sentence
    
    if current_chunk.strip():
        chunks.append(current_chunk.strip())
    
    return chunks

def process_and_index_with_chunks(directory_path):
    """
    Process documents and store them as chunks in ChromaDB
    """
    documents = []
    metadatas = []
    ids = []
    embeddings_list = []

    for filename in os.listdir(directory_path):
        if filename.endswith(".pdf"):
            file_path = os.path.join(directory_path, filename)
            print(f"Processing {filename}...")

            # Extract full text
            text = extract_text_from_pdf(file_path)
            
            # Get metadata
            file_metadata = parse_metadata_from_filename(filename)
            file_metadata["source_file"] = filename
            file_metadata["full_text"] = text  # Keep full text in metadata

            # Split into chunks
            chunks = chunk_text(text)
            print(f"  Split into {len(chunks)} chunks")
            
            # Create embedding for each chunk and add to collection
            for i, chunk in enumerate(chunks):
                chunk_id = f"{filename}_chunk_{i}"
                embedding = model.encode(chunk).tolist()
                
                documents.append(chunk)
                embeddings_list.append(embedding)
                metadatas.append(file_metadata)  # Same metadata for all chunks
                ids.append(chunk_id)

    # Add to ChromaDB
    collection.add(
        documents=documents,
        embeddings=embeddings_list,
        metadatas=metadatas,
        ids=ids
    )
    print(f"✅ Indexed {len(documents)} chunks from {len(os.listdir(directory_path))} documents")

def reindex_with_larger_chunks():
    """
    Re-index with larger chunk sizes for more content
    """
    print("🔄 Re-indexing with larger chunks (1000 tokens)...")
    
    # Clear existing collection
    try:
        chroma_client.delete_collection("mk_library_doc_ceelt")
    except:
        pass
    
    global collection
    collection = chroma_client.create_collection(name="mk_library_doc_ceelt")
    
    # Process with larger chunks
    process_and_index_with_chunks("pdf_store")  # This will use the updated chunk_size
    print("🎉 Re-indexed with larger chunks!")
# --- RUN THIS ONCE TO POPULATE YOUR DATABASE ---

def smart_chunk_text(text, chunk_size=1000, overlap=100):
    """
    Smarter chunking that tries to preserve complete paragraphs
    """
    # Split into paragraphs first
    paragraphs = [p for p in text.split('\n\n') if p.strip()]
    
    chunks = []
    current_chunk = ""
    
    for paragraph in paragraphs:
        if len(current_chunk) + len(paragraph) < chunk_size:
            current_chunk += "\n\n" + paragraph
        else:
            if current_chunk.strip():
                chunks.append(current_chunk.strip())
            current_chunk = paragraph
    
    if current_chunk.strip():
        chunks.append(current_chunk.strip())
    
    return chunks

model = SentenceTransformer('all-MiniLM-L6-v2')
chroma_client = chromadb.PersistentClient(path="chroma_db")


# === GEMINI API CONFIGURATION ===
GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
if GEMINI_API_KEY:
    try:
        genai.configure(api_key=GEMINI_API_KEY)
        
        # Choose one of these working models:
        gemini_model = genai.GenerativeModel('models/gemini-1.5-pro-latest')  # ✅ Best option
        # OR
        gemini_model = genai.GenerativeModel('models/gemini-1.5-flash-latest')  # ✅ Faster option
        # OR  
        gemini_model = genai.GenerativeModel('models/gemini-2.0-flash')  # ✅ Good balance
        
        print("✅ Gemini API configured successfully")
        
    except Exception as e:
        print(f"❌ Gemini configuration failed: {e}")
        gemini_model = None
else:
    gemini_model = None
    print("🔶 Gemini API not configured - using local responses")

# Create (or get) a collection. Think of it as a table for your library data.
collection = chroma_client.create_collection(name="mk_library_mor")

# 🚨 RE-INDEXING STEP (RUN ONCE - THEN COMMENT OUT)
print("Starting re-indexing with full text...")
#reindex_all_documents()  # This will recreate the collection with full text
print("Re-indexing completed!")


# Re-process just this problematic PDF
def reprocess_specific_pdf(filename):
    pdf_path = os.path.join("pdf_store", filename)
    
    if os.path.exists(pdf_path):
        print(f"Re-processing: {filename}")
        
        # Remove existing entry from ChromaDB
        try:
            collection.delete(ids=[filename])
            print(f"Removed old entry for {filename}")
        except:
            print(f"No existing entry to remove for {filename}")
        
        # Extract with enhanced method
        text = extract_text_from_pdf(pdf_path)
        print(f"Extracted text length: {len(text)}")
        
        if len(text) > 1000:
            # Get metadata
            metadata = parse_metadata_from_filename(filename)
            metadata["full_text"] = text
            
            # Create embedding and add to collection
            embedding = model.encode(text).tolist()
            doc_snippet = text[:1000] + "..." if len(text) > 1000 else text
            
            collection.add(
                documents=[doc_snippet],
                embeddings=[embedding],
                metadatas=[metadata],
                ids=[filename]
            )
            print(f"Successfully re-indexed {filename}")
            return True
        else:
            print(f"Warning: Very little text extracted ({len(text)} chars)")
            return False
    return False

# Run this for the problematic PDF
reprocess_specific_pdf("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf")



# Add this debug function to check what text was actually extracted
def debug_pdf_text(pdf_filename):
    file_path = os.path.join("pdf_store", pdf_filename)
    if os.path.exists(file_path):
        full_text = extract_text_from_pdf(file_path)
        print(f"=== TEXT EXTRACTED FROM {pdf_filename} ===")
        print(full_text[:1000])  # First 1000 chars
        return full_text
    return None

# Test with the specific PDF
debug_pdf_text("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf")


pdf_store = "pdf_store"
process_and_index_with_chunks(pdf_store)



def semantic_search(query, n_results=5, year_filter=None, department_filter=None, author_filter=None):
    """Search that returns individual chunks"""
    query_embedding = model.encode([query]).tolist()
    
    # Build filter
    where_filter = {}
    if year_filter and year_filter != "All":
        where_filter["year"] = {"$gte": int(year_filter)}
    if department_filter and department_filter != "All":
        where_filter["department"] = {"$eq": department_filter}
    if author_filter and author_filter != "All":
        where_filter["author"] = {"$eq": author_filter}
    
    # Query ChromaDB
    results = collection.query(
        query_embeddings=query_embedding,
        n_results=n_results,
        where=where_filter if where_filter else None,
        include=['metadatas', 'documents', 'distances']
    )
    
    # Format results
    output = []
    for meta, doc_text, distance in zip(results['metadatas'][0], results['documents'][0], results['distances'][0]):
        similarity_score = 1 - (distance / 2)
        
        output.append({
            "title": meta.get('title', 'Research Document'),
            "author": meta.get('author', 'Unknown Author'),
            "year": meta.get('year', ''),
            "department": meta.get('department', 'General Studies'),
            "source": meta.get('source_file', ''),
            "content": doc_text,  # This is now the actual chunk content
            "relevance": f"{similarity_score:.1%}",
            "chunk": True  # Flag that this is a chunk
        })
    
    return output

def prepare_context(relevant_docs):
    """Prepare context from relevant documents"""
    context = "Based on the following research documents:\n\n"
    
    for i, doc in enumerate(relevant_docs, 1):
        context += f"Document {i}: {doc['title']} by {doc['author']} ({doc['year']})\n"
        context += f"Content: {doc['content'][:250]}...\n\n"
    
    return context


def generate_contextual_response(question, relevant_docs):
    """Smart response with adaptive content length"""
    if not relevant_docs:
        return "🔍 **I couldn't find specific research on this topic.**"
    
    response = "**📚 Research Findings:**\n\n"
    
    for i, doc in enumerate(relevant_docs[:3], 1):
        content = doc['content']
        
        # Show more content for highly relevant results
        if float(doc['relevance'].strip('%')) > 70:  # Highly relevant
            preview = content[:800] + "..." if len(content) > 800 else content
        else:  # Moderately relevant
            preview = content[:400] + "..." if len(content) > 400 else content
        
        response += f"**{i}. {doc['title']}**\n"
        response += f"   👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n"
        response += f"   📖 {preview}\n"
        response += f"   🎯 Relevance: {doc['relevance']}\n\n"
    
    response += "**📋 Source References:**\n"
    for doc in relevant_docs[:3]:
        response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n"
    
    return response


def generate_local_response(question, relevant_docs):
    """Enhanced response formatting for better readability"""
    if not relevant_docs:
        return "🔍 **I couldn't find specific research on this topic.**\n\n**Try:**\n• Using broader search terms\n• Adjusting the filters\n• Asking about general research areas"
    
    # Start with a more natural introduction
    response = "**📚 I found some relevant research for you:**\n\n"
    
    for i, doc in enumerate(relevant_docs[:3], 1):
        # Create a cleaner, more readable snippet
        content = doc['content']
        
        # Remove excessive whitespace and formatting issues
        content = re.sub(r'\s+', ' ', content).strip()
        
        # Create a better preview - focus on the actual content
        if len(content) > 120:
            # Try to find a complete sentence
            sentences = content.split('.')
            if len(sentences) > 1 and len(sentences[0]) > 20:
                preview = sentences[0] + '.'
            else:
                preview = content[:120] + '...'
        else:
            preview = content
        
        response += f"**{i}. {doc['title']}**\n"
        response += f"   👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n"
        response += f"   📖 {preview}\n\n"
    
    # Add more natural follow-up suggestions
    response += "**💡 You might want to ask:**\n"
    response += "• 'Can you tell me more about the first study?'\n"
    response += "• 'What methodology was used in this research?'\n" 
    response += "• 'What were the main findings or conclusions?'\n"
    response += "• 'Are there similar studies on this topic?'"
    
    return response

def create_text_snippet(text, max_words=10, query_terms=None):
    """
    Creates a clean text snippet from the full text.
    - Shows the beginning of the content (not metadata like declarations)
    - Highlights query terms if provided
    - Limits to a specific number of words
    """
    # Remove extra whitespace and make lowercase for processing
    clean_text = ' '.join(text.split())
    
    # Try to find the actual content (skip declarations, acknowledgements, etc.)
    # Look for common academic document sections to find the main content
    content_starters = [
        "abstract", "chapter", "introduction", "background", 
        "this study", "research", "the purpose", "objective"
    ]
    
    # Find where the actual content begins
    content_start = 0
    lower_text = clean_text.lower()
    for starter in content_starters:
        pos = lower_text.find(starter)
        if pos != -1 and (content_start == 0 or pos < content_start):
            content_start = pos
    
    # If we found a content start, use that section
    if content_start > 0:
        snippet_text = clean_text[content_start:]
    else:
        snippet_text = clean_text
    
    # Truncate to max_words
    words = snippet_text.split()
    if len(words) > max_words:
        snippet = ' '.join(words[:max_words]) + '...'
    else:
        snippet = snippet_text
    
    # Optional: Highlight query terms if provided
    if query_terms:
        for term in query_terms:
            if term.lower() in snippet.lower():
                # Simple highlighting with HTML
                snippet = snippet.replace(term, f"<strong>{term}</strong>")
                snippet = snippet.replace(term.lower(), f"<strong>{term.lower()}</strong>")
                snippet = snippet.replace(term.upper(), f"<strong>{term.upper()}</strong>")
    
    return snippet

def call_gemini_api(question, context):
    """More specific prompt for research objectives"""
    prompt = f"""As a research assistant, analyze this context to find the SPECIFIC RESEARCH OBJECTIVES.

QUESTION: {question}

CONTEXT EXCERPTS:
{context}

Instructions:
1. Look for sections titled: "Objectives", "Research Objectives", "Study Objectives"
2. If no specific objectives section, look for research goals or aims mentioned in introduction
3. If found, list the specific objectives clearly
4. If not found, state that objectives could not be located in the provided excerpts

Provide a structured response:"""

    try:
        response = gemini_model.generate_content(prompt)
        return response.text
    except Exception as e:
        raise Exception(f"Gemini API call failed: {str(e)}")

def chat_with_research(question, chat_history, year_filter="All", department_filter="All", author_filter="All"):
    if not question.strip():
        return chat_history, ""
    
    try:
        # Find relevant research
        relevant_docs = semantic_search(
            question, 
            n_results=3,
            year_filter=year_filter,
            department_filter=department_filter, 
            author_filter=author_filter
        )
        
        if not relevant_docs:
            response = "🔍 **I couldn't find specific research on this topic.**\n\n"
            response += "**Try:**\n• Using different keywords\n• Adjusting the filters\n• Asking about broader research areas"
            
            chat_history.append((question, response))
            return chat_history, ""
        
        # === NEW: AI-GENERATED SUMMARY SECTION ===
        if GEMINI_API_KEY and gemini_model:
            try:
                # Prepare context for AI summary
                context = "Research Context:\n"
                for i, doc in enumerate(relevant_docs[:3], 1):
                    context += f"\nDocument {i}: {doc['title']} by {doc['author']} ({doc['year']})\n"
                    context += f"Content: {doc['content'][:300]}...\n"
                
                # Generate AI summary
                summary_prompt = f"""Based on the following research excerpts, provide a concise summary (about 150 words) that directly answers this question: {question}

                {context}

                Instructions:
                - Provide a direct, comprehensive answer to the question
                - short in-text citation, APA 7 format
                - Write in a natural, conversational tone
                - Keep it around 150 words
                - Style: Professional academic tone, no references, direct answer only
                - Focus on the key insights from the research"""
                
                ai_response = gemini_model.generate_content(summary_prompt)
                summary = ai_response.text
                
                # Format the response with summary first, then references
                response = f"**🤖 AI Research Summary:**\n\n{summary}\n\n"
                response += "**📚 Source References:**\n"
                for doc in relevant_docs:
                    response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n"
                    
            except Exception as e:
                print(f"AI summary failed: {e}")
                # Fallback to regular response
                response = generate_contextual_response(question, relevant_docs)
        else:
            # Local mode without AI summary
            response = generate_contextual_response(question, relevant_docs)
        
    except Exception as e:
        response = f"⚠️ **I encountered a technical issue**\n\nPlease try again or ask a different question.\n\n*Error: {str(e)}*"
    
    chat_history.append((question, response))
    return chat_history, ""

# Function to get unique values for dropdowns from the collection's metadata
def get_unique_metadata_values(metadata_field):
    # Get all metadata (be cautious with very large collections)
    all_metadata = collection.get(include=['metadatas'])['metadatas']
    # Extract the specific field, handling missing keys
    values = [meta.get(metadata_field, '') for meta in all_metadata]
    # Get unique, non-empty values and sort them
    unique_values = sorted(list(set(filter(None, values))))
    return ["All"] + unique_values # Add "All" option

# Fetch unique values for our filters (run this once when the app starts)
unique_departments = get_unique_metadata_values('department')
unique_authors = get_unique_metadata_values('author')
# For years, we might just want a list of decades or a slider. Using a dropdown for simplicity.
unique_years = sorted(list(set(meta.get('year', '2000') for meta in collection.get(include=['metadatas'])['metadatas'])))
unique_years = ["All"] + unique_years

def run_advanced_search(query, num_results, year_filter, department_filter, author_filter):
    results = semantic_search(
        query, 
        n_results=num_results,
        year_filter=year_filter, 
        department_filter=department_filter, 
        author_filter=author_filter
    )
    
    # Group results by document source
    grouped_results = {}
    for res in results:
        source = res['source']
        if source not in grouped_results:
            grouped_results[source] = {
                'title': res['title'],
                'author': res['author'],
                'year': res['year'],
                'department': res['department'],
                'source': res['source'],
                'chunks': [],
                'best_relevance': 0.0  # Store as float for comparison
            }
        
        grouped_results[source]['chunks'].append(res['content'])
        
        # Convert relevance percentage to float for comparison
        current_rel = grouped_results[source]['best_relevance']
        new_rel = float(res['relevance'].strip('%')) / 100  # Convert "85.0%" to 0.85
        
        if new_rel > current_rel:
            grouped_results[source]['best_relevance'] = new_rel
    
    # Convert back to percentage string for display
    for source in grouped_results:
        grouped_results[source]['relevance'] = f"{grouped_results[source]['best_relevance'] * 100:.1f}%"
    
    # Convert to list and sort by relevance
    unique_docs = list(grouped_results.values())
    unique_docs.sort(key=lambda x: x['best_relevance'], reverse=True)
    
    # Now generate HTML output
    output_html = """
    <div style='
        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
        color: #000000 !important;
        line-height: 1.6;
    '>
    """
    
    if not unique_docs:
        output_html += "<p style='color: #000000 !important; padding: 2em; text-align: center;'>No results found matching your criteria.</p>"
        return output_html
    
    for res in unique_docs:
        # Create preview from chunks
        if res['chunks']:
            preview_text = " ".join([chunk[:200] for chunk in res['chunks'][:2]])
            if len(preview_text) > 400:
                preview_text = preview_text[:400] + "..."
            if len(res['chunks']) > 2:
                preview_text += f" [+{len(res['chunks'])-2} more relevant sections]"
        else:
            preview_text = "No content preview available"
        
        cloud_url = ARTICLE_URLS.get(res['source'], "")
        
        if cloud_url:
            download_btn = f"""
            <div style='text-align: center; margin: 15px 0;'>
                <a href='{cloud_url}' target='_blank' 
                   style='display: inline-block; padding: 12px 24px; background: #28a745; color: white; text-decoration: none; border-radius: 6px; font-weight: bold;'>
                   📥 Download PDF
                </a>
            </div>
            """
        else:
            download_btn = """
            <div style='text-align: center; margin: 15px 0; padding: 10px; background: #ffe6e6; border-radius: 5px;'>
                <p style='color: #d63031; margin: 0;'>⚠️ Download not available</p>
            </div>
            """

        output_html += f"""
        <div style='margin-bottom: 2em; padding: 1.5em; border: 2px solid #e0e0e0; border-radius: 10px; background: #ffffff;'>
            <h3 style='margin-top: 0; margin-bottom: 1em; color: #000000 !important; font-size: 1.4em; padding-bottom: 0.5em; border-bottom: 3px solid #3498db;'>{res['title']}</h3>
            
            <div style='display: grid; grid-template-columns: auto 1fr; gap: 0.5em 1em; margin-bottom: 1.5em; padding: 1em; background: #f8f9fa; border-radius: 8px;'>
                <span style='font-weight: bold; color: #000000 !important;'>👤 Author:</span>
                <span style='color: #000000 !important;'>{res['author']}</span>
                
                <span style='font-weight: bold; color: #000000 !important;'>📅 Year:</span>
                <span style='color: #000000 !important;'>{res['year']}</span>
                
                <span style='font-weight: bold; color: #000000 !important;'>🏫 Department:</span>
                <span style='color: #000000 !important;'>{res['department']}</span>
            </div>
            
            {download_btn}
            
            <div style='background: #e8f4fc; padding: 1em; border-radius: 8px; margin-bottom: 1em; text-align: center;'>
                <span style='color: #e74c3c !important; font-weight: bold; font-size: 1.1em;'>🎯 Relevance: {res['relevance']}</span>
            </div>
            
            <div style='background: #f8f9fa; padding: 1.2em; border-radius: 8px;'>
                <h4 style='margin-top: 0; color: #000000 !important; margin-bottom: 0.5em;'>📖 Preview:</h4>
                <p style='margin: 0; line-height: 1.6; color: #000000 !important;'>{preview_text}</p>
            </div>
        </div>
        """
    
    output_html += "</div>"
    return output_html

# Create the advanced interface with dropdowns
iface = gr.Interface(
    fn=run_advanced_search,
    inputs=[
        gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."),
        gr.Dropdown(choices=unique_years, label="Published After Year", value="All"),
        gr.Dropdown(choices=unique_departments, label="Department", value="All"),
        gr.Dropdown(choices=unique_authors, label="Author", value="All")
    ],
    outputs=gr.HTML(label="Filtered Search Results"),
    title="🏛️ Mount Kenya University - Advanced Library Search",
    description="Find relevant resources using semantic search. Filter by year, department, or author to narrow down results."
)

# Create the advanced interface with dropdowns AND result count control
iface = gr.Interface(
    fn=run_advanced_search,
    inputs=[
        gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."),
        gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results"),  # Slider option
        # gr.Number(value=10, label="Number of Results", precision=0),  # Number input option
        gr.Dropdown(choices=unique_years, label="Published After Year", value="All"),
        gr.Dropdown(choices=unique_departments, label="Department", value="All"),
        gr.Dropdown(choices=unique_authors, label="Author", value="All")
    ],
    outputs=gr.HTML(label="Filtered Search Results"),
    title="🏛️ Mount Kenya University - Advanced Library Search",
    description="Find relevant resources using semantic search. Choose how many results to see and filter by year, department, or author."
)

# --- NEW: Create Tabbed Interface ---
with gr.Blocks(title="MKU Smart Library Search", theme=gr.themes.Default()) as demo:
    gr.Markdown("# 🏛️ Mount Kenya University - Smart Library Search")
    gr.Markdown("Explore our academic resources through semantic search or chat with our research database.")

    file_download = gr.File(visible=False, label="Download PDF")

    # ====== HIDDEN COMPONENTS FOR DOCUMENT TRACKING ======
    current_pdf_title = gr.Textbox(value="", visible=False)
    current_pdf_filename = gr.Textbox(value="", visible=False)
    # ====== END HIDDEN COMPONENTS ======

    
    # Tab 1: Your Existing Semantic Search
    with gr.Tab("🔍 Semantic Search"):
        gr.Markdown("### Search our library collection with advanced filters")
        with gr.Row():
            with gr.Column():
                search_query = gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture...")
                num_results = gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results")
                year_filter = gr.Dropdown(choices=unique_years, label="Published After Year", value="All")
                department_filter = gr.Dropdown(choices=unique_departments, label="Department", value="All")
                author_filter = gr.Dropdown(choices=unique_authors, label="Author", value="All")
                search_btn = gr.Button("Search", variant="primary")
            
            with gr.Column():
                search_output = gr.HTML(label="Search Results")
        
        # Connect your existing function
        search_btn.click(
            fn=run_advanced_search,
            inputs=[search_query, num_results, year_filter, department_filter, author_filter],
            outputs=search_output
        )
        
    #tab2  chat interface    
    with gr.Tab("💬 Chat with Research"):
        gr.Markdown("## 🤖 Research Discussion Assistant")

        # === CHAT INTERFACE ===
        with gr.Row():
            with gr.Column(scale=3):
                chatbot = gr.Chatbot(
                    label="Research Conversation",
                    height=400,
                    value=[
                        ("👋", "Hello! I can help you explore research papers.")
                    ]
                )
            
            with gr.Column(scale=1):
                gr.Markdown("### 💡 Chat Tips")
                gr.Markdown("""
                **Use filters to:**
                - Focus on recent research
                - Explore specific departments  
                - Find authors' work
                - Narrow down results
                """)
        
        # === MESSAGE INPUT ===
        with gr.Row():
            msg = gr.Textbox(
                label="Your research question",
                placeholder="e.g., 'What are recent findings about AI in education?'",
                scale=4
            )
            submit_btn = gr.Button("Send", variant="primary", scale=1)
        
        clear_btn = gr.Button("🔄 Clear Conversation")
        
        # === EVENT HANDLERS ===
        submit_btn.click(
            chat_with_research,
            inputs=[msg, chatbot, year_filter, department_filter, author_filter],
            outputs=[chatbot, msg]
        )
        
        msg.submit(
            chat_with_research,
            inputs=[msg, chatbot, year_filter, department_filter, author_filter],
            outputs=[chatbot, msg]
        )
        
        clear_btn.click(lambda: [], None, chatbot)



if __name__ == "__main__":
    # Display configuration status
    if GEMINI_API_KEY:
        print("✅ Hugging Face API enabled")
    else:
        print("🔶 Local mode - API token not set")
        print("💡 Get token: https://huggingface.co/settings/tokens")
    
    demo.launch(
        share=True,
        server_name="0.0.0.0",
        server_port=7860
    )