"""
Comprehensive Knowledge Base Converter
Converts the full ALU knowledge base to ALU Brain JSON format
"""

import json
import re
from pathlib import Path

def parse_section(section_text, section_title):
    """Parse a section and create multiple entries"""
    entries = []
    
    # Split into subsections
    subsections = re.split(r'\n###\s+', section_text)
    
    for subsection in subsections:
        if not subsection.strip():
            continue
            
        lines = subsection.split('\n')
        subtitle = lines[0].strip() if lines else ""
        content = '\n'.join(lines[1:]).strip()
        
        if not content:
            continue
        
        # Create entry for this subsection
        entry = {
            "id": f"alu_kb_{len(entries)}_{subtitle.lower().replace(' ', '_')[:30]}",
            "question": f"{section_title}: {subtitle}" if subtitle else section_title,
            "answer": content,
            "type": "text",
            "keywords": extract_keywords(section_title + " " + subtitle + " " + content),
            "category": categorize_section(section_title),
            "source": "ALU Comprehensive Knowledge Base"
        }
        entries.append(entry)
    
    return entries

def extract_keywords(text):
    """Extract relevant keywords from text"""
    # Common words to exclude
    stopwords = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should', 'could', 'may', 'might', 'must', 'can', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'what', 'which', 'who', 'when', 'where', 'why', 'how', 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very'}
    
    # Extract words
    words = re.findall(r'\b[a-z]{3,}\b', text.lower())
    
    # Filter and get unique keywords
    keywords = list(set([w for w in words if w not in stopwords]))[:15]  # Limit to 15 keywords
    
    return keywords

def categorize_section(title):
    """Categorize section based on title"""
    title_lower = title.lower()
    
    if any(word in title_lower for word in ['calendar', 'academic', 'term', 'schedule']):
        return "Academic Calendar"
    elif any(word in title_lower for word in ['spd', 'internship', 'career', 'job', 'placement']):
        return "Career Services"
    elif any(word in title_lower for word in ['immigration', 'visa', 'permit']):
        return "Immigration Support"
    elif any(word in title_lower for word in ['insurance', 'medical', 'health', 'eden care']):
        return "Health & Wellness"
    elif any(word in title_lower for word in ['student life', 'club', 'residential', 'housing']):
        return "Campus Life"
    elif any(word in title_lower for word in ['program', 'bel', 'degree', 'module', 'course']):
        return "Academic Programs"
    elif any(word in title_lower for word in ['policy', 'conduct', 'disciplinary', 'code']):
        return "Policies & Conduct"
    elif any(word in title_lower for word in ['event', 'student-led']):
        return "Student Events"
    elif any(word in title_lower for word in ['faq', 'question', 'frequently']):
        return "FAQs"
    else:
        return "General Information"

def convert_comprehensive_kb():
    """Main conversion function"""
    
    print("=" * 70)
    print("🧠 COMPREHENSIVE KNOWLEDGE BASE CONVERTER")
    print("=" * 70)
    print()
    
    # Read the comprehensive KB
    kb_file = Path("data/alu_knowledge/alu_ultimate_knowledge_base.txt")
    
    if not kb_file.exists():
        print(f"❌ Knowledge base file not found: {kb_file}")
        return False
    
    print(f"📖 Reading knowledge base from: {kb_file}")
    with open(kb_file, 'r', encoding='utf-8') as f:
        content = f.read()
    
    print(f"✅ Loaded {len(content)} characters")
    print()
    
    # Split by major sections (## headers)
    sections = re.split(r'\n##\s+', content)
    
    all_entries = []
    entry_id = 1
    
    print("🔄 Processing sections...")
    for i, section in enumerate(sections[1:], 1):  # Skip first empty section
        if not section.strip():
            continue
        
        lines = section.split('\n')
        section_title = lines[0].strip()
        section_content = '\n'.join(lines[1:])
        
        print(f"  {i}. Processing: {section_title[:50]}...")
        
        # Parse this section
        section_entries = parse_section(section_content, section_title)
        
        # Add unique IDs
        for entry in section_entries:
            entry['id'] = f"alu_kb_{entry_id:04d}"
            entry_id += 1
        
        all_entries.extend(section_entries)
    
    print()
    print(f"✅ Created {len(all_entries)} knowledge base entries")
    print()
    
    # Create the brain JSON structure
    brain_data = {
        "category": "comprehensive_knowledge",
        "description": "Complete ALU Knowledge Base - Academic Calendar, SPD Hub, Immigration, Programs, Policies, and More",
        "version": "2.0",
        "last_updated": "2024-11-20",
        "total_entries": len(all_entries),
        "entries": all_entries
    }
    
    # Save to alu_brain directory
    output_file = Path("alu_brain/comprehensive_knowledge.json")
    print(f"💾 Saving to: {output_file}")
    
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(brain_data, f, indent=2, ensure_ascii=False)
    
    print(f"✅ Successfully saved {len(all_entries)} entries")
    print()
    
    # Show statistics
    categories = {}
    for entry in all_entries:
        cat = entry['category']
        categories[cat] = categories.get(cat, 0) + 1
    
    print("📊 Entries by Category:")
    for cat, count in sorted(categories.items(), key=lambda x: x[1], reverse=True):
        print(f"  • {cat}: {count} entries")
    
    print()
    print("=" * 70)
    print("✅ CONVERSION COMPLETE!")
    print("=" * 70)
    print()
    print("📝 Sample entries:")
    for entry in all_entries[:5]:
        print(f"  • {entry['question'][:60]}...")
    
    print()
    print("🚀 Next steps:")
    print("  1. Review the generated JSON file")
    print("  2. Commit and push to Hugging Face")
    print("  3. Restart the Space")
    print("  4. Test with questions like:")
    print("     - 'How to create a club at ALU?'")
    print("     - 'Who founded ALU?'")
    print("     - 'When is the next term?'")
    print()
    
    return True

if __name__ == "__main__":
    convert_comprehensive_kb()