feat: Add SQLite metadata dump script and UI, remove PostgreSQL

Dockerfile

@@ -0,0 +1,44 @@
+FROM timescale/timescaledb-ha:pg17
+
+USER root
+
+# Install Python 3, pip, and venv
+RUN apt-get update && \
+    apt-get install -y python3 python3-pip python3-venv curl nginx && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Setup directories for Hugging Face Spaces (UID 1000)
+# HF Persistent Storage is mounted at /data if enabled.
+RUN mkdir -p /data/postgres /var/lib/postgresql/data /app /var/log/nginx /var/lib/nginx /etc/nginx && \
+    chown -R 1000:1000 /data /var/lib/postgresql/data /var/run/postgresql /app /var/log/nginx /var/lib/nginx /etc/nginx
+
+# Switch to HF user
+USER 1000
+WORKDIR /app
+
+# Create virtual environment and install dependencies
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+COPY requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy Nginx config
+COPY nginx.conf /etc/nginx/nginx.conf
+
+# Postgres variables
+ENV POSTGRES_PASSWORD=password
+ENV PGDATA=/data/postgres
+
+# Copy initialization SQL
+COPY init.sql /docker-entrypoint-initdb.d/
+
+# Copy application code
+COPY app.py run.sh ingest_worker.py api.py /app/
+
+# Expose port 7860 (Hugging Face default)
+EXPOSE 7860
+
+# Execute the supervisor script
+CMD ["bash", "/app/run.sh"]

README.md

@@ -1,11 +1,11 @@
 ---
-title: Test
-emoji: 🌍
+title: Samela
+emoji: 👀
 colorFrom: red
 colorTo: purple
 sdk: docker
 pinned: false
-short_description: tesst
+short_description: samela
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

api.py

@@ -0,0 +1,156 @@
+import os
+from fastapi import FastAPI, Query, HTTPException
+from sqlalchemy import create_engine, text
+from typing import Optional
+
+DB_URI = os.getenv("DB_URI", "postgresql://postgres:password@localhost:5432/postgres")
+engine = create_engine(DB_URI)
+
+app = FastAPI(
+    title="Shamela4 Library API",
+    description="REST API to query the Shamela4 dataset including categories, authors, books, and pages.",
+    version="1.0.0"
+)
+
+def fetch_data(query: str, params: dict = {}):
+    try:
+        with engine.connect() as conn:
+            result = conn.execute(text(query), params)
+            return [dict(row._mapping) for row in result]
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+def fetch_one(query: str, params: dict = {}):
+    try:
+        with engine.connect() as conn:
+            result = conn.execute(text(query), params).fetchone()
+            if result:
+                return dict(result._mapping)
+            return None
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/stats", tags=["Stats"])
+def get_statistics():
+    """Get database statistics (counts)."""
+    try:
+        with engine.connect() as conn:
+            books_count = conn.execute(text("SELECT COUNT(book_id) FROM books")).scalar() or 0
+            authors_count = conn.execute(text("SELECT COUNT(author_id) FROM authors")).scalar() or 0
+            categories_count = conn.execute(text("SELECT COUNT(category_id) FROM categories")).scalar() or 0
+            pages_count = conn.execute(text("SELECT COUNT(page_id) FROM pages")).scalar() or 0
+            synced_books = conn.execute(text("SELECT COUNT(DISTINCT book_id) FROM pages")).scalar() or 0
+            return {
+                "total_categories": categories_count,
+                "total_authors": authors_count,
+                "total_books": books_count,
+                "synced_books": synced_books,
+                "total_pages_downloaded": pages_count,
+                "sync_percentage": round((synced_books / books_count * 100), 2) if books_count > 0 else 0
+            }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/categories", tags=["Categories"])
+def get_categories(limit: int = Query(50, le=1000), offset: int = 0):
+    """Get list of categories."""
+    query = "SELECT category_id, category_name_ar, name_en, sort_order FROM categories ORDER BY sort_order LIMIT :limit OFFSET :offset"
+    return fetch_data(query, {"limit": limit, "offset": offset})
+
+@app.get("/api/categories/{category_id}", tags=["Categories"])
+def get_category_detail(category_id: int):
+    """Get details of a specific category."""
+    query = "SELECT category_id, category_name_ar, name_en, sort_order FROM categories WHERE category_id = :category_id"
+    category = fetch_one(query, {"category_id": category_id})
+    if not category:
+        raise HTTPException(status_code=404, detail="Category not found")
+    return category
+
+@app.get("/api/authors", tags=["Authors"])
+def get_authors(search: Optional[str] = None, limit: int = Query(50, le=1000), offset: int = 0):
+    """Get list of authors. Optionally search by name."""
+    if search:
+        query = "SELECT author_id, name_ar, death_hijri FROM authors WHERE name_ar LIKE :search ORDER BY death_hijri LIMIT :limit OFFSET :offset"
+        return fetch_data(query, {"search": f"%{search}%", "limit": limit, "offset": offset})
+    else:
+        query = "SELECT author_id, name_ar, death_hijri FROM authors ORDER BY death_hijri LIMIT :limit OFFSET :offset"
+        return fetch_data(query, {"limit": limit, "offset": offset})
+
+@app.get("/api/authors/{author_id}", tags=["Authors"])
+def get_author_detail(author_id: int):
+    """Get details of a specific author and their books."""
+    query = "SELECT * FROM authors WHERE author_id = :author_id"
+    author = fetch_one(query, {"author_id": author_id})
+    if not author:
+        raise HTTPException(status_code=404, detail="Author not found")
+        
+    books_query = "SELECT book_id, title_ar, category_id, volume_count_observed FROM books WHERE main_author_id = :author_id ORDER BY book_id"
+    author['books'] = fetch_data(books_query, {"author_id": author_id})
+    return author
+
+@app.get("/api/books", tags=["Books"])
+def get_books(search: Optional[str] = None, category_id: Optional[int] = None, author_id: Optional[int] = None, limit: int = Query(50, le=1000), offset: int = 0):
+    """Get list of books. Filter by category, author, or search keyword."""
+    where_clauses = []
+    params = {"limit": limit, "offset": offset}
+    
+    if search:
+        where_clauses.append("(title_ar LIKE :search OR main_author_name_ar LIKE :search)")
+        params["search"] = f"%{search}%"
+    if category_id:
+        where_clauses.append("category_id = :category_id")
+        params["category_id"] = category_id
+    if author_id:
+        where_clauses.append("main_author_id = :author_id")
+        params["author_id"] = author_id
+        
+    where_str = " WHERE " + " AND ".join(where_clauses) if where_clauses else ""
+    query = f"SELECT book_id, title_ar, category_id, main_author_id, main_author_name_ar, main_author_death_hijri, volume_count_observed, version_major FROM books {where_str} ORDER BY book_id LIMIT :limit OFFSET :offset"
+    return fetch_data(query, params)
+
+@app.get("/api/books/{book_id}", tags=["Books"])
+def get_book_detail(book_id: int):
+    """Get details of a specific book, including synopsis and TOC."""
+    query = "SELECT * FROM books WHERE book_id = :book_id"
+    book = fetch_one(query, {"book_id": book_id})
+    if not book:
+        raise HTTPException(status_code=404, detail="Book not found")
+        
+    # Get TOC (Daftar Isi) if available
+    toc_query = "SELECT title_id, title_text, page_id, parent_id FROM toc WHERE book_id = :book_id ORDER BY title_id"
+    book['toc'] = fetch_data(toc_query, {"book_id": book_id})
+    
+    # Get total pages
+    pages_query = "SELECT COUNT(*) as total_pages FROM pages WHERE book_id = :book_id"
+    pages_result = fetch_one(pages_query, {"book_id": book_id})
+    book['total_pages'] = pages_result['total_pages'] if pages_result else 0
+    
+    return book
+
+@app.get("/api/books/{book_id}/pages", tags=["Pages"])
+def get_book_pages(book_id: int, limit: int = Query(100, le=500), offset: int = 0):
+    """Get pages of a specific book, ordered by sequence_num."""
+    query = "SELECT * FROM pages WHERE book_id = :book_id ORDER BY sequence_num LIMIT :limit OFFSET :offset"
+    return fetch_data(query, {"book_id": book_id, "limit": limit, "offset": offset})
+
+@app.get("/api/search", tags=["Search"])
+def search_text(query: str, limit: int = Query(20, le=100), offset: int = 0):
+    """Full-text search inside book pages."""
+    # We use PostgreSQL to_tsquery for full-text search. The body column has a GIN index on to_tsvector('arabic', body).
+    # We format the query string to join words with & (AND) for tsquery
+    words = [w for w in query.split() if w.strip()]
+    if not words:
+        return []
+    tsquery = " & ".join(words)
+    
+    sql = """
+        SELECT p.page_id, p.book_id, p.page_num, p.sequence_num, 
+               ts_headline('arabic', p.body, to_tsquery('arabic', :tsquery)) as highlight,
+               b.title_ar as book_title
+        FROM pages p
+        JOIN books b ON p.book_id = b.book_id
+        WHERE to_tsvector('arabic', p.body) @@ to_tsquery('arabic', :tsquery)
+        ORDER BY p.book_id, p.sequence_num
+        LIMIT :limit OFFSET :offset
+    """
+    return fetch_data(sql, {"tsquery": tsquery, "limit": limit, "offset": offset})

app.py

@@ -0,0 +1,41 @@
+import streamlit as st
+import os
+
+st.set_page_config(page_title="Shamela4 SQLite Ingest", layout="centered", page_icon="⚙️")
+
+st.title("⚙️ Shamela4 SQLite Ingest")
+
+st.markdown("<br>", unsafe_allow_html=True)
+st.info("Aplikasi ini secara khusus digunakan untuk mengambil data (ingest) metadata kitab dari HuggingFace dan mengubahnya menjadi format dump database SQLite (.sql). Seluruh proses ini tidak memerlukan koneksi ke PostgreSQL.", icon="ℹ️")
+st.markdown("<br>", unsafe_allow_html=True)
+
+st.markdown("### 💾 Ekspor Skema & Metadata (SQLite)")
+st.write("Unduh skema lengkap database (termasuk struktur tabel `pages`, `toc` dll yang kosong) beserta *insert data* untuk metadata (`authors`, `categories`, `books`).")
+st.markdown("---")
+
+col_btn1, col_btn2 = st.columns(2)
+
+with col_btn1:
+    if st.button("🔄 Generate Ulang File SQL", use_container_width=True):
+        with st.spinner("Sedang men-generate file metadata_dump.sql dari HuggingFace... (Ini bisa memakan waktu beberapa saat)"):
+            try:
+                from create_sqlite_dump import generate_sqlite_dump
+                generate_sqlite_dump()
+                st.success("File berhasil diperbarui!")
+            except Exception as e:
+                st.error(f"Terjadi kesalahan: {e}")
+
+with col_btn2:
+    if os.path.exists("metadata_dump.sql"):
+        with open("metadata_dump.sql", "r", encoding="utf-8") as f:
+            sql_data = f.read()
+        st.download_button(
+            label="⬇️ Download metadata_dump.sql",
+            data=sql_data,
+            file_name="metadata_dump.sql",
+            mime="application/sql",
+            type="primary",
+            use_container_width=True
+        )
+    else:
+        st.button("🚫 File belum tersedia", disabled=True, use_container_width=True)

create_sqlite_dump.py

@@ -0,0 +1,172 @@
+import sqlite3
+import pandas as pd
+import json
+import numpy as np
+
+def generate_sqlite_dump():
+    # Connect to in-memory SQLite database
+    conn = sqlite3.connect(':memory:')
+    cursor = conn.cursor()
+
+    # 1. Create SQLite Schema for metadata
+    schema = """
+    CREATE TABLE IF NOT EXISTS authors (
+        author_id INTEGER PRIMARY KEY,
+        name_ar TEXT,
+        death_hijri INTEGER,
+        death_hijri_text TEXT,
+        alpha_sort TEXT,
+        biography TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS categories (
+        category_id INTEGER PRIMARY KEY,
+        category_name_ar TEXT,
+        name_en TEXT,
+        sort_order INTEGER
+    );
+
+    CREATE TABLE IF NOT EXISTS books (
+        book_id INTEGER PRIMARY KEY,
+        shamela_id INTEGER,
+        title_ar TEXT,
+        book_type INTEGER,
+        book_type_label TEXT,
+        category_id INTEGER REFERENCES categories(category_id),
+        main_author_id INTEGER REFERENCES authors(author_id),
+        main_author_name_ar TEXT,
+        main_author_death_hijri INTEGER,
+        main_author_death_hijri_text TEXT,
+        authors_text TEXT,
+        hijri_era TEXT,
+        printed BOOLEAN,
+        is_hidden BOOLEAN,
+        parent_id INTEGER,
+        group_id INTEGER,
+        version_major INTEGER,
+        version_minor INTEGER,
+        betaka_text TEXT,
+        meta TEXT,
+        volume_count_observed INTEGER,
+        has_multi_part BOOLEAN,
+        authors_json TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS book_authors (
+        book_id INTEGER REFERENCES books(book_id),
+        author_id INTEGER REFERENCES authors(author_id),
+        role TEXT,
+        name_ar TEXT,
+        death_hijri INTEGER,
+        PRIMARY KEY (book_id, author_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS pages (
+        page_id INTEGER PRIMARY KEY,
+        book_id INTEGER REFERENCES books(book_id),
+        shamela_page_id INTEGER,
+        part TEXT,
+        page_num INTEGER,
+        sequence_num INTEGER,
+        body TEXT,
+        footnotes TEXT,
+        hints TEXT,
+        services_raw TEXT
+    );
+
+    CREATE INDEX IF NOT EXISTS pages_body_idx ON pages (body);
+
+    CREATE TABLE IF NOT EXISTS toc (
+        title_id INTEGER PRIMARY KEY,
+        book_id INTEGER REFERENCES books(book_id),
+        page_id INTEGER REFERENCES pages(page_id),
+        parent_id INTEGER REFERENCES toc(title_id),
+        shamela_title_id INTEGER,
+        title_text TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS quran_verses (
+        verse_id INTEGER PRIMARY KEY,
+        surah_id INTEGER,
+        ayah_id INTEGER,
+        text_ar TEXT,
+        data TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS narrators (
+        narrator_id INTEGER PRIMARY KEY,
+        name_ar TEXT,
+        data TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS root_dictionary (
+        root_id INTEGER PRIMARY KEY,
+        token TEXT,
+        data TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS hadith_xrefs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        book_id INTEGER REFERENCES books(book_id),
+        page_id INTEGER REFERENCES pages(page_id),
+        data TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS tafsir_xrefs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        book_id INTEGER REFERENCES books(book_id),
+        page_id INTEGER REFERENCES pages(page_id),
+        data TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS page_isnads (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        book_id INTEGER REFERENCES books(book_id),
+        page_id INTEGER REFERENCES pages(page_id),
+        narrator_id INTEGER REFERENCES narrators(narrator_id),
+        data TEXT
+    );
+    """
+    cursor.executescript(schema)
+    
+    # 2. Download and insert metadata
+    print("Downloading Categories...")
+    df_cat = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/categories.parquet")
+    if 'id' in df_cat.columns: df_cat.rename(columns={'id': 'category_id'}, inplace=True)
+    if 'name_ar' in df_cat.columns and 'category_name_ar' not in df_cat.columns: df_cat.rename(columns={'name_ar': 'category_name_ar'}, inplace=True)
+    if 'category_name' in df_cat.columns and 'category_name_ar' not in df_cat.columns: df_cat.rename(columns={'category_name': 'category_name_ar'}, inplace=True)
+    cols_cat = ['category_id', 'category_name_ar', 'name_en', 'sort_order']
+    df_cat = df_cat[[c for c in cols_cat if c in df_cat.columns]]
+    df_cat.to_sql("categories", conn, if_exists="append", index=False)
+
+    print("Downloading Authors...")
+    df_auth = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/authors.parquet")
+    if 'id' in df_auth.columns: df_auth.rename(columns={'id': 'author_id'}, inplace=True)
+    cols_auth = ['author_id', 'name_ar', 'death_hijri', 'death_hijri_text', 'alpha_sort', 'biography']
+    df_auth = df_auth[[c for c in cols_auth if c in df_auth.columns]]
+    df_auth.to_sql("authors", conn, if_exists="append", index=False)
+
+    print("Downloading Books Metadata...")
+    df_books = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/book_metadata.parquet")
+    cols_books = ['book_id', 'shamela_id', 'title_ar', 'book_type', 'book_type_label', 'category_id', 'main_author_id', 'main_author_name_ar', 'main_author_death_hijri', 'main_author_death_hijri_text', 'authors_text', 'hijri_era', 'printed', 'is_hidden', 'parent_id', 'group_id', 'version_major', 'version_minor', 'betaka_text', 'meta', 'volume_count_observed', 'has_multi_part', 'authors_json']
+    df_books = df_books[[c for c in cols_books if c in df_books.columns]]
+    if 'meta' in df_books.columns:
+        df_books['meta'] = df_books['meta'].apply(lambda x: json.dumps(x.tolist()) if isinstance(x, np.ndarray) else (json.dumps(x) if isinstance(x, dict) else (x if pd.isna(x) else json.dumps(x))))
+    if 'authors_json' in df_books.columns:
+        df_books['authors_json'] = df_books['authors_json'].apply(lambda x: json.dumps(x.tolist()) if isinstance(x, np.ndarray) else (json.dumps(x) if isinstance(x, dict) else (x if pd.isna(x) else json.dumps(x))))
+    df_books.to_sql("books", conn, if_exists="append", index=False)
+
+    # 3. Dump to .sql file
+    output_filename = 'metadata_dump.sql'
+    print(f"Dumping to {output_filename}...")
+    with open(output_filename, 'w', encoding='utf-8') as f:
+        for line in conn.iterdump():
+            # Include only schemas and INSERTs for metadata tables.
+            # conn.iterdump() does a full backup of the schema and inserts for tables in memory.
+            f.write('%s\n' % line)
+    
+    print(f"Done! The SQL dump is ready at: {output_filename}")
+    conn.close()
+
+if __name__ == "__main__":
+    generate_sqlite_dump()

ingest_worker.py

@@ -0,0 +1,237 @@
+import time
+import json
+import requests
+import pandas as pd
+from sqlalchemy import create_engine, text
+from huggingface_hub import HfFileSystem
+
+DB_URI = "postgresql://postgres:password@localhost:5432/postgres"
+engine = create_engine(DB_URI)
+fs = HfFileSystem()
+
+import io
+import csv
+
+session = requests.Session()
+adapter = requests.adapters.HTTPAdapter(pool_connections=20, pool_maxsize=20)
+session.mount('https://', adapter)
+
+def copy_to_db(table_name, df, engine):
+    """Fast bulk insert using PostgreSQL COPY via psycopg2."""
+    if df.empty:
+        return
+        
+    # Clean data to prevent Postgres COPY errors (carriage returns, null bytes, and float64 floats)
+    for col in df.columns:
+        if str(df[col].dtype) == 'float64':
+            df[col] = df[col].astype('Int64')
+        elif str(df[col].dtype) == 'object':
+            df[col] = df[col].apply(lambda x: x.replace('\r', ' ').replace('\0', '') if isinstance(x, str) else x)
+
+    # Convert string JSON to ensure no weird quoting issues
+    buffer = io.StringIO()
+    df.to_csv(buffer, index=False, header=False, sep='\t', quoting=csv.QUOTE_MINIMAL, na_rep='\\N')
+    buffer.seek(0)
+    
+    # We use psycopg2 raw connection
+    conn = engine.raw_connection()
+    try:
+        with conn.cursor() as cur:
+            columns = ', '.join(df.columns)
+            sql = f"COPY {table_name} ({columns}) FROM STDIN WITH (FORMAT CSV, DELIMITER '\t', NULL '\\N', QUOTE '\"', ESCAPE '\"')"
+            cur.copy_expert(sql, buffer)
+        conn.commit()
+    except Exception as e:
+        conn.rollback()
+        raise e
+    finally:
+        conn.close()
+
+def build_path_mapping(fs):
+    print("Building path mapping from HuggingFace (this might take a few seconds)...")
+    paths = fs.glob("datasets/AuthenticIlm/Shamela4_Full_DB/*/*")
+    mapping = {}
+    for p in paths:
+        parts = p.split('/')
+        if len(parts) >= 4:
+            book_dir = parts[-1]
+            try:
+                book_id_str = book_dir.split('__')[0]
+                if book_id_str.isdigit():
+                    mapping[int(book_id_str)] = f"{parts[-2]}/{parts[-1]}/pages.jsonl"
+            except:
+                pass
+    print(f"Path mapping built for {len(mapping)} books.")
+    return mapping
+
+def process_book(book_id, file_path):
+    if not file_path:
+        return False, "Not found in mapping"
+        
+    # TOC insertion moved to the end of process_book to prevent FK violations
+    
+    url = f"https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/{file_path}"
+    try:
+        with session.get(url, stream=True, timeout=30) as response:
+            if response.status_code != 200:
+                return False, "Download failed"
+        
+            pages = []
+            
+            for line in response.iter_lines(chunk_size=65536):
+                if line:
+                    try:
+                        page_data = json.loads(line)
+                        if 'services_raw' in page_data and page_data['services_raw'] is not None:
+                            page_data['services_raw'] = json.dumps(page_data['services_raw'])
+                        
+                        pages.append(page_data)
+                        
+                        if len(pages) >= 5000:
+                            pages_df = pd.DataFrame(pages)
+                            cols = ['page_id', 'book_id', 'shamela_page_id', 'part', 'page_num', 'sequence_num', 'body', 'footnotes', 'hints', 'services_raw']
+                            pages_df = pages_df[[c for c in cols if c in pages_df.columns]]
+                            copy_to_db('pages', pages_df, engine)
+                            pages = []
+                    except Exception as e:
+                        pass
+                        
+            if len(pages) > 0:
+                try:
+                    pages_df = pd.DataFrame(pages)
+                    cols = ['page_id', 'book_id', 'shamela_page_id', 'part', 'page_num', 'sequence_num', 'body', 'footnotes', 'hints', 'services_raw']
+                    pages_df = pages_df[[c for c in cols if c in pages_df.columns]]
+                    copy_to_db('pages', pages_df, engine)
+                except Exception as e:
+                    print(f"Error saving final pages for book {book_id}: {e}")
+
+            # Now that pages are inserted, we can safely insert TOC which depends on pages(page_id)
+            toc_file_path = file_path.replace("pages.jsonl", "toc.jsonl")
+            toc_url = f"https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/{toc_file_path}"
+            toc_entries = []
+            try:
+                with session.get(toc_url, stream=True, timeout=30) as toc_response:
+                    if toc_response.status_code == 200:
+                        for line in toc_response.iter_lines(chunk_size=65536):
+                            if line:
+                                try:
+                                    toc_entries.append(json.loads(line))
+                                except Exception as e:
+                                    pass
+            except Exception as e:
+                print(f"Error downloading TOC for {book_id}: {e}")
+                
+            if len(toc_entries) > 0:
+                try:
+                    # Ensure correct columns mapping if json has extra fields
+                    toc_df = pd.DataFrame(toc_entries)
+                    expected_cols = ['title_id', 'book_id', 'page_id', 'parent_id', 'shamela_title_id', 'title_text']
+                    toc_df = toc_df[[c for c in expected_cols if c in toc_df.columns]]
+                    
+                    # Fix Shamela edge-case where page_id=0 for general headings, which violates FK constraints
+                    if 'page_id' in toc_df.columns:
+                        toc_df.loc[toc_df['page_id'] == 0, 'page_id'] = None
+                        
+                    copy_to_db('toc', toc_df, engine)
+                except Exception as e:
+                    print(f"Error saving TOC for book {book_id}: {e}")
+                
+            return True, "Success"
+    except Exception as e:
+        return False, str(e)
+
+def process_wrapper(args):
+    book_id, file_path = args
+    print(f"Processing book {book_id}...")
+    success, msg = process_book(book_id, file_path)
+    if success:
+        print(f"Book {book_id} processed successfully.")
+    else:
+        print(f"Failed to process book {book_id}: {msg}")
+
+def load_metadata():
+    import numpy as np
+    print("Downloading Categories...")
+    df = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/categories.parquet")
+    if 'id' in df.columns: df.rename(columns={'id': 'category_id'}, inplace=True)
+    if 'name_ar' in df.columns and 'category_name_ar' not in df.columns: df.rename(columns={'name_ar': 'category_name_ar'}, inplace=True)
+    if 'category_name' in df.columns and 'category_name_ar' not in df.columns: df.rename(columns={'category_name': 'category_name_ar'}, inplace=True)
+    cols = ['category_id', 'category_name_ar', 'name_en', 'sort_order']
+    df[[c for c in cols if c in df.columns]].to_sql("categories", engine, if_exists="append", index=False)
+    
+    print("Downloading Authors...")
+    df = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/authors.parquet")
+    if 'id' in df.columns: df.rename(columns={'id': 'author_id'}, inplace=True)
+    cols = ['author_id', 'name_ar', 'death_hijri', 'death_hijri_text', 'alpha_sort', 'biography']
+    df[[c for c in cols if c in df.columns]].to_sql("authors", engine, if_exists="append", index=False)
+    
+    print("Downloading Books Metadata...")
+    df = pd.read_parquet("https://huggingface.co/datasets/AuthenticIlm/Shamela4_Full_DB/resolve/main/_meta/book_metadata.parquet")
+    cols = ['book_id', 'shamela_id', 'title_ar', 'book_type', 'book_type_label', 'category_id', 'main_author_id', 'main_author_name_ar', 'main_author_death_hijri', 'main_author_death_hijri_text', 'authors_text', 'hijri_era', 'printed', 'is_hidden', 'parent_id', 'group_id', 'version_major', 'version_minor', 'betaka_text', 'meta', 'volume_count_observed', 'has_multi_part', 'authors_json']
+    df = df[[c for c in cols if c in df.columns]]
+    if 'meta' in df.columns:
+        df['meta'] = df['meta'].apply(lambda x: json.dumps(x.tolist()) if isinstance(x, np.ndarray) else (json.dumps(x) if isinstance(x, dict) else (x if pd.isna(x) else json.dumps(x))))
+    if 'authors_json' in df.columns:
+        df['authors_json'] = df['authors_json'].apply(lambda x: json.dumps(x.tolist()) if isinstance(x, np.ndarray) else (json.dumps(x) if isinstance(x, dict) else (x if pd.isna(x) else json.dumps(x))))
+    df.to_sql("books", engine, if_exists="append", index=False)
+    print("Metadata loading complete!")
+
+def run_worker():
+    print("Ingest Worker Started...")
+    from concurrent.futures import ThreadPoolExecutor
+    path_mapping = None
+    
+    while True:
+        try:
+            # Get list of all books
+            df_books = pd.read_sql("SELECT book_id FROM books", engine)
+            if df_books.empty:
+                print("No books found in DB. Auto-loading metadata...")
+                try:
+                    load_metadata()
+                except Exception as e:
+                    print(f"Failed to load metadata: {e}")
+                    time.sleep(60)
+                continue
+                
+            if path_mapping is None:
+                path_mapping = build_path_mapping(fs)
+                
+            all_books = set(df_books['book_id'].tolist())
+            
+            # Get list of processed books
+            df_processed = pd.read_sql("SELECT DISTINCT book_id FROM pages", engine)
+            processed_books = set(df_processed['book_id'].tolist())
+            
+            pending_books = list(all_books - processed_books)
+            print(f"Total books: {len(all_books)}, Processed: {len(processed_books)}, Pending: {len(pending_books)}")
+            
+            if not pending_books:
+                print("All books processed. Creating full-text index if it doesn't exist...")
+                with engine.begin() as conn:
+                    conn.execute(text("CREATE INDEX IF NOT EXISTS pages_body_idx ON pages USING GIN (to_tsvector('arabic', body));"))
+                print("Index created. Sleeping for 1 hour...")
+                time.sleep(3600)
+                continue
+                
+            # If we have pending books, drop the index for faster ingestion
+            with engine.begin() as conn:
+                conn.execute(text("DROP INDEX IF EXISTS pages_body_idx;"))
+                
+            # Prepare arguments for multiprocessing
+            tasks = [(b, path_mapping.get(b)) for b in pending_books]
+            
+            # Process books concurrently with ThreadPoolExecutor
+            with ThreadPoolExecutor(max_workers=5) as executor:
+                executor.map(process_wrapper, tasks)
+                
+            print("Batch finished. Checking again...")
+            time.sleep(5)
+                
+        except Exception as e:
+            print(f"Worker Error: {e}")
+            time.sleep(60)
+
+if __name__ == "__main__":
+    time.sleep(10)
+    run_worker()

init.sql

@@ -0,0 +1,128 @@
+CREATE EXTENSION IF NOT EXISTS ai CASCADE;
+
+-- 1. Authors
+CREATE TABLE IF NOT EXISTS authors (
+    author_id INT PRIMARY KEY,
+    name_ar TEXT,
+    death_hijri INT,
+    death_hijri_text TEXT,
+    alpha_sort TEXT,
+    biography TEXT
+);
+
+-- 2. Categories
+CREATE TABLE IF NOT EXISTS categories (
+    category_id INT PRIMARY KEY,
+    category_name_ar TEXT,
+    name_en TEXT,
+    sort_order INT
+);
+
+-- 3. Books
+CREATE TABLE IF NOT EXISTS books (
+    book_id INT PRIMARY KEY,
+    shamela_id INT,
+    title_ar TEXT,
+    book_type INT,
+    book_type_label TEXT,
+    category_id INT REFERENCES categories(category_id),
+    main_author_id INT REFERENCES authors(author_id),
+    main_author_name_ar TEXT,
+    main_author_death_hijri INT,
+    main_author_death_hijri_text TEXT,
+    authors_text TEXT,
+    hijri_era TEXT,
+    printed BOOLEAN,
+    is_hidden BOOLEAN,
+    parent_id INT,
+    group_id INT,
+    version_major INT,
+    version_minor INT,
+    betaka_text TEXT,
+    meta JSONB,
+    volume_count_observed INT,
+    has_multi_part BOOLEAN,
+    authors_json JSONB
+);
+
+-- Book Authors (Many-to-Many relation between books and authors)
+CREATE TABLE IF NOT EXISTS book_authors (
+    book_id INT REFERENCES books(book_id),
+    author_id INT REFERENCES authors(author_id),
+    role TEXT,
+    name_ar TEXT,
+    death_hijri INT,
+    PRIMARY KEY (book_id, author_id)
+);
+
+-- 4. Pages
+CREATE TABLE IF NOT EXISTS pages (
+    page_id BIGINT PRIMARY KEY,
+    book_id INT REFERENCES books(book_id),
+    shamela_page_id INT,
+    part TEXT,
+    page_num INT,
+    sequence_num INT,
+    body TEXT,
+    footnotes TEXT,
+    hints TEXT,
+    services_raw JSONB
+);
+
+-- Index for Full-Text Search on Arabic text
+CREATE INDEX IF NOT EXISTS pages_body_idx ON pages USING GIN (to_tsvector('arabic', body));
+
+-- 5. Table of Contents (TOC)
+CREATE TABLE IF NOT EXISTS toc (
+    title_id BIGINT PRIMARY KEY,
+    book_id INT REFERENCES books(book_id),
+    page_id BIGINT REFERENCES pages(page_id),
+    parent_id BIGINT REFERENCES toc(title_id),
+    shamela_title_id INT,
+    title_text TEXT
+);
+
+-- 6. Other Metadata Tables (Narrators, Quran, Roots, Xrefs)
+-- We use JSONB for flexible schema mapping until exact column definitions are parsed
+
+CREATE TABLE IF NOT EXISTS quran_verses (
+    verse_id INT PRIMARY KEY,
+    surah_id INT,
+    ayah_id INT,
+    text_ar TEXT,
+    data JSONB
+);
+
+CREATE TABLE IF NOT EXISTS narrators (
+    narrator_id INT PRIMARY KEY,
+    name_ar TEXT,
+    data JSONB
+);
+
+CREATE TABLE IF NOT EXISTS root_dictionary (
+    root_id INT PRIMARY KEY,
+    token TEXT,
+    data JSONB
+);
+
+CREATE TABLE IF NOT EXISTS hadith_xrefs (
+    id SERIAL PRIMARY KEY,
+    book_id INT REFERENCES books(book_id),
+    page_id BIGINT REFERENCES pages(page_id),
+    data JSONB
+);
+
+CREATE TABLE IF NOT EXISTS tafsir_xrefs (
+    id SERIAL PRIMARY KEY,
+    book_id INT REFERENCES books(book_id),
+    page_id BIGINT REFERENCES pages(page_id),
+    data JSONB
+);
+
+CREATE TABLE IF NOT EXISTS page_isnads (
+    id SERIAL PRIMARY KEY,
+    book_id INT REFERENCES books(book_id),
+    page_id BIGINT REFERENCES pages(page_id),
+    narrator_id INT REFERENCES narrators(narrator_id),
+    data JSONB
+);

mcp_server.py

@@ -0,0 +1,110 @@
+import os
+import uvicorn
+from mcp.server.fastmcp import FastMCP
+from sqlalchemy import create_engine, text
+
+# 1. Database Connection (Optimized for 2 CPU / 16GB RAM)
+DB_URI = os.getenv("DB_URI", "postgresql://postgres:password@localhost:5432/postgres")
+# Pool size 5 prevents too many context switches on 2 vCPUs
+engine = create_engine(DB_URI, pool_size=5, max_overflow=10)
+
+# 2. Initialize FastMCP
+mcp = FastMCP("Shamela4")
+
+def fetch_data(query: str, params: dict = {}):
+    try:
+        with engine.connect() as conn:
+            result = conn.execute(text(query), params)
+            return [dict(row._mapping) for row in result]
+    except Exception as e:
+        return {"error": str(e)}
+
+def fetch_one(query: str, params: dict = {}):
+    try:
+        with engine.connect() as conn:
+            result = conn.execute(text(query), params).fetchone()
+            if result:
+                return dict(result._mapping)
+            return None
+    except Exception as e:
+        return {"error": str(e)}
+
+@mcp.tool()
+def get_database_stats() -> dict:
+    """Get database statistics: counts of books, authors, categories, and pages."""
+    try:
+        with engine.connect() as conn:
+            books_count = conn.execute(text("SELECT COUNT(book_id) FROM books")).scalar() or 0
+            authors_count = conn.execute(text("SELECT COUNT(author_id) FROM authors")).scalar() or 0
+            categories_count = conn.execute(text("SELECT COUNT(category_id) FROM categories")).scalar() or 0
+            pages_count = conn.execute(text("SELECT COUNT(page_id) FROM pages")).scalar() or 0
+            return {
+                "total_categories": categories_count,
+                "total_authors": authors_count,
+                "total_books": books_count,
+                "total_pages": pages_count,
+            }
+    except Exception as e:
+        return {"error": str(e)}
+
+@mcp.tool()
+def search_categories(limit: int = 50) -> list:
+    """Get list of categories in the library."""
+    limit = min(limit, 100) # Hard limit
+    return fetch_data("SELECT category_id, category_name_ar, name_en FROM categories ORDER BY sort_order LIMIT :limit", {"limit": limit})
+
+@mcp.tool()
+def search_authors(search_term: str, limit: int = 10) -> list:
+    """Search for authors by their Arabic name."""
+    limit = min(limit, 20)
+    query = "SELECT author_id, name_ar, death_hijri FROM authors WHERE name_ar LIKE :search ORDER BY death_hijri LIMIT :limit"
+    return fetch_data(query, {"search": f"%{search_term}%", "limit": limit})
+
+@mcp.tool()
+def search_books(title_or_author: str, limit: int = 10) -> list:
+    """Search for books by title or author name."""
+    limit = min(limit, 20)
+    query = """
+        SELECT book_id, title_ar, main_author_name_ar, main_author_death_hijri 
+        FROM books 
+        WHERE title_ar LIKE :search OR main_author_name_ar LIKE :search 
+        ORDER BY book_id LIMIT :limit
+    """
+    return fetch_data(query, {"search": f"%{title_or_author}%", "limit": limit})
+
+@mcp.tool()
+def get_book_toc(book_id: int) -> list:
+    """Get the Table of Contents (Daftar Isi) for a specific book_id."""
+    query = "SELECT title_id, title_text, page_id, parent_id FROM toc WHERE book_id = :book_id ORDER BY title_id"
+    return fetch_data(query, {"book_id": book_id})
+
+@mcp.tool()
+def search_text_in_books(query: str, limit: int = 5) -> list:
+    """
+    Full-text search inside book pages. 
+    Use this to find specific Arabic text/words inside the books.
+    Returns highly relevant text snippets (highlights).
+    """
+    limit = min(limit, 10) # Strict limit for efficiency on 16GB RAM
+    
+    # Use websearch_to_tsquery for efficient and safe natural language parsing
+    sql = """
+        SELECT p.book_id, p.page_num, p.sequence_num, b.title_ar as book_title,
+               ts_headline('arabic', p.body, websearch_to_tsquery('arabic', :query)) as snippet
+        FROM pages p
+        JOIN books b ON p.book_id = b.book_id
+        WHERE to_tsvector('arabic', p.body) @@ websearch_to_tsquery('arabic', :query)
+        ORDER BY p.book_id, p.sequence_num
+        LIMIT :limit
+    """
+    return fetch_data(sql, {"query": query, "limit": limit})
+
+@mcp.tool()
+def get_page_content(book_id: int, page_num: int) -> dict:
+    """Retrieve the exact, full text of a specific page number in a book."""
+    query = "SELECT page_num, body, footnotes FROM pages WHERE book_id = :book_id AND page_num = :page_num LIMIT 1"
+    return fetch_one(query, {"book_id": book_id, "page_num": page_num})
+
+if __name__ == "__main__":
+    print("Starting Shamela4 FastMCP Server with SSE on port 8001...")
+    mcp.run(transport="sse", host="0.0.0.0", port=8001)

nginx.conf

@@ -0,0 +1,79 @@
+worker_processes 1;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+    
+    client_body_temp_path /tmp/client_body;
+    proxy_temp_path /tmp/proxy_temp;
+    fastcgi_temp_path /tmp/fastcgi_temp;
+    uwsgi_temp_path /tmp/uwsgi_temp;
+    scgi_temp_path /tmp/scgi_temp;
+    access_log /tmp/access.log;
+    error_log /tmp/error.log;
+
+    sendfile        on;
+    keepalive_timeout  65;
+
+    server {
+        listen 7860;
+        server_name localhost;
+
+        # Route /api and /docs to FastAPI (Port 8000)
+        location /api {
+            proxy_pass http://127.0.0.1:8000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /docs {
+            proxy_pass http://127.0.0.1:8000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        location /openapi.json {
+            proxy_pass http://127.0.0.1:8000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        # MCP SSE Endpoints
+        location /sse {
+            proxy_pass http://127.0.0.1:8001;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            
+            # SSE specific headers to keep connection alive
+            proxy_set_header Connection '';
+            proxy_http_version 1.1;
+            chunked_transfer_encoding off;
+            proxy_buffering off;
+            proxy_cache off;
+            proxy_read_timeout 24h;
+        }
+
+        location /messages {
+            proxy_pass http://127.0.0.1:8001;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+        }
+
+        # Route everything else to Streamlit (Port 8501)
+        location / {
+            proxy_pass http://127.0.0.1:8501;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            
+            # WebSocket support for Streamlit
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+        }
+    }
+}

requirements.txt

@@ -0,0 +1,12 @@
+streamlit
+psycopg2-binary
+sqlalchemy
+datasets
+polars
+pandas
+pyarrow
+huggingface-hub
+fastapi
+uvicorn
+beautifulsoup4
+mcp

run.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+# Run standard postgres entrypoint in background
+# This will initialize the DB (running init.sql) if it's the first time
+echo "Starting PostgreSQL in background..."
+bash /docker-entrypoint.sh postgres &
+
+# Wait for PostgreSQL to become available
+echo "Waiting for PostgreSQL to start..."
+until pg_isready -h localhost -p 5432 -U postgres; do
+  sleep 2
+done
+echo "PostgreSQL is ready!"
+
+# Start the Streamlit application in the foreground
+echo "Starting Background Ingest Worker..."
+python3 -u ingest_worker.py > /tmp/ingest.log 2>&1 &
+
+echo "Starting FastAPI Server..."
+uvicorn api:app --host 0.0.0.0 --port 8000 &
+
+echo "Starting Streamlit..."
+streamlit run app.py --server.port 8501 --server.address 0.0.0.0 &
+
+echo "Starting MCP SSE Server..."
+python3 -u mcp_server.py > /tmp/mcp.log 2>&1 &
+
+echo "Starting Nginx Proxy..."
+nginx -g "daemon off;"