Spaces:
Running
Running
Commit ·
603ead8
1
Parent(s): 5c4239f
feat: complete turbovec search backend integration, ignore tvim file in git, and stabilize mobile UI transitions
Browse files- .gitignore +4 -0
- webapp/tipitaka-api/app/services/rag_service.py +161 -250
- webapp/tipitaka-api/app/services/search_service.py +52 -150
- webapp/tipitaka-api/benchmark_qdrant_vs_turbovec.py +117 -0
- webapp/tipitaka-api/build_turbovec_index.py +139 -0
- webapp/tipitaka-api/download_assets.py +17 -3
- webapp/tipitaka-api/requirements.txt +2 -0
- webapp/tipitaka-api/test_rag_query.py +1 -1
- webapp/tipitaka-api/tests/test_rag_service.py +7 -6
- webapp/tipitaka-web/src/components/layout/ReaderPanel.tsx +94 -92
.gitignore
CHANGED
|
@@ -171,3 +171,7 @@ webapp/.serena/
|
|
| 171 |
embedding_cache.db
|
| 172 |
source_texts.db
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
embedding_cache.db
|
| 172 |
source_texts.db
|
| 173 |
|
| 174 |
+
# ── Turbovec local data ──
|
| 175 |
+
webapp/tipitaka-api/data/
|
| 176 |
+
*.tvim
|
| 177 |
+
|
webapp/tipitaka-api/app/services/rag_service.py
CHANGED
|
@@ -2,8 +2,6 @@ import logging
|
|
| 2 |
import os
|
| 3 |
from pathlib import Path
|
| 4 |
from collections import OrderedDict
|
| 5 |
-
import qdrant_client
|
| 6 |
-
from qdrant_client.http import models as qmodels
|
| 7 |
import httpx
|
| 8 |
from app.config import get_settings
|
| 9 |
from app.database.sqlite_db import get_db
|
|
@@ -22,178 +20,105 @@ ST_EMBED_MODEL = "jinaai/jina-embeddings-v5-text-small-retrieval"
|
|
| 22 |
|
| 23 |
logger = logging.getLogger(__name__)
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
class RAGService:
|
| 26 |
def __init__(self):
|
| 27 |
settings = get_settings()
|
| 28 |
-
self.qdrant_path = settings.QDRANT_PATH
|
| 29 |
-
self.snapshot_dir = Path(settings.SNAPSHOT_DIR)
|
| 30 |
self.ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
self.model = None # Flag: None = not verified, 'ready' = OK
|
| 34 |
self.reranker = None
|
| 35 |
-
self.
|
| 36 |
-
self.
|
| 37 |
-
self.
|
| 38 |
-
self.
|
| 39 |
-
self._st_model = None # sentence-transformers model (fallback)
|
| 40 |
|
| 41 |
-
#
|
| 42 |
-
|
|
|
|
| 43 |
|
| 44 |
-
|
| 45 |
self._load_model()
|
| 46 |
self._load_reranker()
|
| 47 |
|
| 48 |
-
|
| 49 |
-
"""Heavy initialization: detect mode, extract snapshots, create Qdrant client."""
|
| 50 |
-
settings = get_settings()
|
| 51 |
-
qdrant_url = getattr(settings, "QDRANT_URL", None)
|
| 52 |
-
|
| 53 |
-
# Auto-detect Qdrant server if not configured
|
| 54 |
-
if not qdrant_url:
|
| 55 |
-
try:
|
| 56 |
-
with httpx.Client() as client:
|
| 57 |
-
response = client.get("http://localhost:6333/healthz", timeout=1.0)
|
| 58 |
-
if response.status_code == 200:
|
| 59 |
-
qdrant_url = "http://localhost:6333"
|
| 60 |
-
logger.info(f"Auto-detected running Qdrant Server at {qdrant_url}")
|
| 61 |
-
except Exception:
|
| 62 |
-
pass
|
| 63 |
-
|
| 64 |
-
is_local = not bool(qdrant_url)
|
| 65 |
-
|
| 66 |
-
try:
|
| 67 |
-
if is_local:
|
| 68 |
-
self._init_qdrant_local()
|
| 69 |
-
else:
|
| 70 |
-
self._init_qdrant_server(qdrant_url)
|
| 71 |
-
except Exception as e:
|
| 72 |
-
logger.error(f"RAG Initialization Error: {e}")
|
| 73 |
-
self.client = None
|
| 74 |
-
|
| 75 |
-
def _init_qdrant_local(self):
|
| 76 |
-
"""Initialize Qdrant in Local Mode."""
|
| 77 |
-
logger.info(f"Initializing Qdrant in Local Mode at {self.qdrant_path}")
|
| 78 |
-
storage_path = Path(self.qdrant_path)
|
| 79 |
-
storage_path.mkdir(parents=True, exist_ok=True)
|
| 80 |
-
|
| 81 |
-
lock_file = storage_path / ".lock"
|
| 82 |
-
if lock_file.exists():
|
| 83 |
-
try:
|
| 84 |
-
logger.warning(f"Removing stale Qdrant lock file: {lock_file}")
|
| 85 |
-
lock_file.unlink()
|
| 86 |
-
except Exception as e:
|
| 87 |
-
logger.error(f"Failed to remove lock file: {e}")
|
| 88 |
-
|
| 89 |
-
self.client = qdrant_client.QdrantClient(path=str(self.qdrant_path))
|
| 90 |
-
logger.info("Qdrant Local Client initialized.")
|
| 91 |
-
|
| 92 |
-
def _init_qdrant_server(self, qdrant_url: str):
|
| 93 |
-
"""Initialize Qdrant in Server Mode — connect and restore snapshots if needed."""
|
| 94 |
-
logger.info(f"Connecting to Qdrant Server at {qdrant_url}")
|
| 95 |
-
self.client = qdrant_client.QdrantClient(url=qdrant_url)
|
| 96 |
|
|
|
|
|
|
|
| 97 |
try:
|
| 98 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
except Exception as e:
|
| 100 |
-
logger.error(f"Failed to
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
settings = get_settings()
|
| 104 |
-
for col_name in self.collections:
|
| 105 |
-
actual_col = None
|
| 106 |
-
if col_name in all_cols:
|
| 107 |
-
actual_col = col_name
|
| 108 |
-
else:
|
| 109 |
-
matches = [c for c in all_cols if c.startswith(f"{col_name}_") or c.startswith(col_name)]
|
| 110 |
-
if matches:
|
| 111 |
-
actual_col = matches[0]
|
| 112 |
|
| 113 |
-
|
| 114 |
-
if col_name == settings.QDRANT_COLLECTION:
|
| 115 |
-
self.actual_chunks_col = actual_col
|
| 116 |
-
else:
|
| 117 |
-
snap_filename = f"{col_name}.snapshot"
|
| 118 |
-
target_snap = ALLOWED_SNAP_ROOT / snap_filename
|
| 119 |
-
if not target_snap.exists():
|
| 120 |
-
target_snap = self.snapshot_dir / snap_filename
|
| 121 |
-
|
| 122 |
-
if target_snap.exists():
|
| 123 |
-
import os
|
| 124 |
-
logger.info(f"Restoring server collection '{col_name}' from {target_snap}...")
|
| 125 |
-
abs_snap_path = os.path.abspath(target_snap).replace("\\", "/")
|
| 126 |
-
if not abs_snap_path.startswith("/"):
|
| 127 |
-
abs_snap_path = "/" + abs_snap_path
|
| 128 |
-
try:
|
| 129 |
-
self.client.recover_snapshot(col_name, location=f"file://{abs_snap_path}")
|
| 130 |
-
if col_name == settings.QDRANT_COLLECTION:
|
| 131 |
-
self.actual_chunks_col = col_name
|
| 132 |
-
except Exception as e:
|
| 133 |
-
logger.error(f"Failed to restore: {e}")
|
| 134 |
|
| 135 |
def _load_model(self):
|
| 136 |
-
"""Verify Ollama
|
| 137 |
if self.model is None:
|
| 138 |
-
logger.info(
|
| 139 |
try:
|
| 140 |
r = httpx.get(f"{self.ollama_url}/api/tags", timeout=5)
|
| 141 |
r.raise_for_status()
|
| 142 |
self.model = "ready"
|
| 143 |
-
logger.info(
|
| 144 |
except Exception as e:
|
| 145 |
-
logger.error(f"Ollama not accessible: {e}. Pre-loading sentence-transformers
|
| 146 |
self.model = "fallback"
|
| 147 |
try:
|
| 148 |
settings = get_settings()
|
| 149 |
-
logger.info(f"Loading sentence-transformers model: {settings.ST_EMBED_MODEL}")
|
| 150 |
from sentence_transformers import SentenceTransformer
|
| 151 |
self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
|
| 152 |
-
logger.info("Sentence-transformers model loaded
|
| 153 |
except Exception as st_err:
|
| 154 |
-
logger.error(f"Failed to load sentence-transformers
|
| 155 |
self.model = None
|
| 156 |
|
| 157 |
def _load_reranker(self):
|
| 158 |
-
"""Pre-load
|
| 159 |
if self.reranker is None:
|
| 160 |
logger.info(f"Pre-loading ONNX Reranker from {RERANK_MODEL_PATH}...")
|
| 161 |
try:
|
| 162 |
from app.services.onnx_reranker import ONNXReranker
|
| 163 |
self.reranker = ONNXReranker(RERANK_MODEL_PATH)
|
| 164 |
-
logger.info("ONNX Reranker
|
| 165 |
except Exception as e:
|
| 166 |
logger.error(f"Failed to pre-load ONNX Reranker: {e}")
|
| 167 |
-
# Fallback flag
|
| 168 |
self.reranker = "error"
|
| 169 |
|
| 170 |
def _get_embedding(self, text: str) -> list:
|
| 171 |
-
"""Get embedding vector — wrapper around batched implementation."""
|
| 172 |
res = self._get_embeddings([text])
|
| 173 |
return res[0] if res else []
|
| 174 |
|
| 175 |
def _get_embeddings(self, texts: list[str]) -> list[list[float]]:
|
| 176 |
-
"""Get embedding vectors for a list of texts using batching."""
|
| 177 |
if not texts:
|
| 178 |
return []
|
| 179 |
|
| 180 |
results = [None] * len(texts)
|
| 181 |
-
missing_indices = []
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
# 1. Check LRU cache first
|
| 185 |
for idx, text in enumerate(texts):
|
| 186 |
if text in self._embed_cache:
|
| 187 |
-
self._embed_cache.move_to_end(text)
|
| 188 |
results[idx] = self._embed_cache[text]
|
| 189 |
else:
|
| 190 |
missing_indices.append(idx)
|
| 191 |
missing_texts.append(text)
|
| 192 |
-
|
| 193 |
if not missing_texts:
|
| 194 |
return results
|
| 195 |
|
| 196 |
-
# 2. Try Ollama batch embedding (only if Ollama is ready)
|
| 197 |
ollama_failed = True
|
| 198 |
if self.model == "ready":
|
| 199 |
try:
|
|
@@ -209,199 +134,185 @@ class RAGService:
|
|
| 209 |
results[idx] = emb
|
| 210 |
ollama_failed = False
|
| 211 |
except Exception as e:
|
| 212 |
-
logger.info(f"Ollama batch embedding unavailable ({e}), falling back to
|
| 213 |
-
ollama_failed = True
|
| 214 |
self.model = "fallback"
|
| 215 |
|
| 216 |
-
# 3. Fallback to sentence-transformers batch embedding
|
| 217 |
if ollama_failed:
|
| 218 |
try:
|
| 219 |
if self._st_model is None:
|
| 220 |
settings = get_settings()
|
| 221 |
-
logger.info(f"Loading sentence-transformers model: {settings.ST_EMBED_MODEL}")
|
| 222 |
from sentence_transformers import SentenceTransformer
|
| 223 |
self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
|
| 224 |
|
| 225 |
-
embeddings = self._st_model.encode(
|
|
|
|
|
|
|
| 226 |
for idx, emb in zip(missing_indices, embeddings):
|
| 227 |
self._store_cache(texts[idx], emb)
|
| 228 |
results[idx] = emb
|
| 229 |
return results
|
| 230 |
except Exception as e:
|
| 231 |
-
logger.error(f"
|
| 232 |
for idx in missing_indices:
|
| 233 |
results[idx] = []
|
| 234 |
return results
|
| 235 |
-
return results
|
| 236 |
|
|
|
|
| 237 |
|
| 238 |
def _store_cache(self, text: str, embedding: list):
|
| 239 |
-
"""Store embedding in LRU cache."""
|
| 240 |
self._embed_cache[text] = embedding
|
| 241 |
if len(self._embed_cache) > self._embed_cache_max:
|
| 242 |
-
self._embed_cache.popitem(last=False)
|
|
|
|
|
|
|
| 243 |
|
| 244 |
async def query(self, text: str, n_results: int = 10, threshold: float = 0.2) -> str:
|
| 245 |
"""
|
| 246 |
-
Hybrid Search
|
| 247 |
-
1. FTS5
|
| 248 |
-
2.
|
| 249 |
-
3. Merge & Deduplicate
|
| 250 |
-
4. Rerank via Jina v2 ONNX
|
| 251 |
"""
|
| 252 |
-
if
|
| 253 |
return ""
|
| 254 |
-
|
| 255 |
-
# Ensure models are ready
|
| 256 |
if self.model is None:
|
| 257 |
await anyio.to_thread.run_sync(self._load_model)
|
| 258 |
if self.reranker is None:
|
| 259 |
await anyio.to_thread.run_sync(self._load_reranker)
|
| 260 |
|
| 261 |
candidates = []
|
| 262 |
-
seen_keys = set()
|
| 263 |
|
| 264 |
try:
|
| 265 |
-
#
|
| 266 |
-
logger.info(f"
|
|
|
|
| 267 |
def _blocking_fts():
|
| 268 |
-
fts_results = []
|
| 269 |
db = get_db()
|
| 270 |
with db.get_connection() as conn:
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
WHERE pages_fts MATCH ?
|
| 279 |
-
LIMIT 30
|
| 280 |
-
)
|
| 281 |
-
"""
|
| 282 |
-
# Sanitize FTS query: wrap in quotes for literal or keep simple
|
| 283 |
-
sanitized_query = text.replace('"', '').strip()
|
| 284 |
-
if not sanitized_query: return []
|
| 285 |
-
|
| 286 |
-
# Split into tokens by whitespace and append * for prefix matching
|
| 287 |
-
tokens = [t.strip() for t in sanitized_query.split() if t.strip()]
|
| 288 |
-
processed_tokens = []
|
| 289 |
-
for t in tokens:
|
| 290 |
-
if "*" not in t:
|
| 291 |
-
processed_tokens.append(f'"{t}"*')
|
| 292 |
-
else:
|
| 293 |
-
processed_tokens.append(t)
|
| 294 |
-
fts_query = " AND ".join(processed_tokens)
|
| 295 |
-
|
| 296 |
try:
|
| 297 |
-
cursor = conn.execute(
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
"payload": {
|
| 304 |
-
"volume":
|
| 305 |
-
"page":
|
| 306 |
-
"content":
|
| 307 |
},
|
| 308 |
-
"key":
|
| 309 |
-
}
|
|
|
|
|
|
|
| 310 |
except Exception as e:
|
| 311 |
-
logger.warning(f"FTS5
|
| 312 |
-
|
| 313 |
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
if cand['key'] not in seen_keys:
|
| 317 |
-
candidates.append(cand)
|
| 318 |
-
seen_keys.add(cand['key'])
|
| 319 |
-
|
| 320 |
-
# --- PHASE 2: Vector Search (Qdrant) ---
|
| 321 |
-
logger.info(f"Starting Vector search for: {text[:30]}...")
|
| 322 |
-
def _blocking_vector():
|
| 323 |
-
query_vector = self._get_embedding(text)
|
| 324 |
-
if hasattr(self.client, "search"):
|
| 325 |
-
hits = self.client.search(
|
| 326 |
-
collection_name=self.actual_chunks_col,
|
| 327 |
-
query_vector=("dense", query_vector),
|
| 328 |
-
limit=30,
|
| 329 |
-
with_payload=True,
|
| 330 |
-
score_threshold=threshold
|
| 331 |
-
)
|
| 332 |
-
else:
|
| 333 |
-
response = self.client.query_points(
|
| 334 |
-
collection_name=self.actual_chunks_col,
|
| 335 |
-
query=query_vector,
|
| 336 |
-
using="dense",
|
| 337 |
-
limit=30,
|
| 338 |
-
with_payload=True,
|
| 339 |
-
score_threshold=threshold
|
| 340 |
-
)
|
| 341 |
-
hits = response.points
|
| 342 |
-
|
| 343 |
-
vec_results = []
|
| 344 |
-
for hit in hits:
|
| 345 |
-
vol = hit.payload.get("volume", hit.payload.get("volume_id"))
|
| 346 |
-
page = hit.payload.get("page", hit.payload.get("page_number"))
|
| 347 |
-
key = f"{vol}_{page}"
|
| 348 |
-
vec_results.append({
|
| 349 |
-
"id": hit.id,
|
| 350 |
-
"score": hit.score,
|
| 351 |
-
"payload": hit.payload,
|
| 352 |
-
"key": key
|
| 353 |
-
})
|
| 354 |
-
return vec_results
|
| 355 |
-
|
| 356 |
-
vector_candidates = await anyio.to_thread.run_sync(_blocking_vector)
|
| 357 |
-
for cand in vector_candidates:
|
| 358 |
-
if cand['key'] not in seen_keys:
|
| 359 |
candidates.append(cand)
|
| 360 |
-
seen_keys.add(cand[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
|
| 362 |
if not candidates:
|
| 363 |
return ""
|
| 364 |
|
| 365 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
if self.reranker and self.reranker != "error":
|
| 367 |
-
logger.info(f"Reranking {len(candidates)}
|
| 368 |
-
|
| 369 |
def _blocking_rerank():
|
| 370 |
-
|
| 371 |
-
passages = [c['payload'].get("content", "")[:1000] for c in candidates]
|
| 372 |
pairs = [[text, p] for p in passages]
|
| 373 |
-
|
| 374 |
with torch.no_grad():
|
| 375 |
scores = self.reranker.predict(pairs, show_progress_bar=False, batch_size=4)
|
| 376 |
-
|
| 377 |
for i, cand in enumerate(candidates):
|
| 378 |
-
cand[
|
| 379 |
-
|
| 380 |
-
return sorted(candidates, key=lambda x: x.get('rerank_score', 0), reverse=True)
|
| 381 |
|
| 382 |
candidates = await anyio.to_thread.run_sync(_blocking_rerank)
|
| 383 |
-
|
| 384 |
-
#
|
| 385 |
-
final_results = candidates[:n_results]
|
| 386 |
context_parts = []
|
| 387 |
-
for cand in
|
| 388 |
-
payload = cand[
|
| 389 |
content = payload.get("content", "").strip()
|
| 390 |
-
if not content:
|
| 391 |
-
|
| 392 |
-
vol
|
| 393 |
page = payload.get("page")
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
context_parts.append(f"[เล่ม {vol} หน้า {page}]\n{clean_content}")
|
| 401 |
-
|
| 402 |
-
return "\n---\n".join(context_parts) if context_parts else ""
|
| 403 |
-
|
| 404 |
except Exception as e:
|
| 405 |
logger.error(f"Hybrid query error: {e}")
|
| 406 |
return ""
|
| 407 |
-
|
|
|
|
| 2 |
import os
|
| 3 |
from pathlib import Path
|
| 4 |
from collections import OrderedDict
|
|
|
|
|
|
|
| 5 |
import httpx
|
| 6 |
from app.config import get_settings
|
| 7 |
from app.database.sqlite_db import get_db
|
|
|
|
| 20 |
|
| 21 |
logger = logging.getLogger(__name__)
|
| 22 |
|
| 23 |
+
|
| 24 |
+
def _decode_tv_id(uid: int) -> tuple[int, int]:
|
| 25 |
+
"""Decode turbovec uint64 ID back to (volume, page). Encoding: vol*100000+page."""
|
| 26 |
+
return uid // 100_000, uid % 100_000
|
| 27 |
+
|
| 28 |
+
|
| 29 |
class RAGService:
|
| 30 |
def __init__(self):
|
| 31 |
settings = get_settings()
|
|
|
|
|
|
|
| 32 |
self.ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
| 33 |
+
|
| 34 |
+
self.model = None # None=not verified, 'ready'=Ollama OK, 'fallback'=ST
|
|
|
|
| 35 |
self.reranker = None
|
| 36 |
+
self._embed_cache = OrderedDict()
|
| 37 |
+
self._embed_cache_max = 256
|
| 38 |
+
self._st_model = None
|
| 39 |
+
self.tv_index = None # turbovec IdMapIndex
|
|
|
|
| 40 |
|
| 41 |
+
# Turbovec index path (DATA_DIR/tipitaka_chunks.tvim)
|
| 42 |
+
data_dir = Path(settings.DATA_DIR)
|
| 43 |
+
self._tv_path = str(data_dir / "tipitaka_chunks.tvim")
|
| 44 |
|
| 45 |
+
self._init_turbovec()
|
| 46 |
self._load_model()
|
| 47 |
self._load_reranker()
|
| 48 |
|
| 49 |
+
# ── Turbovec init ────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
+
def _init_turbovec(self):
|
| 52 |
+
"""Load turbovec IdMapIndex from disk."""
|
| 53 |
try:
|
| 54 |
+
from turbovec import IdMapIndex
|
| 55 |
+
if not Path(self._tv_path).exists():
|
| 56 |
+
logger.warning(f"turbovec index not found at {self._tv_path} — vector search disabled")
|
| 57 |
+
return
|
| 58 |
+
logger.info(f"Loading turbovec index from {self._tv_path}...")
|
| 59 |
+
self.tv_index = IdMapIndex.load(self._tv_path)
|
| 60 |
+
logger.info("turbovec index loaded ✅")
|
| 61 |
except Exception as e:
|
| 62 |
+
logger.error(f"Failed to load turbovec index: {e}")
|
| 63 |
+
self.tv_index = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
+
# ── Embedding ────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
|
| 67 |
def _load_model(self):
|
| 68 |
+
"""Verify Ollama or preload sentence-transformers as fallback."""
|
| 69 |
if self.model is None:
|
| 70 |
+
logger.info("Verifying Ollama embedding model...")
|
| 71 |
try:
|
| 72 |
r = httpx.get(f"{self.ollama_url}/api/tags", timeout=5)
|
| 73 |
r.raise_for_status()
|
| 74 |
self.model = "ready"
|
| 75 |
+
logger.info("Ollama embedding service ready.")
|
| 76 |
except Exception as e:
|
| 77 |
+
logger.error(f"Ollama not accessible: {e}. Pre-loading sentence-transformers...")
|
| 78 |
self.model = "fallback"
|
| 79 |
try:
|
| 80 |
settings = get_settings()
|
|
|
|
| 81 |
from sentence_transformers import SentenceTransformer
|
| 82 |
self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
|
| 83 |
+
logger.info("Sentence-transformers model loaded ✅")
|
| 84 |
except Exception as st_err:
|
| 85 |
+
logger.error(f"Failed to load sentence-transformers: {st_err}")
|
| 86 |
self.model = None
|
| 87 |
|
| 88 |
def _load_reranker(self):
|
| 89 |
+
"""Pre-load ONNX Reranker."""
|
| 90 |
if self.reranker is None:
|
| 91 |
logger.info(f"Pre-loading ONNX Reranker from {RERANK_MODEL_PATH}...")
|
| 92 |
try:
|
| 93 |
from app.services.onnx_reranker import ONNXReranker
|
| 94 |
self.reranker = ONNXReranker(RERANK_MODEL_PATH)
|
| 95 |
+
logger.info("ONNX Reranker pre-loaded ✅")
|
| 96 |
except Exception as e:
|
| 97 |
logger.error(f"Failed to pre-load ONNX Reranker: {e}")
|
|
|
|
| 98 |
self.reranker = "error"
|
| 99 |
|
| 100 |
def _get_embedding(self, text: str) -> list:
|
|
|
|
| 101 |
res = self._get_embeddings([text])
|
| 102 |
return res[0] if res else []
|
| 103 |
|
| 104 |
def _get_embeddings(self, texts: list[str]) -> list[list[float]]:
|
|
|
|
| 105 |
if not texts:
|
| 106 |
return []
|
| 107 |
|
| 108 |
results = [None] * len(texts)
|
| 109 |
+
missing_indices, missing_texts = [], []
|
| 110 |
+
|
|
|
|
|
|
|
| 111 |
for idx, text in enumerate(texts):
|
| 112 |
if text in self._embed_cache:
|
| 113 |
+
self._embed_cache.move_to_end(text)
|
| 114 |
results[idx] = self._embed_cache[text]
|
| 115 |
else:
|
| 116 |
missing_indices.append(idx)
|
| 117 |
missing_texts.append(text)
|
| 118 |
+
|
| 119 |
if not missing_texts:
|
| 120 |
return results
|
| 121 |
|
|
|
|
| 122 |
ollama_failed = True
|
| 123 |
if self.model == "ready":
|
| 124 |
try:
|
|
|
|
| 134 |
results[idx] = emb
|
| 135 |
ollama_failed = False
|
| 136 |
except Exception as e:
|
| 137 |
+
logger.info(f"Ollama batch embedding unavailable ({e}), falling back to ST")
|
|
|
|
| 138 |
self.model = "fallback"
|
| 139 |
|
|
|
|
| 140 |
if ollama_failed:
|
| 141 |
try:
|
| 142 |
if self._st_model is None:
|
| 143 |
settings = get_settings()
|
|
|
|
| 144 |
from sentence_transformers import SentenceTransformer
|
| 145 |
self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
|
| 146 |
|
| 147 |
+
embeddings = self._st_model.encode(
|
| 148 |
+
missing_texts, normalize_embeddings=True, show_progress_bar=False
|
| 149 |
+
).tolist()
|
| 150 |
for idx, emb in zip(missing_indices, embeddings):
|
| 151 |
self._store_cache(texts[idx], emb)
|
| 152 |
results[idx] = emb
|
| 153 |
return results
|
| 154 |
except Exception as e:
|
| 155 |
+
logger.error(f"ST batch embedding failed: {e}")
|
| 156 |
for idx in missing_indices:
|
| 157 |
results[idx] = []
|
| 158 |
return results
|
|
|
|
| 159 |
|
| 160 |
+
return results
|
| 161 |
|
| 162 |
def _store_cache(self, text: str, embedding: list):
|
|
|
|
| 163 |
self._embed_cache[text] = embedding
|
| 164 |
if len(self._embed_cache) > self._embed_cache_max:
|
| 165 |
+
self._embed_cache.popitem(last=False)
|
| 166 |
+
|
| 167 |
+
# ── Query (Hybrid: FTS5 + turbovec) ─────────────────────────────
|
| 168 |
|
| 169 |
async def query(self, text: str, n_results: int = 10, threshold: float = 0.2) -> str:
|
| 170 |
"""
|
| 171 |
+
Hybrid Search:
|
| 172 |
+
1. FTS5 (SQLite) — keyword matches
|
| 173 |
+
2. turbovec — semantic vector search
|
| 174 |
+
3. Merge & Deduplicate
|
| 175 |
+
4. Rerank via Jina v2 ONNX
|
| 176 |
"""
|
| 177 |
+
if self.tv_index is None and self.model is None:
|
| 178 |
return ""
|
| 179 |
+
|
|
|
|
| 180 |
if self.model is None:
|
| 181 |
await anyio.to_thread.run_sync(self._load_model)
|
| 182 |
if self.reranker is None:
|
| 183 |
await anyio.to_thread.run_sync(self._load_reranker)
|
| 184 |
|
| 185 |
candidates = []
|
| 186 |
+
seen_keys = set()
|
| 187 |
|
| 188 |
try:
|
| 189 |
+
# ── Phase 1: FTS5 ────────────────────────────────────
|
| 190 |
+
logger.info(f"FTS5 search: {text[:40]}...")
|
| 191 |
+
|
| 192 |
def _blocking_fts():
|
|
|
|
| 193 |
db = get_db()
|
| 194 |
with db.get_connection() as conn:
|
| 195 |
+
sanitized = text.replace('"', '').strip()
|
| 196 |
+
if not sanitized:
|
| 197 |
+
return []
|
| 198 |
+
tokens = [t for t in sanitized.split() if t]
|
| 199 |
+
fts_query = " AND ".join(
|
| 200 |
+
f'"{t}"*' if "*" not in t else t for t in tokens
|
| 201 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
try:
|
| 203 |
+
cursor = conn.execute("""
|
| 204 |
+
SELECT id, volume_id, page_number, content_text
|
| 205 |
+
FROM pages
|
| 206 |
+
WHERE id IN (
|
| 207 |
+
SELECT rowid FROM pages_fts
|
| 208 |
+
WHERE pages_fts MATCH ?
|
| 209 |
+
LIMIT 30
|
| 210 |
+
)
|
| 211 |
+
""", (fts_query,))
|
| 212 |
+
return [
|
| 213 |
+
{
|
| 214 |
+
"id": r["id"],
|
| 215 |
+
"score": 0.9,
|
| 216 |
"payload": {
|
| 217 |
+
"volume": r["volume_id"],
|
| 218 |
+
"page": r["page_number"],
|
| 219 |
+
"content": r["content_text"],
|
| 220 |
},
|
| 221 |
+
"key": f"{r['volume_id']}_{r['page_number']}",
|
| 222 |
+
}
|
| 223 |
+
for r in cursor.fetchall()
|
| 224 |
+
]
|
| 225 |
except Exception as e:
|
| 226 |
+
logger.warning(f"FTS5 failed: {e}")
|
| 227 |
+
return []
|
| 228 |
|
| 229 |
+
for cand in await anyio.to_thread.run_sync(_blocking_fts):
|
| 230 |
+
if cand["key"] not in seen_keys:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
candidates.append(cand)
|
| 232 |
+
seen_keys.add(cand["key"])
|
| 233 |
+
|
| 234 |
+
# ── Phase 2: turbovec vector search ──────────────────
|
| 235 |
+
if self.tv_index is not None:
|
| 236 |
+
logger.info(f"turbovec search: {text[:40]}...")
|
| 237 |
+
|
| 238 |
+
def _blocking_vector():
|
| 239 |
+
import numpy as np
|
| 240 |
+
query_vec = self._get_embedding(text)
|
| 241 |
+
if not query_vec:
|
| 242 |
+
return []
|
| 243 |
+
q_np = np.array([query_vec], dtype=np.float32)
|
| 244 |
+
scores, ids = self.tv_index.search(q_np, k=30)
|
| 245 |
+
vec_results = []
|
| 246 |
+
for score, uid in zip(scores[0], ids[0]):
|
| 247 |
+
if score < threshold:
|
| 248 |
+
continue
|
| 249 |
+
vol, page = _decode_tv_id(int(uid))
|
| 250 |
+
vec_results.append({
|
| 251 |
+
"id": int(uid),
|
| 252 |
+
"score": float(score),
|
| 253 |
+
"payload": {"volume": vol, "page": page, "content": ""},
|
| 254 |
+
"key": f"{vol}_{page}",
|
| 255 |
+
})
|
| 256 |
+
return vec_results
|
| 257 |
+
|
| 258 |
+
for cand in await anyio.to_thread.run_sync(_blocking_vector):
|
| 259 |
+
if cand["key"] not in seen_keys:
|
| 260 |
+
candidates.append(cand)
|
| 261 |
+
seen_keys.add(cand["key"])
|
| 262 |
|
| 263 |
if not candidates:
|
| 264 |
return ""
|
| 265 |
|
| 266 |
+
# Enrich missing content payloads from SQLite (e.g. for turbovec candidates)
|
| 267 |
+
missing_content_cands = [c for c in candidates if not c["payload"].get("content")]
|
| 268 |
+
if missing_content_cands:
|
| 269 |
+
def _blocking_enrich():
|
| 270 |
+
db = get_db()
|
| 271 |
+
with db.get_connection() as conn:
|
| 272 |
+
cursor = conn.cursor()
|
| 273 |
+
for cand in missing_content_cands:
|
| 274 |
+
try:
|
| 275 |
+
cursor.execute(
|
| 276 |
+
"SELECT content_text FROM pages WHERE volume_id = ? AND page_number = ?",
|
| 277 |
+
(cand["payload"]["volume"], cand["payload"]["page"])
|
| 278 |
+
)
|
| 279 |
+
r = cursor.fetchone()
|
| 280 |
+
if r:
|
| 281 |
+
cand["payload"]["content"] = r["content_text"]
|
| 282 |
+
except Exception as e:
|
| 283 |
+
logger.warning(f"Failed to fetch content for vol={cand['payload']['volume']} page={cand['payload']['page']}: {e}")
|
| 284 |
+
await anyio.to_thread.run_sync(_blocking_enrich)
|
| 285 |
+
|
| 286 |
+
# ── Phase 3: Rerank ───────────────────────────────────
|
| 287 |
if self.reranker and self.reranker != "error":
|
| 288 |
+
logger.info(f"Reranking {len(candidates)} candidates...")
|
| 289 |
+
|
| 290 |
def _blocking_rerank():
|
| 291 |
+
passages = [c["payload"].get("content", "")[:1000] for c in candidates]
|
|
|
|
| 292 |
pairs = [[text, p] for p in passages]
|
|
|
|
| 293 |
with torch.no_grad():
|
| 294 |
scores = self.reranker.predict(pairs, show_progress_bar=False, batch_size=4)
|
|
|
|
| 295 |
for i, cand in enumerate(candidates):
|
| 296 |
+
cand["rerank_score"] = float(scores[i])
|
| 297 |
+
return sorted(candidates, key=lambda x: x.get("rerank_score", 0), reverse=True)
|
|
|
|
| 298 |
|
| 299 |
candidates = await anyio.to_thread.run_sync(_blocking_rerank)
|
| 300 |
+
|
| 301 |
+
# ── Phase 4: Format ───────────────────────────────────
|
|
|
|
| 302 |
context_parts = []
|
| 303 |
+
for cand in candidates[:n_results]:
|
| 304 |
+
payload = cand["payload"]
|
| 305 |
content = payload.get("content", "").strip()
|
| 306 |
+
if not content:
|
| 307 |
+
continue
|
| 308 |
+
vol = payload.get("volume")
|
| 309 |
page = payload.get("page")
|
| 310 |
+
clean = re.sub(r"---.*?---", "", content, flags=re.DOTALL).strip()
|
| 311 |
+
clean = re.sub(r"^\d{3}\s+", "", clean, flags=re.MULTILINE)
|
| 312 |
+
context_parts.append(f"[เล่ม {vol} หน้า {page}]\n{clean[:2000]}")
|
| 313 |
+
|
| 314 |
+
return "\n---\n".join(context_parts)
|
| 315 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
except Exception as e:
|
| 317 |
logger.error(f"Hybrid query error: {e}")
|
| 318 |
return ""
|
|
|
webapp/tipitaka-api/app/services/search_service.py
CHANGED
|
@@ -193,181 +193,65 @@ class SearchService:
|
|
| 193 |
return self.highlight_multiple(text, [keyword]) if keyword else text
|
| 194 |
|
| 195 |
async def _get_vector_results_batch(self, query_texts: List[str], query_vectors: List[List[float]], limit: int = 30) -> List[List[dict]]:
|
| 196 |
-
"""Fetch semantic vector results from
|
| 197 |
-
if not self.rag_service or not self.rag_service.
|
| 198 |
return [[] for _ in query_texts]
|
| 199 |
|
| 200 |
import anyio
|
| 201 |
-
|
| 202 |
|
| 203 |
def _blocking():
|
| 204 |
-
|
| 205 |
-
client = self.rag_service.client
|
| 206 |
-
|
| 207 |
-
# Map request index to original query indices
|
| 208 |
valid_indices = [idx for idx, vec in enumerate(query_vectors) if vec]
|
| 209 |
if not valid_indices:
|
| 210 |
return [[] for _ in query_texts]
|
| 211 |
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
if hasattr(client, "query_batch_points"):
|
| 216 |
-
requests = [
|
| 217 |
-
qmodels.QueryRequest(
|
| 218 |
-
query=query_vectors[idx],
|
| 219 |
-
using="dense",
|
| 220 |
-
limit=limit,
|
| 221 |
-
with_payload=True,
|
| 222 |
-
score_threshold=0.2
|
| 223 |
-
) for idx in valid_indices
|
| 224 |
-
]
|
| 225 |
-
try:
|
| 226 |
-
batch_responses = client.query_batch_points(
|
| 227 |
-
collection_name=col,
|
| 228 |
-
requests=requests
|
| 229 |
-
)
|
| 230 |
-
for req_idx, resp in zip(valid_indices, batch_responses):
|
| 231 |
-
hits = resp.points if hasattr(resp, "points") else resp
|
| 232 |
-
vec_results = []
|
| 233 |
-
for hit in hits:
|
| 234 |
-
vol = hit.payload.get("volume", hit.payload.get("volume_id"))
|
| 235 |
-
page = hit.payload.get("page", hit.payload.get("page_number"))
|
| 236 |
-
content = hit.payload.get("content", "")
|
| 237 |
-
vec_results.append({
|
| 238 |
-
"volume_id": vol,
|
| 239 |
-
"page_number": page,
|
| 240 |
-
"content_text": content,
|
| 241 |
-
"score": hit.score
|
| 242 |
-
})
|
| 243 |
-
final_results[req_idx] = vec_results
|
| 244 |
-
return final_results
|
| 245 |
-
except Exception as e:
|
| 246 |
-
import logging
|
| 247 |
-
logging.getLogger(__name__).warning(f"Qdrant query_batch_points failed: {e}")
|
| 248 |
-
|
| 249 |
-
# 2. Try legacy search_batch API if available
|
| 250 |
-
if hasattr(client, "search_batch"):
|
| 251 |
-
requests = [
|
| 252 |
-
qmodels.SearchRequest(
|
| 253 |
-
vector=qmodels.NamedVector(name="dense", vector=query_vectors[idx]),
|
| 254 |
-
limit=limit,
|
| 255 |
-
with_payload=True,
|
| 256 |
-
score_threshold=0.2
|
| 257 |
-
) for idx in valid_indices
|
| 258 |
-
]
|
| 259 |
-
try:
|
| 260 |
-
batch_responses = client.search_batch(
|
| 261 |
-
collection_name=col,
|
| 262 |
-
requests=requests
|
| 263 |
-
)
|
| 264 |
-
for req_idx, hits in zip(valid_indices, batch_responses):
|
| 265 |
-
vec_results = []
|
| 266 |
-
for hit in hits:
|
| 267 |
-
vol = hit.payload.get("volume", hit.payload.get("volume_id"))
|
| 268 |
-
page = hit.payload.get("page", hit.payload.get("page_number"))
|
| 269 |
-
content = hit.payload.get("content", "")
|
| 270 |
-
vec_results.append({
|
| 271 |
-
"volume_id": vol,
|
| 272 |
-
"page_number": page,
|
| 273 |
-
"content_text": content,
|
| 274 |
-
"score": hit.score
|
| 275 |
-
})
|
| 276 |
-
final_results[req_idx] = vec_results
|
| 277 |
-
return final_results
|
| 278 |
-
except Exception as e:
|
| 279 |
-
import logging
|
| 280 |
-
logging.getLogger(__name__).warning(f"Qdrant search_batch failed: {e}")
|
| 281 |
-
|
| 282 |
-
# 3. Fallback: run sequential queries in blocking thread
|
| 283 |
-
for idx in valid_indices:
|
| 284 |
-
try:
|
| 285 |
-
if hasattr(client, "search"):
|
| 286 |
-
hits = client.search(
|
| 287 |
-
collection_name=col,
|
| 288 |
-
query_vector=("dense", query_vectors[idx]),
|
| 289 |
-
limit=limit,
|
| 290 |
-
with_payload=True,
|
| 291 |
-
score_threshold=0.2
|
| 292 |
-
)
|
| 293 |
-
else:
|
| 294 |
-
response = client.query_points(
|
| 295 |
-
collection_name=col,
|
| 296 |
-
query=query_vectors[idx],
|
| 297 |
-
using="dense",
|
| 298 |
-
limit=limit,
|
| 299 |
-
with_payload=True,
|
| 300 |
-
score_threshold=0.2
|
| 301 |
-
)
|
| 302 |
-
hits = response.points
|
| 303 |
-
|
| 304 |
-
vec_results = []
|
| 305 |
-
for hit in hits:
|
| 306 |
-
vol = hit.payload.get("volume", hit.payload.get("volume_id"))
|
| 307 |
-
page = hit.payload.get("page", hit.payload.get("page_number"))
|
| 308 |
-
content = hit.payload.get("content", "")
|
| 309 |
-
vec_results.append({
|
| 310 |
-
"volume_id": vol,
|
| 311 |
-
"page_number": page,
|
| 312 |
-
"content_text": content,
|
| 313 |
-
"score": hit.score
|
| 314 |
-
})
|
| 315 |
-
final_results[idx] = vec_results
|
| 316 |
-
except Exception as e:
|
| 317 |
-
import logging
|
| 318 |
-
logging.getLogger(__name__).warning(f"Qdrant single query fallback failed: {e}")
|
| 319 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 320 |
return final_results
|
| 321 |
|
| 322 |
return await anyio.to_thread.run_sync(_blocking)
|
| 323 |
|
| 324 |
async def _get_vector_results(self, query_text: str, limit: int = 30) -> List[dict]:
|
| 325 |
-
"""Fetch semantic vector results from
|
| 326 |
-
if not self.rag_service or not self.rag_service.
|
| 327 |
return []
|
| 328 |
|
| 329 |
import anyio
|
|
|
|
| 330 |
|
| 331 |
def _blocking():
|
| 332 |
query_vector = self.rag_service._get_embedding(query_text)
|
| 333 |
if not query_vector:
|
| 334 |
return []
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
try:
|
| 338 |
-
if hasattr(self.rag_service.client, "search"):
|
| 339 |
-
hits = self.rag_service.client.search(
|
| 340 |
-
collection_name=col,
|
| 341 |
-
query_vector=("dense", query_vector),
|
| 342 |
-
limit=limit,
|
| 343 |
-
with_payload=True,
|
| 344 |
-
score_threshold=0.2
|
| 345 |
-
)
|
| 346 |
-
else:
|
| 347 |
-
response = self.rag_service.client.query_points(
|
| 348 |
-
collection_name=col,
|
| 349 |
-
query=query_vector,
|
| 350 |
-
using="dense",
|
| 351 |
-
limit=limit,
|
| 352 |
-
with_payload=True,
|
| 353 |
-
score_threshold=0.2
|
| 354 |
-
)
|
| 355 |
-
hits = response.points
|
| 356 |
-
except Exception as e:
|
| 357 |
-
import logging
|
| 358 |
-
logging.getLogger(__name__).warning(f"Qdrant query failed: {e}")
|
| 359 |
-
return []
|
| 360 |
-
|
| 361 |
vec_results = []
|
| 362 |
-
for
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
vec_results.append({
|
| 367 |
-
"volume_id": vol,
|
| 368 |
-
"page_number": page,
|
| 369 |
-
"content_text":
|
| 370 |
-
"score":
|
| 371 |
})
|
| 372 |
return vec_results
|
| 373 |
|
|
@@ -495,6 +379,24 @@ class SearchService:
|
|
| 495 |
top_keys = sorted(rrf_scores, key=rrf_scores.get, reverse=True)[:15]
|
| 496 |
top_candidates = [candidate_data[ky] for ky in top_keys]
|
| 497 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 498 |
# ── 5. Reranking (using Jina Reranker v2 ONNX) ────────────
|
| 499 |
if top_candidates:
|
| 500 |
final_results = await self._rerank_candidates(corrected_q, top_candidates)
|
|
|
|
| 193 |
return self.highlight_multiple(text, [keyword]) if keyword else text
|
| 194 |
|
| 195 |
async def _get_vector_results_batch(self, query_texts: List[str], query_vectors: List[List[float]], limit: int = 30) -> List[List[dict]]:
|
| 196 |
+
"""Fetch semantic vector results from turbovec in batch."""
|
| 197 |
+
if not self.rag_service or not self.rag_service.tv_index or not query_vectors:
|
| 198 |
return [[] for _ in query_texts]
|
| 199 |
|
| 200 |
import anyio
|
| 201 |
+
import numpy as np
|
| 202 |
|
| 203 |
def _blocking():
|
| 204 |
+
tv = self.rag_service.tv_index
|
|
|
|
|
|
|
|
|
|
| 205 |
valid_indices = [idx for idx, vec in enumerate(query_vectors) if vec]
|
| 206 |
if not valid_indices:
|
| 207 |
return [[] for _ in query_texts]
|
| 208 |
|
| 209 |
+
# Stack valid query vectors for batch search
|
| 210 |
+
vecs = np.array([query_vectors[idx] for idx in valid_indices], dtype=np.float32)
|
| 211 |
+
all_scores, all_ids = tv.search(vecs, k=limit)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
|
| 213 |
+
final_results = [[] for _ in query_texts]
|
| 214 |
+
for req_pos, req_idx in enumerate(valid_indices):
|
| 215 |
+
vec_results = []
|
| 216 |
+
for score, uid in zip(all_scores[req_pos], all_ids[req_pos]):
|
| 217 |
+
if float(score) < 0.2:
|
| 218 |
+
continue
|
| 219 |
+
vol, page = uid // 100_000, uid % 100_000
|
| 220 |
+
vec_results.append({
|
| 221 |
+
"volume_id": int(vol),
|
| 222 |
+
"page_number": int(page),
|
| 223 |
+
"content_text": "", # fetched from SQLite later
|
| 224 |
+
"score": float(score)
|
| 225 |
+
})
|
| 226 |
+
final_results[req_idx] = vec_results
|
| 227 |
return final_results
|
| 228 |
|
| 229 |
return await anyio.to_thread.run_sync(_blocking)
|
| 230 |
|
| 231 |
async def _get_vector_results(self, query_text: str, limit: int = 30) -> List[dict]:
|
| 232 |
+
"""Fetch semantic vector results from turbovec via RAGService."""
|
| 233 |
+
if not self.rag_service or not self.rag_service.tv_index:
|
| 234 |
return []
|
| 235 |
|
| 236 |
import anyio
|
| 237 |
+
import numpy as np
|
| 238 |
|
| 239 |
def _blocking():
|
| 240 |
query_vector = self.rag_service._get_embedding(query_text)
|
| 241 |
if not query_vector:
|
| 242 |
return []
|
| 243 |
+
q_np = np.array([query_vector], dtype=np.float32)
|
| 244 |
+
scores, ids = self.rag_service.tv_index.search(q_np, k=limit)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
vec_results = []
|
| 246 |
+
for score, uid in zip(scores[0], ids[0]):
|
| 247 |
+
if float(score) < 0.2:
|
| 248 |
+
continue
|
| 249 |
+
vol, page = uid // 100_000, uid % 100_000
|
| 250 |
vec_results.append({
|
| 251 |
+
"volume_id": int(vol),
|
| 252 |
+
"page_number": int(page),
|
| 253 |
+
"content_text": "", # fetched from SQLite later
|
| 254 |
+
"score": float(score)
|
| 255 |
})
|
| 256 |
return vec_results
|
| 257 |
|
|
|
|
| 379 |
top_keys = sorted(rrf_scores, key=rrf_scores.get, reverse=True)[:15]
|
| 380 |
top_candidates = [candidate_data[ky] for ky in top_keys]
|
| 381 |
|
| 382 |
+
# Enrich missing content_text from SQLite (e.g. for turbovec vector results)
|
| 383 |
+
missing_content_cands = [c for c in top_candidates if not c.get("content_text")]
|
| 384 |
+
if missing_content_cands:
|
| 385 |
+
try:
|
| 386 |
+
with self.db.get_connection() as conn:
|
| 387 |
+
cursor = conn.cursor()
|
| 388 |
+
for c in missing_content_cands:
|
| 389 |
+
cursor.execute(
|
| 390 |
+
"SELECT content_text FROM pages WHERE volume_id = ? AND page_number = ?",
|
| 391 |
+
(c["volume_id"], c["page_number"])
|
| 392 |
+
)
|
| 393 |
+
row = cursor.fetchone()
|
| 394 |
+
if row:
|
| 395 |
+
c["content_text"] = row["content_text"]
|
| 396 |
+
except Exception as e:
|
| 397 |
+
import logging
|
| 398 |
+
logging.getLogger(__name__).warning(f"Enriching candidates content failed: {e}")
|
| 399 |
+
|
| 400 |
# ── 5. Reranking (using Jina Reranker v2 ONNX) ────────────
|
| 401 |
if top_candidates:
|
| 402 |
final_results = await self._rerank_candidates(corrected_q, top_candidates)
|
webapp/tipitaka-api/benchmark_qdrant_vs_turbovec.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
import numpy as np
|
| 3 |
+
from qdrant_client import QdrantClient
|
| 4 |
+
from turbovec import IdMapIndex
|
| 5 |
+
import sys
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
# Configurations
|
| 9 |
+
QDRANT_URL = "http://127.0.0.1:6333"
|
| 10 |
+
COLLECTION = "tipitaka_chunks"
|
| 11 |
+
TV_PATH = "F:/_Ai/Tipitaka-AI-Expert/RAG/webapp/tipitaka-api/data/tipitaka_chunks.tvim"
|
| 12 |
+
EMBED_DIM = 1024
|
| 13 |
+
NUM_QUERIES = 100
|
| 14 |
+
K = 30
|
| 15 |
+
|
| 16 |
+
def benchmark():
|
| 17 |
+
print("=" * 60)
|
| 18 |
+
print(" BENCHMARK: QDRANT VS TURBOVEC (SEARCH ONLY)")
|
| 19 |
+
print("=" * 60)
|
| 20 |
+
|
| 21 |
+
# 1. Generate random query vectors to isolate search speed from embedding latency
|
| 22 |
+
np.random.seed(42)
|
| 23 |
+
query_vectors = np.random.randn(NUM_QUERIES, EMBED_DIM).astype(np.float32)
|
| 24 |
+
# Normalize for cosine similarity
|
| 25 |
+
norms = np.linalg.norm(query_vectors, axis=1, keepdims=True)
|
| 26 |
+
query_vectors = query_vectors / norms
|
| 27 |
+
|
| 28 |
+
# 2. Turbovec initialization
|
| 29 |
+
t_start = time.time()
|
| 30 |
+
tv_index = IdMapIndex.load(TV_PATH)
|
| 31 |
+
t_tv_load = (time.time() - t_start) * 1000
|
| 32 |
+
print(f"Turbovec index loaded from disk in: {t_tv_load:.2f} ms")
|
| 33 |
+
|
| 34 |
+
# 3. Qdrant initialization
|
| 35 |
+
t_start = time.time()
|
| 36 |
+
qdrant_client = QdrantClient(url=QDRANT_URL)
|
| 37 |
+
# Warm up / check connection
|
| 38 |
+
try:
|
| 39 |
+
qdrant_client.get_collection(COLLECTION)
|
| 40 |
+
t_qd_connect = (time.time() - t_start) * 1000
|
| 41 |
+
print(f"Connected to Qdrant local server in: {t_qd_connect:.2f} ms")
|
| 42 |
+
except Exception as e:
|
| 43 |
+
print(f"FAILED to connect to Qdrant: {e}")
|
| 44 |
+
return
|
| 45 |
+
|
| 46 |
+
print(f"Running {NUM_QUERIES} queries (k={K}) on both databases...")
|
| 47 |
+
|
| 48 |
+
# --- Benchmark Turbovec ---
|
| 49 |
+
tv_times = []
|
| 50 |
+
# Warmup
|
| 51 |
+
tv_index.search(query_vectors[0:1], k=K)
|
| 52 |
+
|
| 53 |
+
for i in range(NUM_QUERIES):
|
| 54 |
+
q = query_vectors[i:i+1]
|
| 55 |
+
start = time.perf_counter()
|
| 56 |
+
scores, ids = tv_index.search(q, k=K)
|
| 57 |
+
end = time.perf_counter()
|
| 58 |
+
tv_times.append((end - start) * 1000)
|
| 59 |
+
|
| 60 |
+
# --- Benchmark Qdrant ---
|
| 61 |
+
qd_times = []
|
| 62 |
+
# Warmup
|
| 63 |
+
try:
|
| 64 |
+
qdrant_client.query_points(
|
| 65 |
+
collection_name=COLLECTION,
|
| 66 |
+
query=query_vectors[0].tolist(),
|
| 67 |
+
using="dense",
|
| 68 |
+
limit=K,
|
| 69 |
+
with_payload=False
|
| 70 |
+
)
|
| 71 |
+
except Exception as e:
|
| 72 |
+
print(f"Qdrant warmup query_points failed: {e}")
|
| 73 |
+
return
|
| 74 |
+
|
| 75 |
+
for i in range(NUM_QUERIES):
|
| 76 |
+
q_list = query_vectors[i].tolist()
|
| 77 |
+
start = time.perf_counter()
|
| 78 |
+
qdrant_client.query_points(
|
| 79 |
+
collection_name=COLLECTION,
|
| 80 |
+
query=q_list,
|
| 81 |
+
using="dense",
|
| 82 |
+
limit=K,
|
| 83 |
+
with_payload=False
|
| 84 |
+
)
|
| 85 |
+
end = time.perf_counter()
|
| 86 |
+
qd_times.append((end - start) * 1000)
|
| 87 |
+
|
| 88 |
+
# 4. Results
|
| 89 |
+
print("\n" + "=" * 60)
|
| 90 |
+
print(" LATENCY RESULTS (ms)")
|
| 91 |
+
print("=" * 60)
|
| 92 |
+
print(f"{'Metric':<15} | {'Turbovec (4-bit SQ)':<20} | {'Qdrant Server':<20}")
|
| 93 |
+
print("-" * 60)
|
| 94 |
+
print(f"{'Average (Mean)':<15} | {np.mean(tv_times):>17.3f} ms | {np.mean(qd_times):>17.3f} ms")
|
| 95 |
+
print(f"{'Median (p50)':<15} | {np.percentile(tv_times, 50):>17.3f} ms | {np.percentile(qd_times, 50):>17.3f} ms")
|
| 96 |
+
print(f"{'90th Percentile':<15} | {np.percentile(tv_times, 90):>17.3f} ms | {np.percentile(qd_times, 90):>17.3f} ms")
|
| 97 |
+
print(f"{'Min':<15} | {np.min(tv_times):>17.3f} ms | {np.min(qd_times):>17.3f} ms")
|
| 98 |
+
print(f"{'Max':<15} | {np.max(tv_times):>17.3f} ms | {np.max(qd_times):>17.3f} ms")
|
| 99 |
+
print("=" * 60)
|
| 100 |
+
|
| 101 |
+
# Throughput
|
| 102 |
+
print(f"Throughput (QPS):")
|
| 103 |
+
print(f" • Turbovec: {NUM_QUERIES / (sum(tv_times)/1000):.1f} queries/sec")
|
| 104 |
+
print(f" • Qdrant: {NUM_QUERIES / (sum(qd_times)/1000):.1f} queries/sec")
|
| 105 |
+
print("=" * 60)
|
| 106 |
+
|
| 107 |
+
# Disk Space / Memory comparison (estimated)
|
| 108 |
+
print("Resource Footprint Comparison:")
|
| 109 |
+
print(" • Turbovec Index File: 13.2 MB")
|
| 110 |
+
print(" • Qdrant Local Server Storage: ~600+ MB (snapshots + database files)")
|
| 111 |
+
print(" • Memory Overhead:")
|
| 112 |
+
print(" - Turbovec: Loaded dynamically in Python process memory (~14MB overhead).")
|
| 113 |
+
print(" - Qdrant: Requires running a separate Docker daemon container (~150MB to 300MB RAM idle).")
|
| 114 |
+
print("=" * 60)
|
| 115 |
+
|
| 116 |
+
if __name__ == "__main__":
|
| 117 |
+
benchmark()
|
webapp/tipitaka-api/build_turbovec_index.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Build turbovec IdMapIndex from Qdrant tipitaka_chunks collection.
|
| 3 |
+
|
| 4 |
+
ID encoding: volume * 100000 + page (uint64, fully reversible)
|
| 5 |
+
Output: tipitaka_chunks.tvim (turbovec index file)
|
| 6 |
+
|
| 7 |
+
Usage:
|
| 8 |
+
python build_turbovec_index.py
|
| 9 |
+
python build_turbovec_index.py --collection tipitaka_chunks --output tipitaka_chunks.tvim
|
| 10 |
+
"""
|
| 11 |
+
import argparse
|
| 12 |
+
import logging
|
| 13 |
+
import time
|
| 14 |
+
import numpy as np
|
| 15 |
+
from qdrant_client import QdrantClient
|
| 16 |
+
from turbovec import IdMapIndex
|
| 17 |
+
|
| 18 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
QDRANT_URL = "http://localhost:6333"
|
| 22 |
+
COLLECTION = "tipitaka_chunks"
|
| 23 |
+
OUTPUT_PATH = "F:/_Ai/Tipitaka-AI-Expert/RAG/webapp/tipitaka-api/data/tipitaka_chunks.tvim"
|
| 24 |
+
EMBED_DIM = 1024
|
| 25 |
+
BATCH_SIZE = 500
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def encode_id(volume: str, page: str) -> int:
|
| 29 |
+
"""Encode (volume, page) as a single uint64. Reversible via decode_id()."""
|
| 30 |
+
return int(volume) * 100_000 + int(page)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def decode_id(uid: int) -> tuple[int, int]:
|
| 34 |
+
"""Decode uint64 back to (volume, page)."""
|
| 35 |
+
return uid // 100_000, uid % 100_000
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def main(collection: str, output: str):
|
| 39 |
+
logger.info(f"Connecting to Qdrant at {QDRANT_URL}...")
|
| 40 |
+
client = QdrantClient(url=QDRANT_URL)
|
| 41 |
+
|
| 42 |
+
info = client.get_collection(collection)
|
| 43 |
+
total = info.points_count
|
| 44 |
+
logger.info(f"Collection '{collection}': {total} points, dim={EMBED_DIM}")
|
| 45 |
+
|
| 46 |
+
# Build turbovec index
|
| 47 |
+
index = IdMapIndex(dim=EMBED_DIM, bit_width=4)
|
| 48 |
+
|
| 49 |
+
vectors_batch = []
|
| 50 |
+
ids_batch = []
|
| 51 |
+
processed = 0
|
| 52 |
+
skipped = 0
|
| 53 |
+
offset = None
|
| 54 |
+
start = time.time()
|
| 55 |
+
|
| 56 |
+
while True:
|
| 57 |
+
pts, next_offset = client.scroll(
|
| 58 |
+
collection,
|
| 59 |
+
offset=offset,
|
| 60 |
+
limit=BATCH_SIZE,
|
| 61 |
+
with_payload=True,
|
| 62 |
+
with_vectors=True,
|
| 63 |
+
)
|
| 64 |
+
if not pts:
|
| 65 |
+
break
|
| 66 |
+
|
| 67 |
+
for pt in pts:
|
| 68 |
+
payload = pt.payload or {}
|
| 69 |
+
volume = payload.get("volume", payload.get("volume_number", ""))
|
| 70 |
+
page = payload.get("page", payload.get("page_number", ""))
|
| 71 |
+
|
| 72 |
+
# Get dense vector only
|
| 73 |
+
vec = pt.vector
|
| 74 |
+
if isinstance(vec, dict):
|
| 75 |
+
vec = vec.get("dense")
|
| 76 |
+
if vec is None or not volume or not page:
|
| 77 |
+
skipped += 1
|
| 78 |
+
continue
|
| 79 |
+
|
| 80 |
+
try:
|
| 81 |
+
uid = encode_id(volume, page)
|
| 82 |
+
vectors_batch.append(vec)
|
| 83 |
+
ids_batch.append(uid)
|
| 84 |
+
except (ValueError, TypeError):
|
| 85 |
+
skipped += 1
|
| 86 |
+
continue
|
| 87 |
+
|
| 88 |
+
# Add batch to index
|
| 89 |
+
if vectors_batch:
|
| 90 |
+
vecs_np = np.array(vectors_batch, dtype=np.float32)
|
| 91 |
+
ids_np = np.array(ids_batch, dtype=np.uint64)
|
| 92 |
+
index.add_with_ids(vecs_np, ids_np)
|
| 93 |
+
processed += len(vecs_np)
|
| 94 |
+
vectors_batch.clear()
|
| 95 |
+
ids_batch.clear()
|
| 96 |
+
|
| 97 |
+
elapsed = time.time() - start
|
| 98 |
+
rate = processed / elapsed if elapsed > 0 else 0
|
| 99 |
+
eta = (total - processed) / rate if rate > 0 else 0
|
| 100 |
+
logger.info(f" {processed}/{total} ({100*processed//total}%) | {rate:.0f} pts/s | ETA {eta:.0f}s")
|
| 101 |
+
|
| 102 |
+
if next_offset is None:
|
| 103 |
+
break
|
| 104 |
+
offset = next_offset
|
| 105 |
+
|
| 106 |
+
# Save index
|
| 107 |
+
logger.info(f"\nSaving index to {output}...")
|
| 108 |
+
index.write(output)
|
| 109 |
+
|
| 110 |
+
elapsed_total = time.time() - start
|
| 111 |
+
logger.info("=" * 60)
|
| 112 |
+
logger.info(f"Done in {elapsed_total:.1f}s")
|
| 113 |
+
logger.info(f"Indexed: {processed} | Skipped: {skipped}")
|
| 114 |
+
logger.info(f"Saved: {output}")
|
| 115 |
+
logger.info(f"ID encoding: volume * 100000 + page (decode: v=id//100000, p=id%100000)")
|
| 116 |
+
logger.info("=" * 60)
|
| 117 |
+
|
| 118 |
+
# Quick sanity check
|
| 119 |
+
logger.info("\nSanity check — reload and search with random query...")
|
| 120 |
+
from turbovec import IdMapIndex as TVI
|
| 121 |
+
loaded = TVI.load(output)
|
| 122 |
+
query = np.random.randn(1, EMBED_DIM).astype(np.float32)
|
| 123 |
+
scores, ids = loaded.search(query, k=3)
|
| 124 |
+
for s, uid in zip(scores[0], ids[0]):
|
| 125 |
+
vol, pg = decode_id(int(uid))
|
| 126 |
+
logger.info(f" score={s:.4f} vol={vol} page={pg}")
|
| 127 |
+
|
| 128 |
+
client.close()
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
if __name__ == "__main__":
|
| 132 |
+
parser = argparse.ArgumentParser()
|
| 133 |
+
parser.add_argument("--collection", default=COLLECTION)
|
| 134 |
+
parser.add_argument("--output", default=OUTPUT_PATH)
|
| 135 |
+
args = parser.parse_args()
|
| 136 |
+
|
| 137 |
+
import os
|
| 138 |
+
os.makedirs(os.path.dirname(args.output), exist_ok=True)
|
| 139 |
+
main(args.collection, args.output)
|
webapp/tipitaka-api/download_assets.py
CHANGED
|
@@ -20,6 +20,7 @@ DB_PATH = Path(settings.DATABASE_PATH)
|
|
| 20 |
QDRANT_DIR = Path(settings.QDRANT_PATH)
|
| 21 |
SNAPSHOT_DIR = Path(settings.SNAPSHOT_DIR)
|
| 22 |
MODELS_DIR = Path(__file__).resolve().parent / "models"
|
|
|
|
| 23 |
|
| 24 |
BUCKET_ID = "dhammawatthumpra/tipitaka-storage"
|
| 25 |
RERANK_REPO_ID = "jinaai/jina-reranker-v2-base-multilingual"
|
|
@@ -48,10 +49,22 @@ def check_file_exists(path: Path) -> bool:
|
|
| 48 |
|
| 49 |
def download_files() -> None:
|
| 50 |
"""Download missing files from HF bucket."""
|
|
|
|
| 51 |
hf_token = os.getenv("HF_TOKEN", "")
|
| 52 |
from huggingface_hub import hf_hub_download
|
| 53 |
|
| 54 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
pending_core = []
|
| 56 |
for remote, local_path_str in BUCKET_FILES:
|
| 57 |
local_path = Path(local_path_str)
|
|
@@ -73,7 +86,6 @@ def download_files() -> None:
|
|
| 73 |
repo_type="dataset",
|
| 74 |
token=hf_token
|
| 75 |
)
|
| 76 |
-
import shutil
|
| 77 |
shutil.copy(downloaded, dest_path)
|
| 78 |
except Exception as e:
|
| 79 |
print(f"FAILED to download {remote}: {e}")
|
|
@@ -98,7 +110,6 @@ def download_files() -> None:
|
|
| 98 |
filename=filename,
|
| 99 |
token=hf_token if hf_token else None
|
| 100 |
)
|
| 101 |
-
import shutil
|
| 102 |
shutil.copy(downloaded, dest_path)
|
| 103 |
except Exception as e:
|
| 104 |
print(f"FAILED to download reranker file {filename}: {e}")
|
|
@@ -110,6 +121,9 @@ def verify_assets() -> bool:
|
|
| 110 |
if not DB_PATH.exists():
|
| 111 |
missing.append("tipitaka_mcu.db")
|
| 112 |
|
|
|
|
|
|
|
|
|
|
| 113 |
if not (QDRANT_DIR / "meta.json").exists():
|
| 114 |
missing.append("qdrant_storage/meta.json")
|
| 115 |
|
|
|
|
| 20 |
QDRANT_DIR = Path(settings.QDRANT_PATH)
|
| 21 |
SNAPSHOT_DIR = Path(settings.SNAPSHOT_DIR)
|
| 22 |
MODELS_DIR = Path(__file__).resolve().parent / "models"
|
| 23 |
+
TV_PATH = Path(settings.DATA_DIR) / "tipitaka_chunks.tvim"
|
| 24 |
|
| 25 |
BUCKET_ID = "dhammawatthumpra/tipitaka-storage"
|
| 26 |
RERANK_REPO_ID = "jinaai/jina-reranker-v2-base-multilingual"
|
|
|
|
| 49 |
|
| 50 |
def download_files() -> None:
|
| 51 |
"""Download missing files from HF bucket."""
|
| 52 |
+
import shutil
|
| 53 |
hf_token = os.getenv("HF_TOKEN", "")
|
| 54 |
from huggingface_hub import hf_hub_download
|
| 55 |
|
| 56 |
+
# Check and handle Turbovec index file
|
| 57 |
+
if not check_file_exists(TV_PATH):
|
| 58 |
+
packaged_tv = Path(__file__).resolve().parent / "data" / "tipitaka_chunks.tvim"
|
| 59 |
+
if check_file_exists(packaged_tv):
|
| 60 |
+
print(f"Copying packaged Turbovec index: {packaged_tv} -> {TV_PATH}")
|
| 61 |
+
TV_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
shutil.copy(packaged_tv, TV_PATH)
|
| 63 |
+
else:
|
| 64 |
+
# Fallback: add to bucket files so it attempts downloading from HF Space bucket
|
| 65 |
+
BUCKET_FILES.append(("tipitaka_chunks.tvim", str(TV_PATH)))
|
| 66 |
+
|
| 67 |
+
# 1. Download Core Assets (DB, Snapshots, and optionally Turbovec index)
|
| 68 |
pending_core = []
|
| 69 |
for remote, local_path_str in BUCKET_FILES:
|
| 70 |
local_path = Path(local_path_str)
|
|
|
|
| 86 |
repo_type="dataset",
|
| 87 |
token=hf_token
|
| 88 |
)
|
|
|
|
| 89 |
shutil.copy(downloaded, dest_path)
|
| 90 |
except Exception as e:
|
| 91 |
print(f"FAILED to download {remote}: {e}")
|
|
|
|
| 110 |
filename=filename,
|
| 111 |
token=hf_token if hf_token else None
|
| 112 |
)
|
|
|
|
| 113 |
shutil.copy(downloaded, dest_path)
|
| 114 |
except Exception as e:
|
| 115 |
print(f"FAILED to download reranker file {filename}: {e}")
|
|
|
|
| 121 |
if not DB_PATH.exists():
|
| 122 |
missing.append("tipitaka_mcu.db")
|
| 123 |
|
| 124 |
+
if not TV_PATH.exists():
|
| 125 |
+
missing.append("tipitaka_chunks.tvim")
|
| 126 |
+
|
| 127 |
if not (QDRANT_DIR / "meta.json").exists():
|
| 128 |
missing.append("qdrant_storage/meta.json")
|
| 129 |
|
webapp/tipitaka-api/requirements.txt
CHANGED
|
@@ -17,3 +17,5 @@ anyio>=4.0.0
|
|
| 17 |
pytest>=8.0
|
| 18 |
pytest-asyncio>=0.24
|
| 19 |
accelerate>=0.27.0
|
|
|
|
|
|
|
|
|
| 17 |
pytest>=8.0
|
| 18 |
pytest-asyncio>=0.24
|
| 19 |
accelerate>=0.27.0
|
| 20 |
+
turbovec>=0.5.3
|
| 21 |
+
|
webapp/tipitaka-api/test_rag_query.py
CHANGED
|
@@ -18,7 +18,7 @@ async def test_retrieval():
|
|
| 18 |
logger.info("Initializing RAGService...")
|
| 19 |
service = RAGService()
|
| 20 |
|
| 21 |
-
logger.info(f"Target
|
| 22 |
|
| 23 |
queries = [
|
| 24 |
"พระสุทินเสพเมถุนธรรมกับอดีตภรรยา 3 ครั้ง เพราะยังไม่มีสิกขาบท",
|
|
|
|
| 18 |
logger.info("Initializing RAGService...")
|
| 19 |
service = RAGService()
|
| 20 |
|
| 21 |
+
logger.info(f"Target turbovec index is: {service._tv_path}")
|
| 22 |
|
| 23 |
queries = [
|
| 24 |
"พระสุทินเสพเมถุนธรรมกับอดีตภรรยา 3 ครั้ง เพราะยังไม่มีสิกขาบท",
|
webapp/tipitaka-api/tests/test_rag_service.py
CHANGED
|
@@ -14,9 +14,9 @@ class TestEmbeddingCache:
|
|
| 14 |
from app.services.rag_service import RAGService
|
| 15 |
with patch.object(RAGService, '_load_model'), \
|
| 16 |
patch.object(RAGService, '_load_reranker'), \
|
| 17 |
-
patch.object(RAGService, '
|
| 18 |
service = RAGService()
|
| 19 |
-
service.
|
| 20 |
return service
|
| 21 |
|
| 22 |
def test_cache_initialized(self, rag_service):
|
|
@@ -48,12 +48,13 @@ class TestEmbeddingCache:
|
|
| 48 |
|
| 49 |
|
| 50 |
class TestRAGServiceInit:
|
| 51 |
-
def
|
| 52 |
-
"""RAGService constructor should not crash when
|
| 53 |
from app.services.rag_service import RAGService
|
| 54 |
with patch.object(RAGService, '_load_model'), \
|
| 55 |
patch.object(RAGService, '_load_reranker'), \
|
| 56 |
-
patch.object(RAGService, '
|
| 57 |
service = RAGService()
|
| 58 |
mock_init.assert_called_once()
|
| 59 |
-
assert service.
|
|
|
|
|
|
| 14 |
from app.services.rag_service import RAGService
|
| 15 |
with patch.object(RAGService, '_load_model'), \
|
| 16 |
patch.object(RAGService, '_load_reranker'), \
|
| 17 |
+
patch.object(RAGService, '_init_turbovec'):
|
| 18 |
service = RAGService()
|
| 19 |
+
service.tv_index = MagicMock()
|
| 20 |
return service
|
| 21 |
|
| 22 |
def test_cache_initialized(self, rag_service):
|
|
|
|
| 48 |
|
| 49 |
|
| 50 |
class TestRAGServiceInit:
|
| 51 |
+
def test_init_graceful_on_turbovec_failure(self):
|
| 52 |
+
"""RAGService constructor should not crash when turbovec is unavailable."""
|
| 53 |
from app.services.rag_service import RAGService
|
| 54 |
with patch.object(RAGService, '_load_model'), \
|
| 55 |
patch.object(RAGService, '_load_reranker'), \
|
| 56 |
+
patch.object(RAGService, '_init_turbovec') as mock_init:
|
| 57 |
service = RAGService()
|
| 58 |
mock_init.assert_called_once()
|
| 59 |
+
assert service.tv_index is None # _init_turbovec didn't set it
|
| 60 |
+
|
webapp/tipitaka-web/src/components/layout/ReaderPanel.tsx
CHANGED
|
@@ -230,103 +230,105 @@ const ReaderPanel: React.FC = () => {
|
|
| 230 |
/>
|
| 231 |
</ErrorBoundary>
|
| 232 |
<div className="max-w-3xl mx-auto px-4 py-10 lg:py-16">
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
<
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
>
|
| 240 |
-
<div className="w-10 h-10 border-4 border-[#c8860a]/20 border-t-[#c8860a] rounded-full animate-spin" />
|
| 241 |
-
<p className="text-[#c8860a] text-sm animate-pulse">กำลังอัญเชิญข้อความ…</p>
|
| 242 |
-
</motion.div>
|
| 243 |
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
<p key={`l0-${i}`} className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
|
| 270 |
-
{sec.title}
|
| 271 |
-
</p>
|
| 272 |
-
))}
|
| 273 |
-
{content.sections?.filter((s: {level: number}) => s.level === 1).map((sec: {title: string}, i: number) => (
|
| 274 |
-
<p key={`l1-${i}`} className="text-sm text-[#888] tracking-widest uppercase leading-relaxed">
|
| 275 |
-
{sec.title}
|
| 276 |
</p>
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
|
|
|
| 309 |
</div>
|
| 310 |
-
|
| 311 |
-
)}
|
| 312 |
-
|
| 313 |
-
<footer className="mt-12 pt-6 border-t border-[#888]/10 text-center">
|
| 314 |
-
<span className="text-xs text-[#888]">
|
| 315 |
-
— เล่ม {currentVolume} หน้า {currentPage} —
|
| 316 |
-
</span>
|
| 317 |
-
</footer>
|
| 318 |
-
</motion.article>
|
| 319 |
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
</div>
|
| 331 |
</div>
|
| 332 |
);
|
|
|
|
| 230 |
/>
|
| 231 |
</ErrorBoundary>
|
| 232 |
<div className="max-w-3xl mx-auto px-4 py-10 lg:py-16">
|
| 233 |
+
{loading && (
|
| 234 |
+
<div className="flex flex-col items-center justify-center py-48 gap-4">
|
| 235 |
+
<div className="w-10 h-10 border-4 border-[#c8860a]/20 border-t-[#c8860a] rounded-full animate-spin" />
|
| 236 |
+
<p className="text-[#c8860a] text-sm animate-pulse">กำลังอัญเชิญข้อความ…</p>
|
| 237 |
+
</div>
|
| 238 |
+
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
|
| 240 |
+
{!loading && (
|
| 241 |
+
<AnimatePresence custom={direction} mode="popLayout">
|
| 242 |
+
{content ? (
|
| 243 |
+
<motion.article
|
| 244 |
+
key={`${currentVolume}-${currentPage}`}
|
| 245 |
+
custom={direction}
|
| 246 |
+
variants={articleVariants}
|
| 247 |
+
initial="enter"
|
| 248 |
+
animate="center"
|
| 249 |
+
exit="exit"
|
| 250 |
+
transition={{ duration: 0.35, ease: 'easeOut' }}
|
| 251 |
+
className={`rounded-xl p-8 lg:p-12 ${readerCls}`}
|
| 252 |
+
onClick={handleArticleClick}
|
| 253 |
+
>
|
| 254 |
+
{currentPage === 1 || content?.page_number === 1 ? (
|
| 255 |
+
renderFirstPage(
|
| 256 |
+
content.content_html_formatted || content.content_html || '',
|
| 257 |
+
fontSize,
|
| 258 |
+
content.title || ''
|
| 259 |
+
)
|
| 260 |
+
) : (
|
| 261 |
+
<>
|
| 262 |
+
<header className="mb-4 pb-0">
|
| 263 |
+
<p className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
|
| 264 |
+
{content.title}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
</p>
|
| 266 |
+
{content.sections?.filter((s: {level: number}) => s.level === 0).map((sec: {title: string}, i: number) => (
|
| 267 |
+
<p key={`l0-${i}`} className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
|
| 268 |
+
{sec.title}
|
| 269 |
+
</p>
|
| 270 |
+
))}
|
| 271 |
+
{content.sections?.filter((s: {level: number}) => s.level === 1).map((sec: {title: string}, i: number) => (
|
| 272 |
+
<p key={`l1-${i}`} className="text-sm text-[#888] tracking-widest uppercase leading-relaxed">
|
| 273 |
+
{sec.title}
|
| 274 |
+
</p>
|
| 275 |
+
))}
|
| 276 |
+
</header>
|
| 277 |
+
<div className="border-b border-[#c8860a]/15 mb-5" />
|
| 278 |
+
{renderContent(content.content_html_formatted || content.content_html || '', fontSize, content.end_markers || [])}
|
| 279 |
+
</>
|
| 280 |
+
)}
|
| 281 |
|
| 282 |
+
{/* Footnotes Section */}
|
| 283 |
+
{content.footnotes && content.footnotes.length > 0 && (
|
| 284 |
+
<div className="mt-16 pt-8 border-t border-[#c8860a]/20">
|
| 285 |
+
<h5 className="text-[#c8860a] text-xs font-bold tracking-widest uppercase mb-6 flex items-center gap-2">
|
| 286 |
+
<svg className="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
| 287 |
+
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M16 4v12l-4-2-4 2V4M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
|
| 288 |
+
</svg>
|
| 289 |
+
เชิงอรรถ
|
| 290 |
+
</h5>
|
| 291 |
+
<div className="space-y-4">
|
| 292 |
+
{content.footnotes.map((fn: any, idx: number) => {
|
| 293 |
+
const cleanId = fn.id.replace(/[()\[\]-]/g, '').trim();
|
| 294 |
+
return (
|
| 295 |
+
<div
|
| 296 |
+
key={idx}
|
| 297 |
+
id={`fn-item-${cleanId}`}
|
| 298 |
+
className={`flex gap-3 text-[13px] leading-relaxed ${mutedCls} hover:opacity-100 transition-all p-2 -m-2 rounded-lg cursor-pointer group`}
|
| 299 |
+
onClick={() => scrollToElement(`ref-${cleanId}`)}
|
| 300 |
+
title="คลิกเพื่อกลับไปยังเนื้อหา"
|
| 301 |
+
>
|
| 302 |
+
<span className="text-[#c8860a] font-bold min-w-[24px] text-right shrink-0 group-hover:scale-110 transition-transform">{fn.id}</span>
|
| 303 |
+
<span className="font-light">{fn.content}</span>
|
| 304 |
+
</div>
|
| 305 |
+
);
|
| 306 |
+
})}
|
| 307 |
+
</div>
|
| 308 |
</div>
|
| 309 |
+
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
|
| 311 |
+
<footer className="mt-12 pt-6 border-t border-[#888]/10 text-center">
|
| 312 |
+
<span className="text-xs text-[#888]">
|
| 313 |
+
— เล่ม {currentVolume} หน้า {currentPage} —
|
| 314 |
+
</span>
|
| 315 |
+
</footer>
|
| 316 |
+
</motion.article>
|
| 317 |
+
) : (
|
| 318 |
+
<motion.div
|
| 319 |
+
key="empty"
|
| 320 |
+
initial={{ opacity: 0 }} animate={{ opacity: 1 }} exit={{ opacity: 0 }}
|
| 321 |
+
className="text-center py-40 text-[#888]"
|
| 322 |
+
>
|
| 323 |
+
<div className="text-5xl mb-6 opacity-40">📖</div>
|
| 324 |
+
<p className="text-lg">ยังไม่ได้เลือกข้อความ</p>
|
| 325 |
+
<p className="text-sm mt-2 opacity-60">
|
| 326 |
+
เลือกเล่มจากสารบัญ หรือค้นหาคำในพระไตรปิฎก
|
| 327 |
+
</p>
|
| 328 |
+
</motion.div>
|
| 329 |
+
)}
|
| 330 |
+
</AnimatePresence>
|
| 331 |
+
)}
|
| 332 |
</div>
|
| 333 |
</div>
|
| 334 |
);
|