dhammawatthumpra commited on
Commit
603ead8
·
1 Parent(s): 5c4239f

feat: complete turbovec search backend integration, ignore tvim file in git, and stabilize mobile UI transitions

Browse files
.gitignore CHANGED
@@ -171,3 +171,7 @@ webapp/.serena/
171
  embedding_cache.db
172
  source_texts.db
173
 
 
 
 
 
 
171
  embedding_cache.db
172
  source_texts.db
173
 
174
+ # ── Turbovec local data ──
175
+ webapp/tipitaka-api/data/
176
+ *.tvim
177
+
webapp/tipitaka-api/app/services/rag_service.py CHANGED
@@ -2,8 +2,6 @@ import logging
2
  import os
3
  from pathlib import Path
4
  from collections import OrderedDict
5
- import qdrant_client
6
- from qdrant_client.http import models as qmodels
7
  import httpx
8
  from app.config import get_settings
9
  from app.database.sqlite_db import get_db
@@ -22,178 +20,105 @@ ST_EMBED_MODEL = "jinaai/jina-embeddings-v5-text-small-retrieval"
22
 
23
  logger = logging.getLogger(__name__)
24
 
 
 
 
 
 
 
25
  class RAGService:
26
  def __init__(self):
27
  settings = get_settings()
28
- self.qdrant_path = settings.QDRANT_PATH
29
- self.snapshot_dir = Path(settings.SNAPSHOT_DIR)
30
  self.ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
31
- self.collections = [settings.QDRANT_COLLECTION, "tipitaka_scripture"]
32
-
33
- self.model = None # Flag: None = not verified, 'ready' = OK
34
  self.reranker = None
35
- self.actual_chunks_col = settings.QDRANT_COLLECTION # Default name
36
- self._embed_cache = OrderedDict() # LRU cache: text → embedding
37
- self._embed_cache_max = 256 # max cached entries
38
- self.client = None
39
- self._st_model = None # sentence-transformers model (fallback)
40
 
41
- # Delegate heavy Qdrant init to a dedicated method
42
- self._init_qdrant()
 
43
 
44
- # Pre-load models (Embedding + Reranker) — safe to fail gracefully
45
  self._load_model()
46
  self._load_reranker()
47
 
48
- def _init_qdrant(self):
49
- """Heavy initialization: detect mode, extract snapshots, create Qdrant client."""
50
- settings = get_settings()
51
- qdrant_url = getattr(settings, "QDRANT_URL", None)
52
-
53
- # Auto-detect Qdrant server if not configured
54
- if not qdrant_url:
55
- try:
56
- with httpx.Client() as client:
57
- response = client.get("http://localhost:6333/healthz", timeout=1.0)
58
- if response.status_code == 200:
59
- qdrant_url = "http://localhost:6333"
60
- logger.info(f"Auto-detected running Qdrant Server at {qdrant_url}")
61
- except Exception:
62
- pass
63
-
64
- is_local = not bool(qdrant_url)
65
-
66
- try:
67
- if is_local:
68
- self._init_qdrant_local()
69
- else:
70
- self._init_qdrant_server(qdrant_url)
71
- except Exception as e:
72
- logger.error(f"RAG Initialization Error: {e}")
73
- self.client = None
74
-
75
- def _init_qdrant_local(self):
76
- """Initialize Qdrant in Local Mode."""
77
- logger.info(f"Initializing Qdrant in Local Mode at {self.qdrant_path}")
78
- storage_path = Path(self.qdrant_path)
79
- storage_path.mkdir(parents=True, exist_ok=True)
80
-
81
- lock_file = storage_path / ".lock"
82
- if lock_file.exists():
83
- try:
84
- logger.warning(f"Removing stale Qdrant lock file: {lock_file}")
85
- lock_file.unlink()
86
- except Exception as e:
87
- logger.error(f"Failed to remove lock file: {e}")
88
-
89
- self.client = qdrant_client.QdrantClient(path=str(self.qdrant_path))
90
- logger.info("Qdrant Local Client initialized.")
91
-
92
- def _init_qdrant_server(self, qdrant_url: str):
93
- """Initialize Qdrant in Server Mode — connect and restore snapshots if needed."""
94
- logger.info(f"Connecting to Qdrant Server at {qdrant_url}")
95
- self.client = qdrant_client.QdrantClient(url=qdrant_url)
96
 
 
 
97
  try:
98
- all_cols = [c.name for c in self.client.get_collections().collections]
 
 
 
 
 
 
99
  except Exception as e:
100
- logger.error(f"Failed to list collections: {e}")
101
- all_cols = []
102
-
103
- settings = get_settings()
104
- for col_name in self.collections:
105
- actual_col = None
106
- if col_name in all_cols:
107
- actual_col = col_name
108
- else:
109
- matches = [c for c in all_cols if c.startswith(f"{col_name}_") or c.startswith(col_name)]
110
- if matches:
111
- actual_col = matches[0]
112
 
113
- if actual_col:
114
- if col_name == settings.QDRANT_COLLECTION:
115
- self.actual_chunks_col = actual_col
116
- else:
117
- snap_filename = f"{col_name}.snapshot"
118
- target_snap = ALLOWED_SNAP_ROOT / snap_filename
119
- if not target_snap.exists():
120
- target_snap = self.snapshot_dir / snap_filename
121
-
122
- if target_snap.exists():
123
- import os
124
- logger.info(f"Restoring server collection '{col_name}' from {target_snap}...")
125
- abs_snap_path = os.path.abspath(target_snap).replace("\\", "/")
126
- if not abs_snap_path.startswith("/"):
127
- abs_snap_path = "/" + abs_snap_path
128
- try:
129
- self.client.recover_snapshot(col_name, location=f"file://{abs_snap_path}")
130
- if col_name == settings.QDRANT_COLLECTION:
131
- self.actual_chunks_col = col_name
132
- except Exception as e:
133
- logger.error(f"Failed to restore: {e}")
134
 
135
  def _load_model(self):
136
- """Verify Ollama embedding service is accessible. If not, preload sentence-transformers."""
137
  if self.model is None:
138
- logger.info(f"Verifying Ollama embedding model...")
139
  try:
140
  r = httpx.get(f"{self.ollama_url}/api/tags", timeout=5)
141
  r.raise_for_status()
142
  self.model = "ready"
143
- logger.info(f"Ollama embedding service ready.")
144
  except Exception as e:
145
- logger.error(f"Ollama not accessible: {e}. Pre-loading sentence-transformers model...")
146
  self.model = "fallback"
147
  try:
148
  settings = get_settings()
149
- logger.info(f"Loading sentence-transformers model: {settings.ST_EMBED_MODEL}")
150
  from sentence_transformers import SentenceTransformer
151
  self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
152
- logger.info("Sentence-transformers model loaded successfully.")
153
  except Exception as st_err:
154
- logger.error(f"Failed to load sentence-transformers model: {st_err}")
155
  self.model = None
156
 
157
  def _load_reranker(self):
158
- """Pre-load the Reranker model using ONNX for CPU performance."""
159
  if self.reranker is None:
160
  logger.info(f"Pre-loading ONNX Reranker from {RERANK_MODEL_PATH}...")
161
  try:
162
  from app.services.onnx_reranker import ONNXReranker
163
  self.reranker = ONNXReranker(RERANK_MODEL_PATH)
164
- logger.info("ONNX Reranker v2 pre-loaded successfully.")
165
  except Exception as e:
166
  logger.error(f"Failed to pre-load ONNX Reranker: {e}")
167
- # Fallback flag
168
  self.reranker = "error"
169
 
170
  def _get_embedding(self, text: str) -> list:
171
- """Get embedding vector — wrapper around batched implementation."""
172
  res = self._get_embeddings([text])
173
  return res[0] if res else []
174
 
175
  def _get_embeddings(self, texts: list[str]) -> list[list[float]]:
176
- """Get embedding vectors for a list of texts using batching."""
177
  if not texts:
178
  return []
179
 
180
  results = [None] * len(texts)
181
- missing_indices = []
182
- missing_texts = []
183
-
184
- # 1. Check LRU cache first
185
  for idx, text in enumerate(texts):
186
  if text in self._embed_cache:
187
- self._embed_cache.move_to_end(text) # Mark as recently used
188
  results[idx] = self._embed_cache[text]
189
  else:
190
  missing_indices.append(idx)
191
  missing_texts.append(text)
192
-
193
  if not missing_texts:
194
  return results
195
 
196
- # 2. Try Ollama batch embedding (only if Ollama is ready)
197
  ollama_failed = True
198
  if self.model == "ready":
199
  try:
@@ -209,199 +134,185 @@ class RAGService:
209
  results[idx] = emb
210
  ollama_failed = False
211
  except Exception as e:
212
- logger.info(f"Ollama batch embedding unavailable ({e}), falling back to sentence-transformers")
213
- ollama_failed = True
214
  self.model = "fallback"
215
 
216
- # 3. Fallback to sentence-transformers batch embedding
217
  if ollama_failed:
218
  try:
219
  if self._st_model is None:
220
  settings = get_settings()
221
- logger.info(f"Loading sentence-transformers model: {settings.ST_EMBED_MODEL}")
222
  from sentence_transformers import SentenceTransformer
223
  self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
224
 
225
- embeddings = self._st_model.encode(missing_texts, normalize_embeddings=True, show_progress_bar=False).tolist()
 
 
226
  for idx, emb in zip(missing_indices, embeddings):
227
  self._store_cache(texts[idx], emb)
228
  results[idx] = emb
229
  return results
230
  except Exception as e:
231
- logger.error(f"sentence-transformers batch embedding failed: {e}")
232
  for idx in missing_indices:
233
  results[idx] = []
234
  return results
235
- return results
236
 
 
237
 
238
  def _store_cache(self, text: str, embedding: list):
239
- """Store embedding in LRU cache."""
240
  self._embed_cache[text] = embedding
241
  if len(self._embed_cache) > self._embed_cache_max:
242
- self._embed_cache.popitem(last=False) # Remove oldest (LRU)
 
 
243
 
244
  async def query(self, text: str, n_results: int = 10, threshold: float = 0.2) -> str:
245
  """
246
- Hybrid Search Strategy:
247
- 1. FTS5 Search (SQLite) for exact keyword matches.
248
- 2. Vector Search (Qdrant) for semantic relevance.
249
- 3. Merge & Deduplicate.
250
- 4. Rerank via Jina v2 ONNX.
251
  """
252
- if not self.client:
253
  return ""
254
-
255
- # Ensure models are ready
256
  if self.model is None:
257
  await anyio.to_thread.run_sync(self._load_model)
258
  if self.reranker is None:
259
  await anyio.to_thread.run_sync(self._load_reranker)
260
 
261
  candidates = []
262
- seen_keys = set() # For deduplication (vol_page)
263
 
264
  try:
265
- # --- PHASE 1: FTS5 Search (SQLite) ---
266
- logger.info(f"Starting FTS5 search for: {text[:30]}...")
 
267
  def _blocking_fts():
268
- fts_results = []
269
  db = get_db()
270
  with db.get_connection() as conn:
271
- # Search pages_fts and join with pages for metadata
272
- # We limit to 30 for performance
273
- query_sql = """
274
- SELECT id, volume_id, page_number, content_text
275
- FROM pages
276
- WHERE id IN (
277
- SELECT rowid FROM pages_fts
278
- WHERE pages_fts MATCH ?
279
- LIMIT 30
280
- )
281
- """
282
- # Sanitize FTS query: wrap in quotes for literal or keep simple
283
- sanitized_query = text.replace('"', '').strip()
284
- if not sanitized_query: return []
285
-
286
- # Split into tokens by whitespace and append * for prefix matching
287
- tokens = [t.strip() for t in sanitized_query.split() if t.strip()]
288
- processed_tokens = []
289
- for t in tokens:
290
- if "*" not in t:
291
- processed_tokens.append(f'"{t}"*')
292
- else:
293
- processed_tokens.append(t)
294
- fts_query = " AND ".join(processed_tokens)
295
-
296
  try:
297
- cursor = conn.execute(query_sql, (fts_query,))
298
- for row in cursor.fetchall():
299
- key = f"{row['volume_id']}_{row['page_number']}"
300
- fts_results.append({
301
- "id": row['id'],
302
- "score": 0.9, # High initial score for FTS matches
 
 
 
 
 
 
 
303
  "payload": {
304
- "volume": row['volume_id'],
305
- "page": row['page_number'],
306
- "content": row['content_text']
307
  },
308
- "key": key
309
- })
 
 
310
  except Exception as e:
311
- logger.warning(f"FTS5 query failed: {e}")
312
- return fts_results
313
 
314
- fts_candidates = await anyio.to_thread.run_sync(_blocking_fts)
315
- for cand in fts_candidates:
316
- if cand['key'] not in seen_keys:
317
- candidates.append(cand)
318
- seen_keys.add(cand['key'])
319
-
320
- # --- PHASE 2: Vector Search (Qdrant) ---
321
- logger.info(f"Starting Vector search for: {text[:30]}...")
322
- def _blocking_vector():
323
- query_vector = self._get_embedding(text)
324
- if hasattr(self.client, "search"):
325
- hits = self.client.search(
326
- collection_name=self.actual_chunks_col,
327
- query_vector=("dense", query_vector),
328
- limit=30,
329
- with_payload=True,
330
- score_threshold=threshold
331
- )
332
- else:
333
- response = self.client.query_points(
334
- collection_name=self.actual_chunks_col,
335
- query=query_vector,
336
- using="dense",
337
- limit=30,
338
- with_payload=True,
339
- score_threshold=threshold
340
- )
341
- hits = response.points
342
-
343
- vec_results = []
344
- for hit in hits:
345
- vol = hit.payload.get("volume", hit.payload.get("volume_id"))
346
- page = hit.payload.get("page", hit.payload.get("page_number"))
347
- key = f"{vol}_{page}"
348
- vec_results.append({
349
- "id": hit.id,
350
- "score": hit.score,
351
- "payload": hit.payload,
352
- "key": key
353
- })
354
- return vec_results
355
-
356
- vector_candidates = await anyio.to_thread.run_sync(_blocking_vector)
357
- for cand in vector_candidates:
358
- if cand['key'] not in seen_keys:
359
  candidates.append(cand)
360
- seen_keys.add(cand['key'])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
361
 
362
  if not candidates:
363
  return ""
364
 
365
- # --- PHASE 3: Rerank stage ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
366
  if self.reranker and self.reranker != "error":
367
- logger.info(f"Reranking {len(candidates)} hybrid candidates...")
368
-
369
  def _blocking_rerank():
370
- # Prepare pairs: (query, passage)
371
- passages = [c['payload'].get("content", "")[:1000] for c in candidates]
372
  pairs = [[text, p] for p in passages]
373
-
374
  with torch.no_grad():
375
  scores = self.reranker.predict(pairs, show_progress_bar=False, batch_size=4)
376
-
377
  for i, cand in enumerate(candidates):
378
- cand['rerank_score'] = float(scores[i])
379
-
380
- return sorted(candidates, key=lambda x: x.get('rerank_score', 0), reverse=True)
381
 
382
  candidates = await anyio.to_thread.run_sync(_blocking_rerank)
383
-
384
- # --- PHASE 4: Formatting ---
385
- final_results = candidates[:n_results]
386
  context_parts = []
387
- for cand in final_results:
388
- payload = cand['payload']
389
  content = payload.get("content", "").strip()
390
- if not content: continue
391
-
392
- vol = payload.get("volume")
393
  page = payload.get("page")
394
-
395
- # Cleanup and formatting
396
- clean_content = re.sub(r"---.*?---", "", content, flags=re.DOTALL).strip()
397
- clean_content = re.sub(r"^\d{3}\s+", "", clean_content, flags=re.MULTILINE)
398
- clean_content = clean_content[:2000]
399
-
400
- context_parts.append(f"[เล่ม {vol} หน้า {page}]\n{clean_content}")
401
-
402
- return "\n---\n".join(context_parts) if context_parts else ""
403
-
404
  except Exception as e:
405
  logger.error(f"Hybrid query error: {e}")
406
  return ""
407
-
 
2
  import os
3
  from pathlib import Path
4
  from collections import OrderedDict
 
 
5
  import httpx
6
  from app.config import get_settings
7
  from app.database.sqlite_db import get_db
 
20
 
21
  logger = logging.getLogger(__name__)
22
 
23
+
24
+ def _decode_tv_id(uid: int) -> tuple[int, int]:
25
+ """Decode turbovec uint64 ID back to (volume, page). Encoding: vol*100000+page."""
26
+ return uid // 100_000, uid % 100_000
27
+
28
+
29
  class RAGService:
30
  def __init__(self):
31
  settings = get_settings()
 
 
32
  self.ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
33
+
34
+ self.model = None # None=not verified, 'ready'=Ollama OK, 'fallback'=ST
 
35
  self.reranker = None
36
+ self._embed_cache = OrderedDict()
37
+ self._embed_cache_max = 256
38
+ self._st_model = None
39
+ self.tv_index = None # turbovec IdMapIndex
 
40
 
41
+ # Turbovec index path (DATA_DIR/tipitaka_chunks.tvim)
42
+ data_dir = Path(settings.DATA_DIR)
43
+ self._tv_path = str(data_dir / "tipitaka_chunks.tvim")
44
 
45
+ self._init_turbovec()
46
  self._load_model()
47
  self._load_reranker()
48
 
49
+ # ── Turbovec init ────────────────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
 
51
+ def _init_turbovec(self):
52
+ """Load turbovec IdMapIndex from disk."""
53
  try:
54
+ from turbovec import IdMapIndex
55
+ if not Path(self._tv_path).exists():
56
+ logger.warning(f"turbovec index not found at {self._tv_path} — vector search disabled")
57
+ return
58
+ logger.info(f"Loading turbovec index from {self._tv_path}...")
59
+ self.tv_index = IdMapIndex.load(self._tv_path)
60
+ logger.info("turbovec index loaded ✅")
61
  except Exception as e:
62
+ logger.error(f"Failed to load turbovec index: {e}")
63
+ self.tv_index = None
 
 
 
 
 
 
 
 
 
 
64
 
65
+ # ── Embedding ────────────────────────────────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
 
67
  def _load_model(self):
68
+ """Verify Ollama or preload sentence-transformers as fallback."""
69
  if self.model is None:
70
+ logger.info("Verifying Ollama embedding model...")
71
  try:
72
  r = httpx.get(f"{self.ollama_url}/api/tags", timeout=5)
73
  r.raise_for_status()
74
  self.model = "ready"
75
+ logger.info("Ollama embedding service ready.")
76
  except Exception as e:
77
+ logger.error(f"Ollama not accessible: {e}. Pre-loading sentence-transformers...")
78
  self.model = "fallback"
79
  try:
80
  settings = get_settings()
 
81
  from sentence_transformers import SentenceTransformer
82
  self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
83
+ logger.info("Sentence-transformers model loaded ")
84
  except Exception as st_err:
85
+ logger.error(f"Failed to load sentence-transformers: {st_err}")
86
  self.model = None
87
 
88
  def _load_reranker(self):
89
+ """Pre-load ONNX Reranker."""
90
  if self.reranker is None:
91
  logger.info(f"Pre-loading ONNX Reranker from {RERANK_MODEL_PATH}...")
92
  try:
93
  from app.services.onnx_reranker import ONNXReranker
94
  self.reranker = ONNXReranker(RERANK_MODEL_PATH)
95
+ logger.info("ONNX Reranker pre-loaded ")
96
  except Exception as e:
97
  logger.error(f"Failed to pre-load ONNX Reranker: {e}")
 
98
  self.reranker = "error"
99
 
100
  def _get_embedding(self, text: str) -> list:
 
101
  res = self._get_embeddings([text])
102
  return res[0] if res else []
103
 
104
  def _get_embeddings(self, texts: list[str]) -> list[list[float]]:
 
105
  if not texts:
106
  return []
107
 
108
  results = [None] * len(texts)
109
+ missing_indices, missing_texts = [], []
110
+
 
 
111
  for idx, text in enumerate(texts):
112
  if text in self._embed_cache:
113
+ self._embed_cache.move_to_end(text)
114
  results[idx] = self._embed_cache[text]
115
  else:
116
  missing_indices.append(idx)
117
  missing_texts.append(text)
118
+
119
  if not missing_texts:
120
  return results
121
 
 
122
  ollama_failed = True
123
  if self.model == "ready":
124
  try:
 
134
  results[idx] = emb
135
  ollama_failed = False
136
  except Exception as e:
137
+ logger.info(f"Ollama batch embedding unavailable ({e}), falling back to ST")
 
138
  self.model = "fallback"
139
 
 
140
  if ollama_failed:
141
  try:
142
  if self._st_model is None:
143
  settings = get_settings()
 
144
  from sentence_transformers import SentenceTransformer
145
  self._st_model = SentenceTransformer(settings.ST_EMBED_MODEL, trust_remote_code=True)
146
 
147
+ embeddings = self._st_model.encode(
148
+ missing_texts, normalize_embeddings=True, show_progress_bar=False
149
+ ).tolist()
150
  for idx, emb in zip(missing_indices, embeddings):
151
  self._store_cache(texts[idx], emb)
152
  results[idx] = emb
153
  return results
154
  except Exception as e:
155
+ logger.error(f"ST batch embedding failed: {e}")
156
  for idx in missing_indices:
157
  results[idx] = []
158
  return results
 
159
 
160
+ return results
161
 
162
  def _store_cache(self, text: str, embedding: list):
 
163
  self._embed_cache[text] = embedding
164
  if len(self._embed_cache) > self._embed_cache_max:
165
+ self._embed_cache.popitem(last=False)
166
+
167
+ # ── Query (Hybrid: FTS5 + turbovec) ─────────────────────────────
168
 
169
  async def query(self, text: str, n_results: int = 10, threshold: float = 0.2) -> str:
170
  """
171
+ Hybrid Search:
172
+ 1. FTS5 (SQLite) keyword matches
173
+ 2. turbovec semantic vector search
174
+ 3. Merge & Deduplicate
175
+ 4. Rerank via Jina v2 ONNX
176
  """
177
+ if self.tv_index is None and self.model is None:
178
  return ""
179
+
 
180
  if self.model is None:
181
  await anyio.to_thread.run_sync(self._load_model)
182
  if self.reranker is None:
183
  await anyio.to_thread.run_sync(self._load_reranker)
184
 
185
  candidates = []
186
+ seen_keys = set()
187
 
188
  try:
189
+ # ── Phase 1: FTS5 ────────────────────────────────────
190
+ logger.info(f"FTS5 search: {text[:40]}...")
191
+
192
  def _blocking_fts():
 
193
  db = get_db()
194
  with db.get_connection() as conn:
195
+ sanitized = text.replace('"', '').strip()
196
+ if not sanitized:
197
+ return []
198
+ tokens = [t for t in sanitized.split() if t]
199
+ fts_query = " AND ".join(
200
+ f'"{t}"*' if "*" not in t else t for t in tokens
201
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
  try:
203
+ cursor = conn.execute("""
204
+ SELECT id, volume_id, page_number, content_text
205
+ FROM pages
206
+ WHERE id IN (
207
+ SELECT rowid FROM pages_fts
208
+ WHERE pages_fts MATCH ?
209
+ LIMIT 30
210
+ )
211
+ """, (fts_query,))
212
+ return [
213
+ {
214
+ "id": r["id"],
215
+ "score": 0.9,
216
  "payload": {
217
+ "volume": r["volume_id"],
218
+ "page": r["page_number"],
219
+ "content": r["content_text"],
220
  },
221
+ "key": f"{r['volume_id']}_{r['page_number']}",
222
+ }
223
+ for r in cursor.fetchall()
224
+ ]
225
  except Exception as e:
226
+ logger.warning(f"FTS5 failed: {e}")
227
+ return []
228
 
229
+ for cand in await anyio.to_thread.run_sync(_blocking_fts):
230
+ if cand["key"] not in seen_keys:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
231
  candidates.append(cand)
232
+ seen_keys.add(cand["key"])
233
+
234
+ # ── Phase 2: turbovec vector search ──────────────────
235
+ if self.tv_index is not None:
236
+ logger.info(f"turbovec search: {text[:40]}...")
237
+
238
+ def _blocking_vector():
239
+ import numpy as np
240
+ query_vec = self._get_embedding(text)
241
+ if not query_vec:
242
+ return []
243
+ q_np = np.array([query_vec], dtype=np.float32)
244
+ scores, ids = self.tv_index.search(q_np, k=30)
245
+ vec_results = []
246
+ for score, uid in zip(scores[0], ids[0]):
247
+ if score < threshold:
248
+ continue
249
+ vol, page = _decode_tv_id(int(uid))
250
+ vec_results.append({
251
+ "id": int(uid),
252
+ "score": float(score),
253
+ "payload": {"volume": vol, "page": page, "content": ""},
254
+ "key": f"{vol}_{page}",
255
+ })
256
+ return vec_results
257
+
258
+ for cand in await anyio.to_thread.run_sync(_blocking_vector):
259
+ if cand["key"] not in seen_keys:
260
+ candidates.append(cand)
261
+ seen_keys.add(cand["key"])
262
 
263
  if not candidates:
264
  return ""
265
 
266
+ # Enrich missing content payloads from SQLite (e.g. for turbovec candidates)
267
+ missing_content_cands = [c for c in candidates if not c["payload"].get("content")]
268
+ if missing_content_cands:
269
+ def _blocking_enrich():
270
+ db = get_db()
271
+ with db.get_connection() as conn:
272
+ cursor = conn.cursor()
273
+ for cand in missing_content_cands:
274
+ try:
275
+ cursor.execute(
276
+ "SELECT content_text FROM pages WHERE volume_id = ? AND page_number = ?",
277
+ (cand["payload"]["volume"], cand["payload"]["page"])
278
+ )
279
+ r = cursor.fetchone()
280
+ if r:
281
+ cand["payload"]["content"] = r["content_text"]
282
+ except Exception as e:
283
+ logger.warning(f"Failed to fetch content for vol={cand['payload']['volume']} page={cand['payload']['page']}: {e}")
284
+ await anyio.to_thread.run_sync(_blocking_enrich)
285
+
286
+ # ── Phase 3: Rerank ───────────────────────────────────
287
  if self.reranker and self.reranker != "error":
288
+ logger.info(f"Reranking {len(candidates)} candidates...")
289
+
290
  def _blocking_rerank():
291
+ passages = [c["payload"].get("content", "")[:1000] for c in candidates]
 
292
  pairs = [[text, p] for p in passages]
 
293
  with torch.no_grad():
294
  scores = self.reranker.predict(pairs, show_progress_bar=False, batch_size=4)
 
295
  for i, cand in enumerate(candidates):
296
+ cand["rerank_score"] = float(scores[i])
297
+ return sorted(candidates, key=lambda x: x.get("rerank_score", 0), reverse=True)
 
298
 
299
  candidates = await anyio.to_thread.run_sync(_blocking_rerank)
300
+
301
+ # ── Phase 4: Format ───────────────────────────────────
 
302
  context_parts = []
303
+ for cand in candidates[:n_results]:
304
+ payload = cand["payload"]
305
  content = payload.get("content", "").strip()
306
+ if not content:
307
+ continue
308
+ vol = payload.get("volume")
309
  page = payload.get("page")
310
+ clean = re.sub(r"---.*?---", "", content, flags=re.DOTALL).strip()
311
+ clean = re.sub(r"^\d{3}\s+", "", clean, flags=re.MULTILINE)
312
+ context_parts.append(f"[เล่ม {vol} หน้า {page}]\n{clean[:2000]}")
313
+
314
+ return "\n---\n".join(context_parts)
315
+
 
 
 
 
316
  except Exception as e:
317
  logger.error(f"Hybrid query error: {e}")
318
  return ""
 
webapp/tipitaka-api/app/services/search_service.py CHANGED
@@ -193,181 +193,65 @@ class SearchService:
193
  return self.highlight_multiple(text, [keyword]) if keyword else text
194
 
195
  async def _get_vector_results_batch(self, query_texts: List[str], query_vectors: List[List[float]], limit: int = 30) -> List[List[dict]]:
196
- """Fetch semantic vector results from Qdrant in batch."""
197
- if not self.rag_service or not self.rag_service.client or not query_vectors:
198
  return [[] for _ in query_texts]
199
 
200
  import anyio
201
- from qdrant_client.http import models as qmodels
202
 
203
  def _blocking():
204
- col = self.rag_service.actual_chunks_col
205
- client = self.rag_service.client
206
-
207
- # Map request index to original query indices
208
  valid_indices = [idx for idx, vec in enumerate(query_vectors) if vec]
209
  if not valid_indices:
210
  return [[] for _ in query_texts]
211
 
212
- final_results = [[] for _ in query_texts]
213
-
214
- # 1. Try modern query_batch_points API first (Qdrant Client 1.8+)
215
- if hasattr(client, "query_batch_points"):
216
- requests = [
217
- qmodels.QueryRequest(
218
- query=query_vectors[idx],
219
- using="dense",
220
- limit=limit,
221
- with_payload=True,
222
- score_threshold=0.2
223
- ) for idx in valid_indices
224
- ]
225
- try:
226
- batch_responses = client.query_batch_points(
227
- collection_name=col,
228
- requests=requests
229
- )
230
- for req_idx, resp in zip(valid_indices, batch_responses):
231
- hits = resp.points if hasattr(resp, "points") else resp
232
- vec_results = []
233
- for hit in hits:
234
- vol = hit.payload.get("volume", hit.payload.get("volume_id"))
235
- page = hit.payload.get("page", hit.payload.get("page_number"))
236
- content = hit.payload.get("content", "")
237
- vec_results.append({
238
- "volume_id": vol,
239
- "page_number": page,
240
- "content_text": content,
241
- "score": hit.score
242
- })
243
- final_results[req_idx] = vec_results
244
- return final_results
245
- except Exception as e:
246
- import logging
247
- logging.getLogger(__name__).warning(f"Qdrant query_batch_points failed: {e}")
248
-
249
- # 2. Try legacy search_batch API if available
250
- if hasattr(client, "search_batch"):
251
- requests = [
252
- qmodels.SearchRequest(
253
- vector=qmodels.NamedVector(name="dense", vector=query_vectors[idx]),
254
- limit=limit,
255
- with_payload=True,
256
- score_threshold=0.2
257
- ) for idx in valid_indices
258
- ]
259
- try:
260
- batch_responses = client.search_batch(
261
- collection_name=col,
262
- requests=requests
263
- )
264
- for req_idx, hits in zip(valid_indices, batch_responses):
265
- vec_results = []
266
- for hit in hits:
267
- vol = hit.payload.get("volume", hit.payload.get("volume_id"))
268
- page = hit.payload.get("page", hit.payload.get("page_number"))
269
- content = hit.payload.get("content", "")
270
- vec_results.append({
271
- "volume_id": vol,
272
- "page_number": page,
273
- "content_text": content,
274
- "score": hit.score
275
- })
276
- final_results[req_idx] = vec_results
277
- return final_results
278
- except Exception as e:
279
- import logging
280
- logging.getLogger(__name__).warning(f"Qdrant search_batch failed: {e}")
281
-
282
- # 3. Fallback: run sequential queries in blocking thread
283
- for idx in valid_indices:
284
- try:
285
- if hasattr(client, "search"):
286
- hits = client.search(
287
- collection_name=col,
288
- query_vector=("dense", query_vectors[idx]),
289
- limit=limit,
290
- with_payload=True,
291
- score_threshold=0.2
292
- )
293
- else:
294
- response = client.query_points(
295
- collection_name=col,
296
- query=query_vectors[idx],
297
- using="dense",
298
- limit=limit,
299
- with_payload=True,
300
- score_threshold=0.2
301
- )
302
- hits = response.points
303
-
304
- vec_results = []
305
- for hit in hits:
306
- vol = hit.payload.get("volume", hit.payload.get("volume_id"))
307
- page = hit.payload.get("page", hit.payload.get("page_number"))
308
- content = hit.payload.get("content", "")
309
- vec_results.append({
310
- "volume_id": vol,
311
- "page_number": page,
312
- "content_text": content,
313
- "score": hit.score
314
- })
315
- final_results[idx] = vec_results
316
- except Exception as e:
317
- import logging
318
- logging.getLogger(__name__).warning(f"Qdrant single query fallback failed: {e}")
319
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  return final_results
321
 
322
  return await anyio.to_thread.run_sync(_blocking)
323
 
324
  async def _get_vector_results(self, query_text: str, limit: int = 30) -> List[dict]:
325
- """Fetch semantic vector results from Qdrant via RAGService."""
326
- if not self.rag_service or not self.rag_service.client:
327
  return []
328
 
329
  import anyio
 
330
 
331
  def _blocking():
332
  query_vector = self.rag_service._get_embedding(query_text)
333
  if not query_vector:
334
  return []
335
-
336
- col = self.rag_service.actual_chunks_col
337
- try:
338
- if hasattr(self.rag_service.client, "search"):
339
- hits = self.rag_service.client.search(
340
- collection_name=col,
341
- query_vector=("dense", query_vector),
342
- limit=limit,
343
- with_payload=True,
344
- score_threshold=0.2
345
- )
346
- else:
347
- response = self.rag_service.client.query_points(
348
- collection_name=col,
349
- query=query_vector,
350
- using="dense",
351
- limit=limit,
352
- with_payload=True,
353
- score_threshold=0.2
354
- )
355
- hits = response.points
356
- except Exception as e:
357
- import logging
358
- logging.getLogger(__name__).warning(f"Qdrant query failed: {e}")
359
- return []
360
-
361
  vec_results = []
362
- for hit in hits:
363
- vol = hit.payload.get("volume", hit.payload.get("volume_id"))
364
- page = hit.payload.get("page", hit.payload.get("page_number"))
365
- content = hit.payload.get("content", "")
366
  vec_results.append({
367
- "volume_id": vol,
368
- "page_number": page,
369
- "content_text": content,
370
- "score": hit.score
371
  })
372
  return vec_results
373
 
@@ -495,6 +379,24 @@ class SearchService:
495
  top_keys = sorted(rrf_scores, key=rrf_scores.get, reverse=True)[:15]
496
  top_candidates = [candidate_data[ky] for ky in top_keys]
497
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
498
  # ── 5. Reranking (using Jina Reranker v2 ONNX) ────────────
499
  if top_candidates:
500
  final_results = await self._rerank_candidates(corrected_q, top_candidates)
 
193
  return self.highlight_multiple(text, [keyword]) if keyword else text
194
 
195
  async def _get_vector_results_batch(self, query_texts: List[str], query_vectors: List[List[float]], limit: int = 30) -> List[List[dict]]:
196
+ """Fetch semantic vector results from turbovec in batch."""
197
+ if not self.rag_service or not self.rag_service.tv_index or not query_vectors:
198
  return [[] for _ in query_texts]
199
 
200
  import anyio
201
+ import numpy as np
202
 
203
  def _blocking():
204
+ tv = self.rag_service.tv_index
 
 
 
205
  valid_indices = [idx for idx, vec in enumerate(query_vectors) if vec]
206
  if not valid_indices:
207
  return [[] for _ in query_texts]
208
 
209
+ # Stack valid query vectors for batch search
210
+ vecs = np.array([query_vectors[idx] for idx in valid_indices], dtype=np.float32)
211
+ all_scores, all_ids = tv.search(vecs, k=limit)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
 
213
+ final_results = [[] for _ in query_texts]
214
+ for req_pos, req_idx in enumerate(valid_indices):
215
+ vec_results = []
216
+ for score, uid in zip(all_scores[req_pos], all_ids[req_pos]):
217
+ if float(score) < 0.2:
218
+ continue
219
+ vol, page = uid // 100_000, uid % 100_000
220
+ vec_results.append({
221
+ "volume_id": int(vol),
222
+ "page_number": int(page),
223
+ "content_text": "", # fetched from SQLite later
224
+ "score": float(score)
225
+ })
226
+ final_results[req_idx] = vec_results
227
  return final_results
228
 
229
  return await anyio.to_thread.run_sync(_blocking)
230
 
231
  async def _get_vector_results(self, query_text: str, limit: int = 30) -> List[dict]:
232
+ """Fetch semantic vector results from turbovec via RAGService."""
233
+ if not self.rag_service or not self.rag_service.tv_index:
234
  return []
235
 
236
  import anyio
237
+ import numpy as np
238
 
239
  def _blocking():
240
  query_vector = self.rag_service._get_embedding(query_text)
241
  if not query_vector:
242
  return []
243
+ q_np = np.array([query_vector], dtype=np.float32)
244
+ scores, ids = self.rag_service.tv_index.search(q_np, k=limit)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  vec_results = []
246
+ for score, uid in zip(scores[0], ids[0]):
247
+ if float(score) < 0.2:
248
+ continue
249
+ vol, page = uid // 100_000, uid % 100_000
250
  vec_results.append({
251
+ "volume_id": int(vol),
252
+ "page_number": int(page),
253
+ "content_text": "", # fetched from SQLite later
254
+ "score": float(score)
255
  })
256
  return vec_results
257
 
 
379
  top_keys = sorted(rrf_scores, key=rrf_scores.get, reverse=True)[:15]
380
  top_candidates = [candidate_data[ky] for ky in top_keys]
381
 
382
+ # Enrich missing content_text from SQLite (e.g. for turbovec vector results)
383
+ missing_content_cands = [c for c in top_candidates if not c.get("content_text")]
384
+ if missing_content_cands:
385
+ try:
386
+ with self.db.get_connection() as conn:
387
+ cursor = conn.cursor()
388
+ for c in missing_content_cands:
389
+ cursor.execute(
390
+ "SELECT content_text FROM pages WHERE volume_id = ? AND page_number = ?",
391
+ (c["volume_id"], c["page_number"])
392
+ )
393
+ row = cursor.fetchone()
394
+ if row:
395
+ c["content_text"] = row["content_text"]
396
+ except Exception as e:
397
+ import logging
398
+ logging.getLogger(__name__).warning(f"Enriching candidates content failed: {e}")
399
+
400
  # ── 5. Reranking (using Jina Reranker v2 ONNX) ────────────
401
  if top_candidates:
402
  final_results = await self._rerank_candidates(corrected_q, top_candidates)
webapp/tipitaka-api/benchmark_qdrant_vs_turbovec.py ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ import numpy as np
3
+ from qdrant_client import QdrantClient
4
+ from turbovec import IdMapIndex
5
+ import sys
6
+ import os
7
+
8
+ # Configurations
9
+ QDRANT_URL = "http://127.0.0.1:6333"
10
+ COLLECTION = "tipitaka_chunks"
11
+ TV_PATH = "F:/_Ai/Tipitaka-AI-Expert/RAG/webapp/tipitaka-api/data/tipitaka_chunks.tvim"
12
+ EMBED_DIM = 1024
13
+ NUM_QUERIES = 100
14
+ K = 30
15
+
16
+ def benchmark():
17
+ print("=" * 60)
18
+ print(" BENCHMARK: QDRANT VS TURBOVEC (SEARCH ONLY)")
19
+ print("=" * 60)
20
+
21
+ # 1. Generate random query vectors to isolate search speed from embedding latency
22
+ np.random.seed(42)
23
+ query_vectors = np.random.randn(NUM_QUERIES, EMBED_DIM).astype(np.float32)
24
+ # Normalize for cosine similarity
25
+ norms = np.linalg.norm(query_vectors, axis=1, keepdims=True)
26
+ query_vectors = query_vectors / norms
27
+
28
+ # 2. Turbovec initialization
29
+ t_start = time.time()
30
+ tv_index = IdMapIndex.load(TV_PATH)
31
+ t_tv_load = (time.time() - t_start) * 1000
32
+ print(f"Turbovec index loaded from disk in: {t_tv_load:.2f} ms")
33
+
34
+ # 3. Qdrant initialization
35
+ t_start = time.time()
36
+ qdrant_client = QdrantClient(url=QDRANT_URL)
37
+ # Warm up / check connection
38
+ try:
39
+ qdrant_client.get_collection(COLLECTION)
40
+ t_qd_connect = (time.time() - t_start) * 1000
41
+ print(f"Connected to Qdrant local server in: {t_qd_connect:.2f} ms")
42
+ except Exception as e:
43
+ print(f"FAILED to connect to Qdrant: {e}")
44
+ return
45
+
46
+ print(f"Running {NUM_QUERIES} queries (k={K}) on both databases...")
47
+
48
+ # --- Benchmark Turbovec ---
49
+ tv_times = []
50
+ # Warmup
51
+ tv_index.search(query_vectors[0:1], k=K)
52
+
53
+ for i in range(NUM_QUERIES):
54
+ q = query_vectors[i:i+1]
55
+ start = time.perf_counter()
56
+ scores, ids = tv_index.search(q, k=K)
57
+ end = time.perf_counter()
58
+ tv_times.append((end - start) * 1000)
59
+
60
+ # --- Benchmark Qdrant ---
61
+ qd_times = []
62
+ # Warmup
63
+ try:
64
+ qdrant_client.query_points(
65
+ collection_name=COLLECTION,
66
+ query=query_vectors[0].tolist(),
67
+ using="dense",
68
+ limit=K,
69
+ with_payload=False
70
+ )
71
+ except Exception as e:
72
+ print(f"Qdrant warmup query_points failed: {e}")
73
+ return
74
+
75
+ for i in range(NUM_QUERIES):
76
+ q_list = query_vectors[i].tolist()
77
+ start = time.perf_counter()
78
+ qdrant_client.query_points(
79
+ collection_name=COLLECTION,
80
+ query=q_list,
81
+ using="dense",
82
+ limit=K,
83
+ with_payload=False
84
+ )
85
+ end = time.perf_counter()
86
+ qd_times.append((end - start) * 1000)
87
+
88
+ # 4. Results
89
+ print("\n" + "=" * 60)
90
+ print(" LATENCY RESULTS (ms)")
91
+ print("=" * 60)
92
+ print(f"{'Metric':<15} | {'Turbovec (4-bit SQ)':<20} | {'Qdrant Server':<20}")
93
+ print("-" * 60)
94
+ print(f"{'Average (Mean)':<15} | {np.mean(tv_times):>17.3f} ms | {np.mean(qd_times):>17.3f} ms")
95
+ print(f"{'Median (p50)':<15} | {np.percentile(tv_times, 50):>17.3f} ms | {np.percentile(qd_times, 50):>17.3f} ms")
96
+ print(f"{'90th Percentile':<15} | {np.percentile(tv_times, 90):>17.3f} ms | {np.percentile(qd_times, 90):>17.3f} ms")
97
+ print(f"{'Min':<15} | {np.min(tv_times):>17.3f} ms | {np.min(qd_times):>17.3f} ms")
98
+ print(f"{'Max':<15} | {np.max(tv_times):>17.3f} ms | {np.max(qd_times):>17.3f} ms")
99
+ print("=" * 60)
100
+
101
+ # Throughput
102
+ print(f"Throughput (QPS):")
103
+ print(f" • Turbovec: {NUM_QUERIES / (sum(tv_times)/1000):.1f} queries/sec")
104
+ print(f" • Qdrant: {NUM_QUERIES / (sum(qd_times)/1000):.1f} queries/sec")
105
+ print("=" * 60)
106
+
107
+ # Disk Space / Memory comparison (estimated)
108
+ print("Resource Footprint Comparison:")
109
+ print(" • Turbovec Index File: 13.2 MB")
110
+ print(" • Qdrant Local Server Storage: ~600+ MB (snapshots + database files)")
111
+ print(" • Memory Overhead:")
112
+ print(" - Turbovec: Loaded dynamically in Python process memory (~14MB overhead).")
113
+ print(" - Qdrant: Requires running a separate Docker daemon container (~150MB to 300MB RAM idle).")
114
+ print("=" * 60)
115
+
116
+ if __name__ == "__main__":
117
+ benchmark()
webapp/tipitaka-api/build_turbovec_index.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Build turbovec IdMapIndex from Qdrant tipitaka_chunks collection.
3
+
4
+ ID encoding: volume * 100000 + page (uint64, fully reversible)
5
+ Output: tipitaka_chunks.tvim (turbovec index file)
6
+
7
+ Usage:
8
+ python build_turbovec_index.py
9
+ python build_turbovec_index.py --collection tipitaka_chunks --output tipitaka_chunks.tvim
10
+ """
11
+ import argparse
12
+ import logging
13
+ import time
14
+ import numpy as np
15
+ from qdrant_client import QdrantClient
16
+ from turbovec import IdMapIndex
17
+
18
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
19
+ logger = logging.getLogger(__name__)
20
+
21
+ QDRANT_URL = "http://localhost:6333"
22
+ COLLECTION = "tipitaka_chunks"
23
+ OUTPUT_PATH = "F:/_Ai/Tipitaka-AI-Expert/RAG/webapp/tipitaka-api/data/tipitaka_chunks.tvim"
24
+ EMBED_DIM = 1024
25
+ BATCH_SIZE = 500
26
+
27
+
28
+ def encode_id(volume: str, page: str) -> int:
29
+ """Encode (volume, page) as a single uint64. Reversible via decode_id()."""
30
+ return int(volume) * 100_000 + int(page)
31
+
32
+
33
+ def decode_id(uid: int) -> tuple[int, int]:
34
+ """Decode uint64 back to (volume, page)."""
35
+ return uid // 100_000, uid % 100_000
36
+
37
+
38
+ def main(collection: str, output: str):
39
+ logger.info(f"Connecting to Qdrant at {QDRANT_URL}...")
40
+ client = QdrantClient(url=QDRANT_URL)
41
+
42
+ info = client.get_collection(collection)
43
+ total = info.points_count
44
+ logger.info(f"Collection '{collection}': {total} points, dim={EMBED_DIM}")
45
+
46
+ # Build turbovec index
47
+ index = IdMapIndex(dim=EMBED_DIM, bit_width=4)
48
+
49
+ vectors_batch = []
50
+ ids_batch = []
51
+ processed = 0
52
+ skipped = 0
53
+ offset = None
54
+ start = time.time()
55
+
56
+ while True:
57
+ pts, next_offset = client.scroll(
58
+ collection,
59
+ offset=offset,
60
+ limit=BATCH_SIZE,
61
+ with_payload=True,
62
+ with_vectors=True,
63
+ )
64
+ if not pts:
65
+ break
66
+
67
+ for pt in pts:
68
+ payload = pt.payload or {}
69
+ volume = payload.get("volume", payload.get("volume_number", ""))
70
+ page = payload.get("page", payload.get("page_number", ""))
71
+
72
+ # Get dense vector only
73
+ vec = pt.vector
74
+ if isinstance(vec, dict):
75
+ vec = vec.get("dense")
76
+ if vec is None or not volume or not page:
77
+ skipped += 1
78
+ continue
79
+
80
+ try:
81
+ uid = encode_id(volume, page)
82
+ vectors_batch.append(vec)
83
+ ids_batch.append(uid)
84
+ except (ValueError, TypeError):
85
+ skipped += 1
86
+ continue
87
+
88
+ # Add batch to index
89
+ if vectors_batch:
90
+ vecs_np = np.array(vectors_batch, dtype=np.float32)
91
+ ids_np = np.array(ids_batch, dtype=np.uint64)
92
+ index.add_with_ids(vecs_np, ids_np)
93
+ processed += len(vecs_np)
94
+ vectors_batch.clear()
95
+ ids_batch.clear()
96
+
97
+ elapsed = time.time() - start
98
+ rate = processed / elapsed if elapsed > 0 else 0
99
+ eta = (total - processed) / rate if rate > 0 else 0
100
+ logger.info(f" {processed}/{total} ({100*processed//total}%) | {rate:.0f} pts/s | ETA {eta:.0f}s")
101
+
102
+ if next_offset is None:
103
+ break
104
+ offset = next_offset
105
+
106
+ # Save index
107
+ logger.info(f"\nSaving index to {output}...")
108
+ index.write(output)
109
+
110
+ elapsed_total = time.time() - start
111
+ logger.info("=" * 60)
112
+ logger.info(f"Done in {elapsed_total:.1f}s")
113
+ logger.info(f"Indexed: {processed} | Skipped: {skipped}")
114
+ logger.info(f"Saved: {output}")
115
+ logger.info(f"ID encoding: volume * 100000 + page (decode: v=id//100000, p=id%100000)")
116
+ logger.info("=" * 60)
117
+
118
+ # Quick sanity check
119
+ logger.info("\nSanity check — reload and search with random query...")
120
+ from turbovec import IdMapIndex as TVI
121
+ loaded = TVI.load(output)
122
+ query = np.random.randn(1, EMBED_DIM).astype(np.float32)
123
+ scores, ids = loaded.search(query, k=3)
124
+ for s, uid in zip(scores[0], ids[0]):
125
+ vol, pg = decode_id(int(uid))
126
+ logger.info(f" score={s:.4f} vol={vol} page={pg}")
127
+
128
+ client.close()
129
+
130
+
131
+ if __name__ == "__main__":
132
+ parser = argparse.ArgumentParser()
133
+ parser.add_argument("--collection", default=COLLECTION)
134
+ parser.add_argument("--output", default=OUTPUT_PATH)
135
+ args = parser.parse_args()
136
+
137
+ import os
138
+ os.makedirs(os.path.dirname(args.output), exist_ok=True)
139
+ main(args.collection, args.output)
webapp/tipitaka-api/download_assets.py CHANGED
@@ -20,6 +20,7 @@ DB_PATH = Path(settings.DATABASE_PATH)
20
  QDRANT_DIR = Path(settings.QDRANT_PATH)
21
  SNAPSHOT_DIR = Path(settings.SNAPSHOT_DIR)
22
  MODELS_DIR = Path(__file__).resolve().parent / "models"
 
23
 
24
  BUCKET_ID = "dhammawatthumpra/tipitaka-storage"
25
  RERANK_REPO_ID = "jinaai/jina-reranker-v2-base-multilingual"
@@ -48,10 +49,22 @@ def check_file_exists(path: Path) -> bool:
48
 
49
  def download_files() -> None:
50
  """Download missing files from HF bucket."""
 
51
  hf_token = os.getenv("HF_TOKEN", "")
52
  from huggingface_hub import hf_hub_download
53
 
54
- # 1. Download Core Assets (DB & Snapshots)
 
 
 
 
 
 
 
 
 
 
 
55
  pending_core = []
56
  for remote, local_path_str in BUCKET_FILES:
57
  local_path = Path(local_path_str)
@@ -73,7 +86,6 @@ def download_files() -> None:
73
  repo_type="dataset",
74
  token=hf_token
75
  )
76
- import shutil
77
  shutil.copy(downloaded, dest_path)
78
  except Exception as e:
79
  print(f"FAILED to download {remote}: {e}")
@@ -98,7 +110,6 @@ def download_files() -> None:
98
  filename=filename,
99
  token=hf_token if hf_token else None
100
  )
101
- import shutil
102
  shutil.copy(downloaded, dest_path)
103
  except Exception as e:
104
  print(f"FAILED to download reranker file {filename}: {e}")
@@ -110,6 +121,9 @@ def verify_assets() -> bool:
110
  if not DB_PATH.exists():
111
  missing.append("tipitaka_mcu.db")
112
 
 
 
 
113
  if not (QDRANT_DIR / "meta.json").exists():
114
  missing.append("qdrant_storage/meta.json")
115
 
 
20
  QDRANT_DIR = Path(settings.QDRANT_PATH)
21
  SNAPSHOT_DIR = Path(settings.SNAPSHOT_DIR)
22
  MODELS_DIR = Path(__file__).resolve().parent / "models"
23
+ TV_PATH = Path(settings.DATA_DIR) / "tipitaka_chunks.tvim"
24
 
25
  BUCKET_ID = "dhammawatthumpra/tipitaka-storage"
26
  RERANK_REPO_ID = "jinaai/jina-reranker-v2-base-multilingual"
 
49
 
50
  def download_files() -> None:
51
  """Download missing files from HF bucket."""
52
+ import shutil
53
  hf_token = os.getenv("HF_TOKEN", "")
54
  from huggingface_hub import hf_hub_download
55
 
56
+ # Check and handle Turbovec index file
57
+ if not check_file_exists(TV_PATH):
58
+ packaged_tv = Path(__file__).resolve().parent / "data" / "tipitaka_chunks.tvim"
59
+ if check_file_exists(packaged_tv):
60
+ print(f"Copying packaged Turbovec index: {packaged_tv} -> {TV_PATH}")
61
+ TV_PATH.parent.mkdir(parents=True, exist_ok=True)
62
+ shutil.copy(packaged_tv, TV_PATH)
63
+ else:
64
+ # Fallback: add to bucket files so it attempts downloading from HF Space bucket
65
+ BUCKET_FILES.append(("tipitaka_chunks.tvim", str(TV_PATH)))
66
+
67
+ # 1. Download Core Assets (DB, Snapshots, and optionally Turbovec index)
68
  pending_core = []
69
  for remote, local_path_str in BUCKET_FILES:
70
  local_path = Path(local_path_str)
 
86
  repo_type="dataset",
87
  token=hf_token
88
  )
 
89
  shutil.copy(downloaded, dest_path)
90
  except Exception as e:
91
  print(f"FAILED to download {remote}: {e}")
 
110
  filename=filename,
111
  token=hf_token if hf_token else None
112
  )
 
113
  shutil.copy(downloaded, dest_path)
114
  except Exception as e:
115
  print(f"FAILED to download reranker file {filename}: {e}")
 
121
  if not DB_PATH.exists():
122
  missing.append("tipitaka_mcu.db")
123
 
124
+ if not TV_PATH.exists():
125
+ missing.append("tipitaka_chunks.tvim")
126
+
127
  if not (QDRANT_DIR / "meta.json").exists():
128
  missing.append("qdrant_storage/meta.json")
129
 
webapp/tipitaka-api/requirements.txt CHANGED
@@ -17,3 +17,5 @@ anyio>=4.0.0
17
  pytest>=8.0
18
  pytest-asyncio>=0.24
19
  accelerate>=0.27.0
 
 
 
17
  pytest>=8.0
18
  pytest-asyncio>=0.24
19
  accelerate>=0.27.0
20
+ turbovec>=0.5.3
21
+
webapp/tipitaka-api/test_rag_query.py CHANGED
@@ -18,7 +18,7 @@ async def test_retrieval():
18
  logger.info("Initializing RAGService...")
19
  service = RAGService()
20
 
21
- logger.info(f"Target collection is: {service.actual_chunks_col}")
22
 
23
  queries = [
24
  "พระสุทินเสพเมถุนธรรมกับอดีตภรรยา 3 ครั้ง เพราะยังไม่มีสิกขาบท",
 
18
  logger.info("Initializing RAGService...")
19
  service = RAGService()
20
 
21
+ logger.info(f"Target turbovec index is: {service._tv_path}")
22
 
23
  queries = [
24
  "พระสุทินเสพเมถุนธรรมกับอดีตภรรยา 3 ครั้ง เพราะยังไม่มีสิกขาบท",
webapp/tipitaka-api/tests/test_rag_service.py CHANGED
@@ -14,9 +14,9 @@ class TestEmbeddingCache:
14
  from app.services.rag_service import RAGService
15
  with patch.object(RAGService, '_load_model'), \
16
  patch.object(RAGService, '_load_reranker'), \
17
- patch.object(RAGService, '_init_qdrant'):
18
  service = RAGService()
19
- service.client = MagicMock()
20
  return service
21
 
22
  def test_cache_initialized(self, rag_service):
@@ -48,12 +48,13 @@ class TestEmbeddingCache:
48
 
49
 
50
  class TestRAGServiceInit:
51
- def test_init_graceful_on_qdrant_failure(self):
52
- """RAGService constructor should not crash when Qdrant is unavailable."""
53
  from app.services.rag_service import RAGService
54
  with patch.object(RAGService, '_load_model'), \
55
  patch.object(RAGService, '_load_reranker'), \
56
- patch.object(RAGService, '_init_qdrant') as mock_init:
57
  service = RAGService()
58
  mock_init.assert_called_once()
59
- assert service.client is None # _init_qdrant didn't set it
 
 
14
  from app.services.rag_service import RAGService
15
  with patch.object(RAGService, '_load_model'), \
16
  patch.object(RAGService, '_load_reranker'), \
17
+ patch.object(RAGService, '_init_turbovec'):
18
  service = RAGService()
19
+ service.tv_index = MagicMock()
20
  return service
21
 
22
  def test_cache_initialized(self, rag_service):
 
48
 
49
 
50
  class TestRAGServiceInit:
51
+ def test_init_graceful_on_turbovec_failure(self):
52
+ """RAGService constructor should not crash when turbovec is unavailable."""
53
  from app.services.rag_service import RAGService
54
  with patch.object(RAGService, '_load_model'), \
55
  patch.object(RAGService, '_load_reranker'), \
56
+ patch.object(RAGService, '_init_turbovec') as mock_init:
57
  service = RAGService()
58
  mock_init.assert_called_once()
59
+ assert service.tv_index is None # _init_turbovec didn't set it
60
+
webapp/tipitaka-web/src/components/layout/ReaderPanel.tsx CHANGED
@@ -230,103 +230,105 @@ const ReaderPanel: React.FC = () => {
230
  />
231
  </ErrorBoundary>
232
  <div className="max-w-3xl mx-auto px-4 py-10 lg:py-16">
233
- <AnimatePresence custom={direction} mode="wait">
234
- {loading ? (
235
- <motion.div
236
- key="loading"
237
- initial={{ opacity: 0 }} animate={{ opacity: 1 }} exit={{ opacity: 0 }}
238
- className="flex flex-col items-center justify-center py-48 gap-4"
239
- >
240
- <div className="w-10 h-10 border-4 border-[#c8860a]/20 border-t-[#c8860a] rounded-full animate-spin" />
241
- <p className="text-[#c8860a] text-sm animate-pulse">กำลังอัญเชิญข้อความ…</p>
242
- </motion.div>
243
 
244
- ) : content ? (
245
- <motion.article
246
- key={`${currentVolume}-${currentPage}`}
247
- custom={direction}
248
- variants={articleVariants}
249
- initial="enter"
250
- animate="center"
251
- exit="exit"
252
- transition={{ duration: 0.35, ease: 'easeOut' }}
253
- className={`rounded-xl p-8 lg:p-12 ${readerCls}`}
254
- onClick={handleArticleClick}
255
- >
256
- {currentPage === 1 || content?.page_number === 1 ? (
257
- renderFirstPage(
258
- content.content_html_formatted || content.content_html || '',
259
- fontSize,
260
- content.title || ''
261
- )
262
- ) : (
263
- <>
264
- <header className="mb-4 pb-0">
265
- <p className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
266
- {content.title}
267
- </p>
268
- {content.sections?.filter((s: {level: number}) => s.level === 0).map((sec: {title: string}, i: number) => (
269
- <p key={`l0-${i}`} className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
270
- {sec.title}
271
- </p>
272
- ))}
273
- {content.sections?.filter((s: {level: number}) => s.level === 1).map((sec: {title: string}, i: number) => (
274
- <p key={`l1-${i}`} className="text-sm text-[#888] tracking-widest uppercase leading-relaxed">
275
- {sec.title}
276
  </p>
277
- ))}
278
- </header>
279
- <div className="border-b border-[#c8860a]/15 mb-5" />
280
- {renderContent(content.content_html_formatted || content.content_html || '', fontSize, content.end_markers || [])}
281
- </>
282
- )}
 
 
 
 
 
 
 
 
 
283
 
284
- {/* Footnotes Section */}
285
- {content.footnotes && content.footnotes.length > 0 && (
286
- <div className="mt-16 pt-8 border-t border-[#c8860a]/20">
287
- <h5 className="text-[#c8860a] text-xs font-bold tracking-widest uppercase mb-6 flex items-center gap-2">
288
- <svg className="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
289
- <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M16 4v12l-4-2-4 2V4M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
290
- </svg>
291
- เชิงอรรถ
292
- </h5>
293
- <div className="space-y-4">
294
- {content.footnotes.map((fn: any, idx: number) => {
295
- const cleanId = fn.id.replace(/[()\[\]-]/g, '').trim();
296
- return (
297
- <div
298
- key={idx}
299
- id={`fn-item-${cleanId}`}
300
- className={`flex gap-3 text-[13px] leading-relaxed ${mutedCls} hover:opacity-100 transition-all p-2 -m-2 rounded-lg cursor-pointer group`}
301
- onClick={() => scrollToElement(`ref-${cleanId}`)}
302
- title="คลิกเพื่อกลับไปยังเนื้อหา"
303
- >
304
- <span className="text-[#c8860a] font-bold min-w-[24px] text-right shrink-0 group-hover:scale-110 transition-transform">{fn.id}</span>
305
- <span className="font-light">{fn.content}</span>
306
- </div>
307
- );
308
- })}
 
309
  </div>
310
- </div>
311
- )}
312
-
313
- <footer className="mt-12 pt-6 border-t border-[#888]/10 text-center">
314
- <span className="text-xs text-[#888]">
315
- — เล่ม {currentVolume} หน้า {currentPage} —
316
- </span>
317
- </footer>
318
- </motion.article>
319
 
320
- ) : (
321
- <div className="text-center py-40 text-[#888]">
322
- <div className="text-5xl mb-6 opacity-40">📖</div>
323
- <p className="text-lg">ยังไม่ได้เลือกข้อความ</p>
324
- <p className="text-sm mt-2 opacity-60">
325
- เลือกเล่มจากสารบัญ หรือค้นหาคำในพระไตรปิฎก
326
- </p>
327
- </div>
328
- )}
329
- </AnimatePresence>
 
 
 
 
 
 
 
 
 
 
 
330
  </div>
331
  </div>
332
  );
 
230
  />
231
  </ErrorBoundary>
232
  <div className="max-w-3xl mx-auto px-4 py-10 lg:py-16">
233
+ {loading && (
234
+ <div className="flex flex-col items-center justify-center py-48 gap-4">
235
+ <div className="w-10 h-10 border-4 border-[#c8860a]/20 border-t-[#c8860a] rounded-full animate-spin" />
236
+ <p className="text-[#c8860a] text-sm animate-pulse">กำลังอัญเชิญข้อความ…</p>
237
+ </div>
238
+ )}
 
 
 
 
239
 
240
+ {!loading && (
241
+ <AnimatePresence custom={direction} mode="popLayout">
242
+ {content ? (
243
+ <motion.article
244
+ key={`${currentVolume}-${currentPage}`}
245
+ custom={direction}
246
+ variants={articleVariants}
247
+ initial="enter"
248
+ animate="center"
249
+ exit="exit"
250
+ transition={{ duration: 0.35, ease: 'easeOut' }}
251
+ className={`rounded-xl p-8 lg:p-12 ${readerCls}`}
252
+ onClick={handleArticleClick}
253
+ >
254
+ {currentPage === 1 || content?.page_number === 1 ? (
255
+ renderFirstPage(
256
+ content.content_html_formatted || content.content_html || '',
257
+ fontSize,
258
+ content.title || ''
259
+ )
260
+ ) : (
261
+ <>
262
+ <header className="mb-4 pb-0">
263
+ <p className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
264
+ {content.title}
 
 
 
 
 
 
 
265
  </p>
266
+ {content.sections?.filter((s: {level: number}) => s.level === 0).map((sec: {title: string}, i: number) => (
267
+ <p key={`l0-${i}`} className="text-xs text-[#888] tracking-widest uppercase leading-relaxed">
268
+ {sec.title}
269
+ </p>
270
+ ))}
271
+ {content.sections?.filter((s: {level: number}) => s.level === 1).map((sec: {title: string}, i: number) => (
272
+ <p key={`l1-${i}`} className="text-sm text-[#888] tracking-widest uppercase leading-relaxed">
273
+ {sec.title}
274
+ </p>
275
+ ))}
276
+ </header>
277
+ <div className="border-b border-[#c8860a]/15 mb-5" />
278
+ {renderContent(content.content_html_formatted || content.content_html || '', fontSize, content.end_markers || [])}
279
+ </>
280
+ )}
281
 
282
+ {/* Footnotes Section */}
283
+ {content.footnotes && content.footnotes.length > 0 && (
284
+ <div className="mt-16 pt-8 border-t border-[#c8860a]/20">
285
+ <h5 className="text-[#c8860a] text-xs font-bold tracking-widest uppercase mb-6 flex items-center gap-2">
286
+ <svg className="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
287
+ <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M16 4v12l-4-2-4 2V4M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
288
+ </svg>
289
+ เชิงอรรถ
290
+ </h5>
291
+ <div className="space-y-4">
292
+ {content.footnotes.map((fn: any, idx: number) => {
293
+ const cleanId = fn.id.replace(/[()\[\]-]/g, '').trim();
294
+ return (
295
+ <div
296
+ key={idx}
297
+ id={`fn-item-${cleanId}`}
298
+ className={`flex gap-3 text-[13px] leading-relaxed ${mutedCls} hover:opacity-100 transition-all p-2 -m-2 rounded-lg cursor-pointer group`}
299
+ onClick={() => scrollToElement(`ref-${cleanId}`)}
300
+ title="คลิกเพื่อกลับไปยังเนื้อหา"
301
+ >
302
+ <span className="text-[#c8860a] font-bold min-w-[24px] text-right shrink-0 group-hover:scale-110 transition-transform">{fn.id}</span>
303
+ <span className="font-light">{fn.content}</span>
304
+ </div>
305
+ );
306
+ })}
307
+ </div>
308
  </div>
309
+ )}
 
 
 
 
 
 
 
 
310
 
311
+ <footer className="mt-12 pt-6 border-t border-[#888]/10 text-center">
312
+ <span className="text-xs text-[#888]">
313
+ เล่ม {currentVolume} หน้า {currentPage} —
314
+ </span>
315
+ </footer>
316
+ </motion.article>
317
+ ) : (
318
+ <motion.div
319
+ key="empty"
320
+ initial={{ opacity: 0 }} animate={{ opacity: 1 }} exit={{ opacity: 0 }}
321
+ className="text-center py-40 text-[#888]"
322
+ >
323
+ <div className="text-5xl mb-6 opacity-40">📖</div>
324
+ <p className="text-lg">ยังไม่ได้เลือกข้อความ</p>
325
+ <p className="text-sm mt-2 opacity-60">
326
+ เลือกเล่มจากสารบัญ หรือค้นหาคำในพระไตรปิฎก
327
+ </p>
328
+ </motion.div>
329
+ )}
330
+ </AnimatePresence>
331
+ )}
332
  </div>
333
  </div>
334
  );