SAkizuki commited on
Commit
d98e7b3
·
verified ·
1 Parent(s): 7043411

Auto-sync from GitHub Actions

Browse files
Files changed (4) hide show
  1. api_fastapi.py +2 -2
  2. core/engine.py +70 -1
  3. mcp_server.py +18 -2
  4. ui_nicegui.py +1 -1
api_fastapi.py CHANGED
@@ -104,8 +104,8 @@ async def search(body: SearchIn) -> SearchOut:
104
  # SearchIn → core.models.SearchRequest(两者字段一一对应,直接解包)
105
  request = SearchRequest(**body.model_dump())
106
 
107
- # 线程池中运阻塞的 search()
108
- response: SearchResponse = await asyncio.to_thread(tagger.search, request)
109
 
110
  # 计数:每次 API 搜索调用均计入搜索、成功、复制;访问不变
111
  await counter.increment()
 
104
  # SearchIn → core.models.SearchRequest(两者字段一一对应,直接解包)
105
  request = SearchRequest(**body.model_dump())
106
 
107
+ # 并发安全的异步 search(信号量串行化 + 线程池
108
+ response: SearchResponse = await tagger.search_async(request)
109
 
110
  # 计数:每次 API 搜索调用均计入搜索、成功、复制;访问不变
111
  await counter.increment()
core/engine.py CHANGED
@@ -37,6 +37,10 @@ from platform_utils import (
37
  )
38
 
39
 
 
 
 
 
40
  # LRU 缓存
41
  class LRUCache:
42
  def __init__(self, maxsize: int):
@@ -158,6 +162,9 @@ class DanbooruTagger:
158
 
159
  _instance: Optional['DanbooruTagger'] = None
160
  _lock: Optional[asyncio.Lock] = None
 
 
 
161
 
162
  @classmethod
163
  def is_ready(cls) -> bool:
@@ -536,6 +543,21 @@ class DanbooruTagger:
536
  self._search_cache.put(cache_key, response)
537
  return response
538
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
539
  def _apply_group_expand(self, final: dict[str, TagResult]) -> None:
540
  """expand 模式:提升同 group 标签的分数。"""
541
  BETA = 0.2
@@ -777,7 +799,11 @@ class DanbooruTagger:
777
  _EN_MAX_COMPOUND = 4 # 复合标签最大单词数
778
 
779
  def _tokenize_en_chunk(self, chunk: str) -> list[str]:
780
- """对一段英文文本做分词:清洗 → 按空格切分 → 过滤停用词/纯数 → 合并已知复合标签。"""
 
 
 
 
781
  cleaned = re.sub(r'[,()\[\]{}:]', ' ', chunk)
782
  raw = [p for p in cleaned.split() if p]
783
  tag_set = getattr(self, '_tag_names_set', None)
@@ -790,6 +816,11 @@ class DanbooruTagger:
790
  continue
791
  if low in STOP_WORDS:
792
  continue
 
 
 
 
 
793
  if part.isdigit(): # 仅过滤纯数字,保留 3d/2b 等含数字的词
794
  continue
795
  tokens.append(low)
@@ -797,6 +828,44 @@ class DanbooruTagger:
797
  return []
798
  return self._merge_compound_english(tokens)
799
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
800
  def _merge_compound_english(self, tokens: list[str]) -> list[str]:
801
  """将相邻英文单词合并为已知的 Danbooru 下划线复合标签。
802
 
 
37
  )
38
 
39
 
40
+ # 限制 PyTorch CPU 线程数,给 asyncio 事件循环留出至少一个核心。
41
+ torch.set_num_threads(max(1, (os.cpu_count() or 2) - 1))
42
+
43
+
44
  # LRU 缓存
45
  class LRUCache:
46
  def __init__(self, maxsize: int):
 
162
 
163
  _instance: Optional['DanbooruTagger'] = None
164
  _lock: Optional[asyncio.Lock] = None
165
+ # 进程级搜索并发信号量:串行化 search(),避免多个 model.encode()
166
+ # 并发抢占 CPU 而拖垮事件循环。
167
+ _search_sem: Optional[asyncio.Semaphore] = None
168
 
169
  @classmethod
170
  def is_ready(cls) -> bool:
 
543
  self._search_cache.put(cache_key, response)
544
  return response
545
 
546
+ @classmethod
547
+ def _get_search_sem(cls) -> asyncio.Semaphore:
548
+ if cls._search_sem is None:
549
+ cls._search_sem = asyncio.Semaphore(1)
550
+ return cls._search_sem
551
+
552
+ async def search_async(self, request: SearchRequest) -> SearchResponse:
553
+ """search() 的并发安全异步封装:信号量串行化 + 线程池执行。
554
+
555
+ 所有异步入口(MCP / API / UI)都应改用本方法,而非各自
556
+ asyncio.to_thread(self.search),以共享同一个并发闸门。
557
+ """
558
+ async with self._get_search_sem():
559
+ return await asyncio.to_thread(self.search, request)
560
+
561
  def _apply_group_expand(self, final: dict[str, TagResult]) -> None:
562
  """expand 模式:提升同 group 标签的分数。"""
563
  BETA = 0.2
 
799
  _EN_MAX_COMPOUND = 4 # 复合标签最大单词数
800
 
801
  def _tokenize_en_chunk(self, chunk: str) -> list[str]:
802
+ """对一段英文文本做分词:清洗 → 按空格切分 → 过滤停用词/纯数 → 变体规范化 → 合并已知复合标签。
803
+
804
+ 变体规范化指:未直接命中 tag_set 的 token 尝试 `连字符→下划线` /
805
+ 复数还原(s/es/ies→y),仅在变体落在 tag_set 才采用。
806
+ """
807
  cleaned = re.sub(r'[,()\[\]{}:]', ' ', chunk)
808
  raw = [p for p in cleaned.split() if p]
809
  tag_set = getattr(self, '_tag_names_set', None)
 
816
  continue
817
  if low in STOP_WORDS:
818
  continue
819
+ # 连字符/复数变体探测(仅当变体落在 tag_set 才采用)
820
+ variant = self._resolve_tag_variant(low)
821
+ if variant:
822
+ tokens.append(variant)
823
+ continue
824
  if part.isdigit(): # 仅过滤纯数字,保留 3d/2b 等含数字的词
825
  continue
826
  tokens.append(low)
 
828
  return []
829
  return self._merge_compound_english(tokens)
830
 
831
+ def _resolve_tag_variant(self, low: str) -> str | None:
832
+ """对未直接命中 tag_set 的英文 token 探测常见变体。
833
+
834
+ 覆盖:连字符→下划线(cat-ears → cat_ears)、复数→单数
835
+ (cats → cat / dresses → dress / bunnies → bunny)。
836
+ 仅在变体落在 _tag_names_set 中才返回,避免 'glass'→'glas' 之类误伤。
837
+
838
+ Returns:
839
+ 命中的 tag 名;都不命中返回 None。
840
+ """
841
+ tag_set = getattr(self, '_tag_names_set', None)
842
+ if not tag_set:
843
+ return None
844
+
845
+ # 连字符直接换成下划线若直接命中 tag 则优先返回
846
+ bases = [low]
847
+ if '-' in low:
848
+ hyphen_normalized = low.replace('-', '_')
849
+ if hyphen_normalized in tag_set:
850
+ return hyphen_normalized
851
+ bases.append(hyphen_normalized)
852
+
853
+ # 对每个基串尝试复数还原(按"剥离短→长"顺序,避免 houses→hous 误判)
854
+ for base in bases:
855
+ if base.endswith('s') and len(base) > 1:
856
+ v = base[:-1]
857
+ if v in tag_set:
858
+ return v
859
+ if base.endswith('es') and len(base) > 2:
860
+ v = base[:-2]
861
+ if v in tag_set:
862
+ return v
863
+ if base.endswith('ies') and len(base) > 3:
864
+ v = base[:-3] + 'y'
865
+ if v in tag_set:
866
+ return v
867
+ return None
868
+
869
  def _merge_compound_english(self, tokens: list[str]) -> list[str]:
870
  """将相邻英文单词合并为已知的 Danbooru 下划线复合标签。
871
 
mcp_server.py CHANGED
@@ -17,6 +17,8 @@ MCP 服务层
17
 
18
  import json
19
  import asyncio
 
 
20
  from mcp.server.fastmcp import FastMCP
21
  from mcp.server.transport_security import TransportSecuritySettings
22
  from core.engine import DanbooruTagger
@@ -25,6 +27,20 @@ import core.counter as counter
25
  import re
26
 
27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  mcp = FastMCP(
29
  name="danbooru-searcher",
30
  transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
@@ -198,7 +214,7 @@ Each result: tag, cn_name, category, final_score, count[, wiki if include_wiki=T
198
  group_mode=group_mode,
199
  max_per_group=max_per_group,
200
  )
201
- response = await asyncio.to_thread(tagger.search, request)
202
  # 计数:每次 MCP 搜索调用均计入搜索、成功、复制;访问不变
203
  await counter.increment()
204
  await counter.increment_success()
@@ -307,7 +323,7 @@ JSON array sorted by aggregated NPMI score (descending). Each result:
307
  use_segmentation=False,
308
  target_layers=['英文']
309
  )
310
- resp = await asyncio.to_thread(tagger.search, req)
311
  if resp.results:
312
  corrections[bad_tag] = resp.results[0].tag
313
  except Exception:
 
17
 
18
  import json
19
  import asyncio
20
+ import logging
21
+ from anyio import BrokenResourceError, ClosedResourceError
22
  from mcp.server.fastmcp import FastMCP
23
  from mcp.server.transport_security import TransportSecuritySettings
24
  from core.engine import DanbooruTagger
 
27
  import re
28
 
29
 
30
+ # ── 过滤客户端断连产生的无害报错噪音 ──────────────────────────────────
31
+ class _SuppressClientDisconnect(logging.Filter):
32
+ def filter(self, record: logging.LogRecord) -> bool:
33
+ exc = record.exc_info[1] if record.exc_info else None
34
+ if isinstance(exc, (BrokenResourceError, ClosedResourceError)):
35
+ return False # 丢弃该日志记录
36
+ return True
37
+
38
+
39
+ _disconnect_filter = _SuppressClientDisconnect()
40
+ logging.getLogger("mcp.server.streamable_http").addFilter(_disconnect_filter)
41
+ logging.getLogger("uvicorn.error").addFilter(_disconnect_filter)
42
+
43
+
44
  mcp = FastMCP(
45
  name="danbooru-searcher",
46
  transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
 
214
  group_mode=group_mode,
215
  max_per_group=max_per_group,
216
  )
217
+ response = await tagger.search_async(request)
218
  # 计数:每次 MCP 搜索调用均计入搜索、成功、复制;访问不变
219
  await counter.increment()
220
  await counter.increment_success()
 
323
  use_segmentation=False,
324
  target_layers=['英文']
325
  )
326
+ resp = await tagger.search_async(req)
327
  if resp.results:
328
  corrections[bad_tag] = resp.results[0].tag
329
  except Exception:
ui_nicegui.py CHANGED
@@ -1179,7 +1179,7 @@ class DanbooruSearchUI:
1179
  group_mode=self.input_group_mode.value if self.input_group_mode else 'off',
1180
  max_per_group=int(self.input_max_per_group.value) if self.input_max_per_group else 2,
1181
  )
1182
- response = await run.io_bound(tagger.search, request)
1183
 
1184
  # 后台计数
1185
  async def silent_counter_update():
 
1179
  group_mode=self.input_group_mode.value if self.input_group_mode else 'off',
1180
  max_per_group=int(self.input_max_per_group.value) if self.input_max_per_group else 2,
1181
  )
1182
+ response = await tagger.search_async(request)
1183
 
1184
  # 后台计数
1185
  async def silent_counter_update():