Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Auto-sync from GitHub Actions
Browse files- api_fastapi.py +2 -2
- core/engine.py +70 -1
- mcp_server.py +18 -2
- ui_nicegui.py +1 -1
api_fastapi.py
CHANGED
|
@@ -104,8 +104,8 @@ async def search(body: SearchIn) -> SearchOut:
|
|
| 104 |
# SearchIn → core.models.SearchRequest(两者字段一一对应,直接解包)
|
| 105 |
request = SearchRequest(**body.model_dump())
|
| 106 |
|
| 107 |
-
#
|
| 108 |
-
response: SearchResponse = await
|
| 109 |
|
| 110 |
# 计数:每次 API 搜索调用均计入搜索、成功、复制;访问不变
|
| 111 |
await counter.increment()
|
|
|
|
| 104 |
# SearchIn → core.models.SearchRequest(两者字段一一对应,直接解包)
|
| 105 |
request = SearchRequest(**body.model_dump())
|
| 106 |
|
| 107 |
+
# 并发安全的异步 search(信号量串行化 + 线程池执行)
|
| 108 |
+
response: SearchResponse = await tagger.search_async(request)
|
| 109 |
|
| 110 |
# 计数:每次 API 搜索调用均计入搜索、成功、复制;访问不变
|
| 111 |
await counter.increment()
|
core/engine.py
CHANGED
|
@@ -37,6 +37,10 @@ from platform_utils import (
|
|
| 37 |
)
|
| 38 |
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
# LRU 缓存
|
| 41 |
class LRUCache:
|
| 42 |
def __init__(self, maxsize: int):
|
|
@@ -158,6 +162,9 @@ class DanbooruTagger:
|
|
| 158 |
|
| 159 |
_instance: Optional['DanbooruTagger'] = None
|
| 160 |
_lock: Optional[asyncio.Lock] = None
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
@classmethod
|
| 163 |
def is_ready(cls) -> bool:
|
|
@@ -536,6 +543,21 @@ class DanbooruTagger:
|
|
| 536 |
self._search_cache.put(cache_key, response)
|
| 537 |
return response
|
| 538 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 539 |
def _apply_group_expand(self, final: dict[str, TagResult]) -> None:
|
| 540 |
"""expand 模式:提升同 group 标签的分数。"""
|
| 541 |
BETA = 0.2
|
|
@@ -777,7 +799,11 @@ class DanbooruTagger:
|
|
| 777 |
_EN_MAX_COMPOUND = 4 # 复合标签最大单词数
|
| 778 |
|
| 779 |
def _tokenize_en_chunk(self, chunk: str) -> list[str]:
|
| 780 |
-
"""对一段英文文本做分词:清洗 → 按空格切分 → 过滤停用词/纯数 → 合并已知复合标签。
|
|
|
|
|
|
|
|
|
|
|
|
|
| 781 |
cleaned = re.sub(r'[,()\[\]{}:]', ' ', chunk)
|
| 782 |
raw = [p for p in cleaned.split() if p]
|
| 783 |
tag_set = getattr(self, '_tag_names_set', None)
|
|
@@ -790,6 +816,11 @@ class DanbooruTagger:
|
|
| 790 |
continue
|
| 791 |
if low in STOP_WORDS:
|
| 792 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 793 |
if part.isdigit(): # 仅过滤纯数字,保留 3d/2b 等含数字的词
|
| 794 |
continue
|
| 795 |
tokens.append(low)
|
|
@@ -797,6 +828,44 @@ class DanbooruTagger:
|
|
| 797 |
return []
|
| 798 |
return self._merge_compound_english(tokens)
|
| 799 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 800 |
def _merge_compound_english(self, tokens: list[str]) -> list[str]:
|
| 801 |
"""将相邻英文单词合并为已知的 Danbooru 下划线复合标签。
|
| 802 |
|
|
|
|
| 37 |
)
|
| 38 |
|
| 39 |
|
| 40 |
+
# 限制 PyTorch CPU 线程数,给 asyncio 事件循环留出至少一个核心。
|
| 41 |
+
torch.set_num_threads(max(1, (os.cpu_count() or 2) - 1))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
# LRU 缓存
|
| 45 |
class LRUCache:
|
| 46 |
def __init__(self, maxsize: int):
|
|
|
|
| 162 |
|
| 163 |
_instance: Optional['DanbooruTagger'] = None
|
| 164 |
_lock: Optional[asyncio.Lock] = None
|
| 165 |
+
# 进程级搜索并发信号量:串行化 search(),避免多个 model.encode()
|
| 166 |
+
# 并发抢占 CPU 而拖垮事件循环。
|
| 167 |
+
_search_sem: Optional[asyncio.Semaphore] = None
|
| 168 |
|
| 169 |
@classmethod
|
| 170 |
def is_ready(cls) -> bool:
|
|
|
|
| 543 |
self._search_cache.put(cache_key, response)
|
| 544 |
return response
|
| 545 |
|
| 546 |
+
@classmethod
|
| 547 |
+
def _get_search_sem(cls) -> asyncio.Semaphore:
|
| 548 |
+
if cls._search_sem is None:
|
| 549 |
+
cls._search_sem = asyncio.Semaphore(1)
|
| 550 |
+
return cls._search_sem
|
| 551 |
+
|
| 552 |
+
async def search_async(self, request: SearchRequest) -> SearchResponse:
|
| 553 |
+
"""search() 的并发安全异步封装:信号量串行化 + 线程池执行。
|
| 554 |
+
|
| 555 |
+
所有异步入口(MCP / API / UI)都应改用本方法,而非各自
|
| 556 |
+
asyncio.to_thread(self.search),以共享同一个并发闸门。
|
| 557 |
+
"""
|
| 558 |
+
async with self._get_search_sem():
|
| 559 |
+
return await asyncio.to_thread(self.search, request)
|
| 560 |
+
|
| 561 |
def _apply_group_expand(self, final: dict[str, TagResult]) -> None:
|
| 562 |
"""expand 模式:提升同 group 标签的分数。"""
|
| 563 |
BETA = 0.2
|
|
|
|
| 799 |
_EN_MAX_COMPOUND = 4 # 复合标签最大单词数
|
| 800 |
|
| 801 |
def _tokenize_en_chunk(self, chunk: str) -> list[str]:
|
| 802 |
+
"""对一段英文文本做分词:清洗 → 按空格切分 → 过滤停用词/纯数 → 变体规范化 → 合并已知复合标签。
|
| 803 |
+
|
| 804 |
+
变体规范化指:未直接命中 tag_set 的 token 尝试 `连字符→下划线` /
|
| 805 |
+
复数还原(s/es/ies→y),仅在变体落在 tag_set 才采用。
|
| 806 |
+
"""
|
| 807 |
cleaned = re.sub(r'[,()\[\]{}:]', ' ', chunk)
|
| 808 |
raw = [p for p in cleaned.split() if p]
|
| 809 |
tag_set = getattr(self, '_tag_names_set', None)
|
|
|
|
| 816 |
continue
|
| 817 |
if low in STOP_WORDS:
|
| 818 |
continue
|
| 819 |
+
# 连字符/复数变体探测(仅当变体落在 tag_set 才采用)
|
| 820 |
+
variant = self._resolve_tag_variant(low)
|
| 821 |
+
if variant:
|
| 822 |
+
tokens.append(variant)
|
| 823 |
+
continue
|
| 824 |
if part.isdigit(): # 仅过滤纯数字,保留 3d/2b 等含数字的词
|
| 825 |
continue
|
| 826 |
tokens.append(low)
|
|
|
|
| 828 |
return []
|
| 829 |
return self._merge_compound_english(tokens)
|
| 830 |
|
| 831 |
+
def _resolve_tag_variant(self, low: str) -> str | None:
|
| 832 |
+
"""对未直接命中 tag_set 的英文 token 探测常见变体。
|
| 833 |
+
|
| 834 |
+
覆盖:连字符→下划线(cat-ears → cat_ears)、复数→单数
|
| 835 |
+
(cats → cat / dresses → dress / bunnies → bunny)。
|
| 836 |
+
仅在变体落在 _tag_names_set 中才返回,避免 'glass'→'glas' 之类误伤。
|
| 837 |
+
|
| 838 |
+
Returns:
|
| 839 |
+
命中的 tag 名;都不命中返回 None。
|
| 840 |
+
"""
|
| 841 |
+
tag_set = getattr(self, '_tag_names_set', None)
|
| 842 |
+
if not tag_set:
|
| 843 |
+
return None
|
| 844 |
+
|
| 845 |
+
# 连字符直接换成下划线若直接命中 tag 则优先返回
|
| 846 |
+
bases = [low]
|
| 847 |
+
if '-' in low:
|
| 848 |
+
hyphen_normalized = low.replace('-', '_')
|
| 849 |
+
if hyphen_normalized in tag_set:
|
| 850 |
+
return hyphen_normalized
|
| 851 |
+
bases.append(hyphen_normalized)
|
| 852 |
+
|
| 853 |
+
# 对每个基串尝试复数还原(按"剥离短→长"顺序,避免 houses→hous 误判)
|
| 854 |
+
for base in bases:
|
| 855 |
+
if base.endswith('s') and len(base) > 1:
|
| 856 |
+
v = base[:-1]
|
| 857 |
+
if v in tag_set:
|
| 858 |
+
return v
|
| 859 |
+
if base.endswith('es') and len(base) > 2:
|
| 860 |
+
v = base[:-2]
|
| 861 |
+
if v in tag_set:
|
| 862 |
+
return v
|
| 863 |
+
if base.endswith('ies') and len(base) > 3:
|
| 864 |
+
v = base[:-3] + 'y'
|
| 865 |
+
if v in tag_set:
|
| 866 |
+
return v
|
| 867 |
+
return None
|
| 868 |
+
|
| 869 |
def _merge_compound_english(self, tokens: list[str]) -> list[str]:
|
| 870 |
"""将相邻英文单词合并为已知的 Danbooru 下划线复合标签。
|
| 871 |
|
mcp_server.py
CHANGED
|
@@ -17,6 +17,8 @@ MCP 服务层
|
|
| 17 |
|
| 18 |
import json
|
| 19 |
import asyncio
|
|
|
|
|
|
|
| 20 |
from mcp.server.fastmcp import FastMCP
|
| 21 |
from mcp.server.transport_security import TransportSecuritySettings
|
| 22 |
from core.engine import DanbooruTagger
|
|
@@ -25,6 +27,20 @@ import core.counter as counter
|
|
| 25 |
import re
|
| 26 |
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
mcp = FastMCP(
|
| 29 |
name="danbooru-searcher",
|
| 30 |
transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
|
|
@@ -198,7 +214,7 @@ Each result: tag, cn_name, category, final_score, count[, wiki if include_wiki=T
|
|
| 198 |
group_mode=group_mode,
|
| 199 |
max_per_group=max_per_group,
|
| 200 |
)
|
| 201 |
-
response = await
|
| 202 |
# 计数:每次 MCP 搜索调用均计入搜索、成功、复制;访问不变
|
| 203 |
await counter.increment()
|
| 204 |
await counter.increment_success()
|
|
@@ -307,7 +323,7 @@ JSON array sorted by aggregated NPMI score (descending). Each result:
|
|
| 307 |
use_segmentation=False,
|
| 308 |
target_layers=['英文']
|
| 309 |
)
|
| 310 |
-
resp = await
|
| 311 |
if resp.results:
|
| 312 |
corrections[bad_tag] = resp.results[0].tag
|
| 313 |
except Exception:
|
|
|
|
| 17 |
|
| 18 |
import json
|
| 19 |
import asyncio
|
| 20 |
+
import logging
|
| 21 |
+
from anyio import BrokenResourceError, ClosedResourceError
|
| 22 |
from mcp.server.fastmcp import FastMCP
|
| 23 |
from mcp.server.transport_security import TransportSecuritySettings
|
| 24 |
from core.engine import DanbooruTagger
|
|
|
|
| 27 |
import re
|
| 28 |
|
| 29 |
|
| 30 |
+
# ── 过滤客户端断连产生的无害报错噪音 ──────────────────────────────────
|
| 31 |
+
class _SuppressClientDisconnect(logging.Filter):
|
| 32 |
+
def filter(self, record: logging.LogRecord) -> bool:
|
| 33 |
+
exc = record.exc_info[1] if record.exc_info else None
|
| 34 |
+
if isinstance(exc, (BrokenResourceError, ClosedResourceError)):
|
| 35 |
+
return False # 丢弃该日志记录
|
| 36 |
+
return True
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
_disconnect_filter = _SuppressClientDisconnect()
|
| 40 |
+
logging.getLogger("mcp.server.streamable_http").addFilter(_disconnect_filter)
|
| 41 |
+
logging.getLogger("uvicorn.error").addFilter(_disconnect_filter)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
mcp = FastMCP(
|
| 45 |
name="danbooru-searcher",
|
| 46 |
transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False),
|
|
|
|
| 214 |
group_mode=group_mode,
|
| 215 |
max_per_group=max_per_group,
|
| 216 |
)
|
| 217 |
+
response = await tagger.search_async(request)
|
| 218 |
# 计数:每次 MCP 搜索调用均计入搜索、成功、复制;访问不变
|
| 219 |
await counter.increment()
|
| 220 |
await counter.increment_success()
|
|
|
|
| 323 |
use_segmentation=False,
|
| 324 |
target_layers=['英文']
|
| 325 |
)
|
| 326 |
+
resp = await tagger.search_async(req)
|
| 327 |
if resp.results:
|
| 328 |
corrections[bad_tag] = resp.results[0].tag
|
| 329 |
except Exception:
|
ui_nicegui.py
CHANGED
|
@@ -1179,7 +1179,7 @@ class DanbooruSearchUI:
|
|
| 1179 |
group_mode=self.input_group_mode.value if self.input_group_mode else 'off',
|
| 1180 |
max_per_group=int(self.input_max_per_group.value) if self.input_max_per_group else 2,
|
| 1181 |
)
|
| 1182 |
-
response = await
|
| 1183 |
|
| 1184 |
# 后台计数
|
| 1185 |
async def silent_counter_update():
|
|
|
|
| 1179 |
group_mode=self.input_group_mode.value if self.input_group_mode else 'off',
|
| 1180 |
max_per_group=int(self.input_max_per_group.value) if self.input_max_per_group else 2,
|
| 1181 |
)
|
| 1182 |
+
response = await tagger.search_async(request)
|
| 1183 |
|
| 1184 |
# 后台计数
|
| 1185 |
async def silent_counter_update():
|