feat: add Korean benchmark models

app.py

@@ -6,6 +6,7 @@ import gradio as gr
 import torch
 from huggingface_hub import hf_hub_download
 from sentence_transformers import SentenceTransformer, models
+from transformers import AutoModel, AutoTokenizer
 
 from benchmarking import (
     BENCHMARK_TABLE_HEADERS,
@@ -49,14 +50,55 @@ def load_model():
     return model
 
 
-@lru_cache(maxsize=4)
+@lru_cache(maxsize=8)
 def load_sentence_transformer(model_id):
     return SentenceTransformer(model_id, device=get_device())
 
 
+class TransformersClsEncoder:
+    def __init__(self, model_id):
+        self.device = get_device()
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self.model = AutoModel.from_pretrained(model_id).to(self.device)
+        self.model.eval()
+
+    def encode(
+        self,
+        texts,
+        normalize_embeddings=True,
+        convert_to_numpy=True,
+        show_progress_bar=False,
+    ):
+        del show_progress_bar
+        inputs = self.tokenizer(
+            list(texts),
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+        )
+        inputs = {key: value.to(self.device) for key, value in inputs.items()}
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+            embeddings = getattr(outputs, "pooler_output", None)
+            if embeddings is None:
+                embeddings = outputs.last_hidden_state[:, 0]
+            if normalize_embeddings:
+                embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
+        if convert_to_numpy:
+            return embeddings.cpu().numpy()
+        return embeddings
+
+
+@lru_cache(maxsize=3)
+def load_transformers_cls_encoder(model_id):
+    return TransformersClsEncoder(model_id)
+
+
 def load_benchmark_model(spec):
     if spec.kind == "current_adapter":
         return load_model()
+    if spec.kind == "transformers_cls":
+        return load_transformers_cls_encoder(spec.model_id)
     return load_sentence_transformer(spec.model_id)
 
 

benchmarking.py

@@ -51,6 +51,32 @@ BENCHMARK_MODEL_SPECS: Tuple[BenchmarkModelSpec, ...] = (
         model_id="jhgan/ko-sroberta-multitask",
         note="Korean sentence embedding baseline",
     ),
+    BenchmarkModelSpec(
+        key="kosimcse-roberta-multitask",
+        label="BM-K/KoSimCSE-roberta-multitask",
+        model_id="BM-K/KoSimCSE-roberta-multitask",
+        kind="transformers_cls",
+        note="Korean KoSimCSE multitask RoBERTa baseline",
+    ),
+    BenchmarkModelSpec(
+        key="kosimcse-roberta",
+        label="BM-K/KoSimCSE-roberta",
+        model_id="BM-K/KoSimCSE-roberta",
+        kind="transformers_cls",
+        note="Korean KoSimCSE RoBERTa baseline",
+    ),
+    BenchmarkModelSpec(
+        key="kr-sbert",
+        label="KR-SBERT V40K klueNLI augSTS",
+        model_id="snunlp/KR-SBERT-V40K-klueNLI-augSTS",
+        note="Korean SBERT sentence-similarity baseline",
+    ),
+    BenchmarkModelSpec(
+        key="kure-v1",
+        label="nlpai-lab/KURE-v1",
+        model_id="nlpai-lab/KURE-v1",
+        note="Korean universal representation embedding baseline",
+    ),
 )
 
 DEFAULT_BENCHMARK_MODEL_IDS: Tuple[str, ...] = tuple(

requirements.txt

@@ -1,4 +1,5 @@
 sentence-transformers
 huggingface_hub
 torch
+transformers
 audioop-lts; python_version>='3.13'

test_benchmarking.py

@@ -15,6 +15,23 @@ class BenchmarkingTest(unittest.TestCase):
         self.assertIn("multilingual-e5-small", DEFAULT_BENCHMARK_MODEL_IDS)
         self.assertIn("multilingual-minilm", DEFAULT_BENCHMARK_MODEL_IDS)
 
+    def test_korean_focused_models_include_kosimcse_and_kure(self):
+        specs_by_key = {spec.key: spec for spec in BENCHMARK_MODEL_SPECS}
+
+        self.assertEqual(
+            specs_by_key["kosimcse-roberta-multitask"].model_id,
+            "BM-K/KoSimCSE-roberta-multitask",
+        )
+        self.assertEqual(
+            specs_by_key["kosimcse-roberta-multitask"].kind,
+            "transformers_cls",
+        )
+        self.assertEqual(
+            specs_by_key["kr-sbert"].model_id,
+            "snunlp/KR-SBERT-V40K-klueNLI-augSTS",
+        )
+        self.assertEqual(specs_by_key["kure-v1"].model_id, "nlpai-lab/KURE-v1")
+
     def test_get_benchmark_specs_preserves_catalog_order_and_ignores_unknown(self):
         selected = ["unknown", "multilingual-minilm", "current-adapter"]