"""Service layer for the mundart-explorer Space. Keeps all logic out of ``app.py``: trains the deterministic char n-gram NB LID baseline from the shipped ``data/lid_train.jsonl`` once at import, and answers two questions about a pasted sentence: * **LID** — written Swiss-German (``gsw``) vs Standard German (``de``), with a confidence derived from the model's per-label log posteriors; * **gsw→de alignment** — the nearest Tatoeba ``gsw→de`` aligned pair (char-n-gram Jaccard over the 334 probe sentences), shown when a sentence is close enough to give a genuine Standard-German rendering rather than a fabricated translation. The baseline is pure stdlib, so this module has no third-party dependencies. """ from __future__ import annotations import json import math import sys from dataclasses import dataclass from pathlib import Path _HERE = Path(__file__).resolve().parent sys.path.insert(0, str(_HERE / "vendor")) from mundart.baseline import CharNgramNB, char_ngrams # noqa: E402 from mundart.eval import LABELS, Example # noqa: E402 _DATA = _HERE / "data" _LABEL_LONG = {"gsw": "Swiss-German (gsw)", "de": "Standard German (de)"} #: Jaccard floor below which the nearest probe pair is too unrelated to surface. _ALIGN_MIN_SIM = 0.10 #: Char n-gram order for the alignment similarity (matches the baseline's range). _SIM_RANGE = (2, 4) def _load_jsonl(path: Path) -> list[dict]: with path.open(encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] def _train_model() -> CharNgramNB: rows = _load_jsonl(_DATA / "lid_train.jsonl") examples = [ Example(id=r["id"], text=r["text"], label=r["label"], source=r["source"]) for r in rows ] return CharNgramNB.train(examples) @dataclass(frozen=True) class ProbePair: gsw: str de: str grams: frozenset[str] def _load_probe() -> list[ProbePair]: return [ ProbePair(gsw=r["gsw"], de=r["de"], grams=frozenset(char_ngrams(r["gsw"], _SIM_RANGE))) for r in _load_jsonl(_DATA / "probe_gsw_de.jsonl") ] _MODEL = _train_model() _PROBE = _load_probe() def _confidence(scores: dict[str, float]) -> dict[str, float]: """Softmax the two log posteriors into [0, 1] class probabilities.""" top = max(scores.values()) exp = {label: math.exp(scores[label] - top) for label in scores} total = sum(exp.values()) return {label: exp[label] / total for label in exp} def identify(text: str) -> dict: """LID a sentence → ``{label, label_long, confidence, probs}``.""" scores = _MODEL.log_scores(text) label = max(sorted(LABELS), key=lambda label: scores[label]) probs = _confidence(scores) return { "label": label, "label_long": _LABEL_LONG[label], "confidence": probs[label], "probs": probs, } def nearest_alignment(text: str) -> dict | None: """Nearest gsw→de probe pair by char-n-gram Jaccard, or ``None`` if too far.""" query = frozenset(char_ngrams(text, _SIM_RANGE)) if not query: return None best, best_sim = None, 0.0 for pair in _PROBE: union = len(query | pair.grams) sim = len(query & pair.grams) / union if union else 0.0 if sim > best_sim: best, best_sim = pair, sim if best is None or best_sim < _ALIGN_MIN_SIM: return None return {"gsw": best.gsw, "de": best.de, "similarity": best_sim} def probe_size() -> int: return len(_PROBE)