Stage DataFog PII v0.1.0: epoch-2 checkpoint, frozen MCP bundle, provenance and licenses (part 2)

reproduction/pii/core.py

@@ -0,0 +1,143 @@
+import json
+from pathlib import Path
+
+import numpy as np
+
+LABELS = ["first_name", "last_name", "email", "phone_number", "street_address", "customer_id", "account_number"]
+BIO = ["O"] + [f"{prefix}-{label}" for label in LABELS for prefix in ("B", "I")]
+MODEL_REVISION = "6ea81172465e8b0ad3fddeed32b986cdcdcffcf0"
+
+
+def byte_positions(text):
+    positions = [0]
+    for ch in text:
+        positions.append(positions[-1] + len(ch.encode("utf-8")))
+    return positions
+
+
+def encode_windows(tokenizer, text, max_length, stride):
+    """Tokenize the complete input before slicing; never use tokenizer overflow.
+
+    tokenizers 0.23.2 early-exits before producing overflow, silently losing text.
+    DistilBERT uses one CLS and one SEP token; raw offsets remain original chars.
+    """
+    capacity = max_length - 2
+    if not 0 <= stride < capacity:
+        raise ValueError("invalid stride or max_length")
+    if tokenizer.cls_token_id is None or tokenizer.sep_token_id is None:
+        raise ValueError("expected DistilBERT CLS/SEP tokenizer")
+    raw = tokenizer(text, add_special_tokens=False, truncation=False, return_offsets_mapping=True, verbose=False)
+    ids, offsets = raw["input_ids"], raw["offset_mapping"]
+    windows = {"input_ids": [], "attention_mask": [], "offset_mapping": []}
+    start = 0
+    while True:
+        end = min(start + capacity, len(ids))
+        window = [tokenizer.cls_token_id] + ids[start:end] + [tokenizer.sep_token_id]
+        windows["input_ids"].append(window)
+        windows["attention_mask"].append([1] * len(window))
+        windows["offset_mapping"].append([(0, 0)] + offsets[start:end] + [(0, 0)])
+        if end == len(ids):
+            break
+        start += capacity - stride
+    return windows
+
+
+def decode_window(logits, offsets, text, max_length, has_previous=False, has_next=False):
+    scores = np.asarray(logits, dtype=np.float64)
+    scores = np.exp(scores - scores.max(axis=-1, keepdims=True))
+    scores /= scores.sum(axis=-1, keepdims=True)
+    ids = scores.argmax(axis=-1)
+    bp = byte_positions(text)
+    findings = []
+    current = None
+    for i, (start, end) in enumerate(offsets):
+        value = int(ids[i])
+        if start == end or value == 0:
+            if current:
+                findings.append(current)
+                current = None
+            continue
+        label = LABELS[(value - 1) // 2]
+        score = float(scores[i, value])
+        if value % 2 == 1 or current is None or current["label"] != label:
+            if current:
+                findings.append(current)
+            current = {"label": label, "start": bp[start], "end": bp[end], "raw_score": score, "confidence": None}
+        else:
+            current["end"] = bp[end]
+            current["raw_score"] = min(current["raw_score"], score)
+    if current:
+        findings.append(current)
+    content = [(a, b) for a, b in offsets if a != b]
+    if len(offsets) == max_length and content:
+        if any(f["start"] == bp[content[0][0]] and f["end"] == bp[content[-1][1]] for f in findings):
+            raise ValueError("entity spans a full window; complete entity boundary is unsupported")
+    if content:
+        for f in findings:
+            f["_boundary_fragment"] = bool((has_previous and f["start"] == bp[content[0][0]]) or (has_next and f["end"] == bp[content[-1][1]]))
+    return findings
+
+
+def resolve_findings(findings):
+    # Prefer an observed containing prediction over a clipped window-edge fragment.
+    # Do not discard an edge prediction without a same-label containing candidate.
+    findings = [f for f in findings if not (f.get("_boundary_fragment", False) and any(
+        g["label"] == f["label"] and g["start"] <= f["start"] and g["end"] >= f["end"]
+        and (g["start"] < f["start"] or g["end"] > f["end"]) for g in findings))]
+    accepted = []
+    for finding in sorted(findings, key=lambda f: (-f["raw_score"], f["start"], f["end"], f["label"])):
+        if not any(finding["start"] < f["end"] and f["start"] < finding["end"] for f in accepted):
+            accepted.append({k:v for k,v in finding.items() if not k.startswith("_")})
+    return sorted(accepted, key=lambda f: (f["start"], f["end"], f["label"]))
+
+
+def result(text, findings, calibration=None, threshold=None):
+    findings = resolve_findings(findings)
+    if calibration and calibration.get("method") == "isotonic":
+        for f in findings:
+            f["confidence"] = float(np.interp(f["raw_score"], calibration["xs"], calibration["ys"]))
+    if threshold is not None:
+        findings = [f for f in findings if (f["confidence"] if f["confidence"] is not None else f["raw_score"]) >= threshold]
+    data = text.encode("utf-8")
+    parts, cursor = [], 0
+    for f in findings:
+        parts.extend([data[cursor:f["start"]], b"[REDACTED]"])
+        cursor = f["end"]
+    parts.append(data[cursor:])
+    return {"findings": findings, "contains_pii": bool(findings), "redacted": b"".join(parts).decode("utf-8")}
+
+
+def load_jsonl(path):
+    with open(path) as handle:
+        return [json.loads(line) for line in handle if line.strip()]
+
+
+class Predictor:
+    def __init__(self, bundle, device="cpu", backend="onnx"):
+        from transformers import AutoTokenizer, AutoModelForTokenClassification
+        self.bundle = Path(bundle)
+        self.config = json.loads((self.bundle / "config.json").read_text())
+        self.tokenizer = AutoTokenizer.from_pretrained(self.bundle, local_files_only=True)
+        self.backend, self.device = backend, device
+        self.calibration = json.loads((self.bundle / self.config["calibration_file"]).read_text()) if self.config.get("calibration_file") else None
+        if backend == "onnx":
+            import onnxruntime as ort
+            options = ort.SessionOptions()
+            options.intra_op_num_threads = 1
+            self.session = ort.InferenceSession(str(self.bundle / "model.onnx"), sess_options=options, providers=["CPUExecutionProvider"])
+        else:
+            self.model = AutoModelForTokenClassification.from_pretrained(self.bundle / "checkpoint", local_files_only=True, attn_implementation="eager").to(device).eval()
+
+    def predict(self, text, calibrated=True):
+        encoded = encode_windows(self.tokenizer, text, self.config["max_length"], self.config["stride"])
+        findings = []
+        for index, (ids, mask, offsets) in enumerate(zip(encoded["input_ids"], encoded["attention_mask"], encoded["offset_mapping"])):
+            if self.backend == "onnx":
+                logits = self.session.run(None, {"input_ids": np.array([ids], dtype=np.int64), "attention_mask": np.array([mask], dtype=np.int64)})[0][0]
+            else:
+                import torch
+                with torch.no_grad():
+                    logits = self.model(input_ids=torch.tensor([ids], device=self.device), attention_mask=torch.tensor([mask], device=self.device)).logits[0].cpu().numpy()
+            findings.extend(decode_window(logits, offsets, text, self.config["max_length"], index > 0, index + 1 < len(encoded["input_ids"])))
+        calibration = self.calibration if calibrated else None
+        return result(text, findings, calibration, calibration.get("threshold") if calibration else None)

reproduction/pii/data.py

@@ -0,0 +1,53 @@
+"""Pinned public-data preparation, Unicode offsets, and leakage-resistant grouping."""
+import ast
+import hashlib
+import json
+import re
+from collections import Counter
+
+LABELS = ('first_name','last_name','email','phone_number','street_address','customer_id','account_number')
+DATA_REVISION = 'b70ffaf5ff39e079776134c5bf4381f00a9fd1ed'
+MODEL_REVISION = '6ea81172465e8b0ad3fddeed32b986cdcdcffcf0'
+
+def digest(text):
+    return hashlib.sha256(text.encode('utf-8')).hexdigest()
+
+def validate(row):
+    """Exclude invalid rows; safely parse observed literals without evaluating code."""
+    try:
+        spans = json.loads(row['spans'])
+    except (ValueError, TypeError):
+        try:
+            spans = ast.literal_eval(row['spans'])
+        except (ValueError, SyntaxError, TypeError):
+            return None, 'parse'
+    if not isinstance(row['text'], str) or not isinstance(spans, list):
+        return None, 'schema'
+    for s in spans:
+        if not isinstance(s, dict) or not all(k in s for k in ('start','end','label','text')):
+            return None, 'span_schema'
+        if type(s['start']) is not int or type(s['end']) is not int or not isinstance(s['label'], str):
+            return None, 'span_schema'
+        if not 0 <= s['start'] < s['end'] <= len(row['text']):
+            return None, 'bounds'
+        if row['text'][s['start']:s['end']] != s['text']:
+            return None, 'source_slice_mismatch'
+    spans.sort(key=lambda s:(s['start'],s['end'],s['label']))
+    if any(a['end'] > b['start'] for a,b in zip(spans,spans[1:])):
+        return None, 'duplicate_or_overlap'
+    return spans, None
+
+def family_key(text, spans):
+    # All annotated labels, not only target labels, identify reusable contexts.
+    parts=[]; previous=0
+    for s in spans:
+        parts.extend((text[previous:s['start']], '<'+s['label']+'>')); previous=s['end']
+    parts.append(text[previous:])
+    return digest(re.sub(r'\s+', ' ', ''.join(parts)).strip().casefold())
+
+def support(rows):
+    docs=Counter(); entities=Counter(); negatives=0
+    for row in rows:
+        labels=[s['label'] for s in row['spans']]
+        entities.update(labels); docs.update(set(labels)); negatives += not labels
+    return {'documents':len(rows),'negative_documents':negatives,'positive_documents_by_label':dict(docs),'entities_by_label':dict(entities)}

reproduction/pii/evaluation.py

@@ -0,0 +1,206 @@
+"""Exact typed-span evaluation and held-out pooled isotonic calibration.
+
+Gold offsets are Unicode character indices; predicted offsets are UTF-8 bytes.
+Predicted-span reliability excludes missed entities; recall is reported separately.
+"""
+import bisect
+import hashlib
+import json
+from pathlib import Path
+
+LABELS = ('first_name', 'last_name', 'email', 'phone_number', 'street_address', 'customer_id', 'account_number')
+
+
+def read_jsonl(path):
+    with open(path, encoding='utf-8') as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def write_json(path, obj):
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    Path(path).write_text(json.dumps(obj, indent=2, allow_nan=False) + '\n')
+
+
+def apply_calibration(score, calibration):
+    xs, ys = calibration['xs'], calibration['ys']
+    if score <= xs[0]:
+        return ys[0]
+    if score >= xs[-1]:
+        return ys[-1]
+    i = bisect.bisect_right(xs, score)
+    return ys[i-1] + (score-xs[i-1])/(xs[i]-xs[i-1])*(ys[i]-ys[i-1])
+
+
+def _index(records):
+    out = {}
+    for r in records:
+        if r['id'] in out:
+            raise ValueError(f"Duplicate document ID: {r['id']}")
+        out[r['id']] = r
+    return out
+
+
+def aligned(gold_records, prediction_records, calibration=None):
+    gold, predictions = _index(gold_records), _index(prediction_records)
+    if gold.keys() != predictions.keys():
+        raise ValueError('Gold/prediction document IDs differ; missing predictions cannot be silently omitted')
+    for doc_id, g in gold.items():
+        p = predictions[doc_id]
+        if g['text'] != p['text']:
+            raise ValueError(f'Text mismatch: {doc_id}')
+        offsets = [0]
+        for ch in g['text']:
+            offsets.append(offsets[-1] + len(ch.encode('utf-8')))
+        boundaries = set(offsets)
+        target = set()
+        for span in g['spans']:
+            a, b, label = span['start'], span['end'], span['label']
+            if label not in LABELS or not (isinstance(a, int) and isinstance(b, int) and 0 <= a < b < len(offsets)):
+                raise ValueError(f'Invalid gold span: {doc_id}: {span}')
+            target.add((label, offsets[a], offsets[b]))
+        found = {}
+        for span in p['findings']:
+            label, a, b = span['label'], span['start'], span['end']
+            raw = float(span['raw_score'])
+            if label not in LABELS or a not in boundaries or b not in boundaries or a >= b or not 0 <= raw <= 1:
+                raise ValueError(f'Invalid prediction: {doc_id}: {span}')
+            score = apply_calibration(raw, calibration) if calibration else span.get('confidence')
+            if score is None:
+                score = raw
+            score = float(score)
+            if not 0 <= score <= 1:
+                raise ValueError(f'Invalid confidence: {score}')
+            key = (label, a, b)
+            if key in found:
+                raise ValueError(f'Duplicate predicted span: {doc_id}: {key}')
+            found[key] = (raw, score)
+        yield doc_id, target, found
+
+
+def _prf(tp, predicted, gold):
+    precision = tp / predicted if predicted else 0.0
+    recall = tp / gold if gold else 0.0
+    return {'tp': tp, 'fp': predicted-tp, 'fn': gold-tp, 'predicted': predicted, 'support': gold,
+            'precision': precision, 'recall': recall, 'f1': 2*tp/(predicted+gold) if predicted+gold else 0.0}
+
+
+def _merge(spans):
+    merged = []
+    for a, b in sorted((s[1], s[2]) for s in spans):
+        if merged and a <= merged[-1][1]:
+            merged[-1][1] = max(b, merged[-1][1])
+        else:
+            merged.append([a, b])
+    return merged
+
+
+def _coverage(gold, pred):
+    g, p = _merge(gold), _merge(pred)
+    i = j = overlap = 0
+    while i < len(g) and j < len(p):
+        overlap += max(0, min(g[i][1], p[j][1])-max(g[i][0], p[j][0]))
+        if g[i][1] <= p[j][1]:
+            i += 1
+        else:
+            j += 1
+    return overlap, sum(b-a for a,b in p), sum(b-a for a,b in g)
+
+
+def reliability(rows, score_index):
+    bins = [{'lower': i/10, 'upper': (i+1)/10, 'support': 0, 'mean_score': 0.0, 'accuracy': 0.0} for i in range(10)]
+    squared = 0.0
+    for row in rows:
+        score, correct = row[score_index], row[2]
+        b = bins[min(9, int(score*10))]
+        b['support'] += 1
+        b['mean_score'] += score
+        b['accuracy'] += correct
+        squared += (score-correct)**2
+    ece = 0.0
+    for b in bins:
+        if b['support']:
+            b['mean_score'] /= b['support']
+            b['accuracy'] /= b['support']
+            ece += b['support'] * abs(b['mean_score']-b['accuracy'])
+        else:
+            b['mean_score'] = b['accuracy'] = None
+    n = len(rows)
+    return {'support': n, 'brier': squared/n if n else None, 'ece': ece/n if n else None,
+            'binning': '10 equal-width bins; left-inclusive, right-exclusive except final bin includes 1', 'bins': bins}
+
+
+def evaluate(gold_records, prediction_records, threshold=None, calibration=None):
+    if threshold is None and calibration:
+        threshold = calibration['threshold']
+    counts = {label: [0, 0, 0] for label in LABELS}
+    coverage = [0, 0, 0]
+    docs = dict(total=0, positive_support=0, clean_support=0, false_positive=0, false_negative=0,
+                with_any_missed_entity=0, with_any_spurious_entity=0)
+    rows = []
+    for _, target, found in aligned(gold_records, prediction_records, calibration):
+        accepted = {k for k, (_, s) in found.items() if threshold is None or s >= threshold}
+        for k, (raw, score) in found.items():
+            rows.append((raw, score, int(k in target)))
+        for label in LABELS:
+            gs, ps = {s for s in target if s[0] == label}, {s for s in accepted if s[0] == label}
+            for i, value in enumerate((len(gs & ps), len(ps), len(gs))):
+                counts[label][i] += value
+        for i, value in enumerate(_coverage(target, accepted)):
+            coverage[i] += value
+        docs['total'] += 1
+        docs['positive_support'] += bool(target)
+        docs['clean_support'] += not target
+        docs['false_positive'] += bool(accepted) and not target
+        docs['false_negative'] += bool(target) and not accepted
+        docs['with_any_missed_entity'] += bool(target-accepted)
+        docs['with_any_spurious_entity'] += bool(accepted-target)
+    return {'threshold': threshold, 'overall': _prf(*(sum(c[i] for c in counts.values()) for i in range(3))),
+            'per_label': {label: _prf(*c) for label, c in counts.items()},
+            'redaction_byte_coverage': _prf(*coverage), 'documents': docs,
+            'raw_reliability': reliability(rows, 0), 'confidence_reliability': reliability(rows, 1),
+            'reliability_population': 'All predicted spans before threshold, correctness = exact typed span. Missed gold entities excluded; see recall.',
+            'confidence_semantics': 'Pooled calibrated exact-span score' if calibration else 'Provided confidence when non-null; otherwise uncalibrated raw span score',
+            'coverage_semantics': 'Union of UTF-8 bytes; ignores labels. Zero-denominator P/R/F1 reported as 0.'}
+
+
+def fit_calibration(fit_gold, fit_predictions, select_gold, select_predictions):
+    fit_gold, select_gold = list(fit_gold), list(select_gold)
+    if {r['id'] for r in fit_gold} & {r['id'] for r in select_gold}:
+        raise ValueError('Calibration fit/select document IDs overlap')
+    hashes = lambda records: {hashlib.sha256(r['text'].encode()).hexdigest() for r in records}
+    if hashes(fit_gold) & hashes(select_gold):
+        raise ValueError('Calibration fit/select texts overlap')
+    fit_rows = [(raw, int(k in target)) for _, target, found in aligned(fit_gold, fit_predictions)
+                for k, (raw, _) in found.items()]
+    if not fit_rows:
+        raise ValueError('No predicted spans available to fit calibration')
+    from sklearn.isotonic import IsotonicRegression
+    model = IsotonicRegression(out_of_bounds='clip').fit([r[0] for r in fit_rows], [r[1] for r in fit_rows])
+    artifact = {'version': 1, 'method': 'isotonic', 'xs': model.X_thresholds_.tolist(), 'ys': model.y_thresholds_.tolist(),
+                'interpolation': 'linear; clip outside fitted x range', 'score_semantics': 'Estimated exact typed-span correctness for predicted spans; pooled labels; raw score = minimum predicted-class token softmax',
+                'experimental': True, 'fit_documents': len(fit_gold), 'fit_predicted_spans': len(fit_rows),
+                'fit_correct_spans': sum(r[1] for r in fit_rows), 'select_documents': len(select_gold),
+                'warnings': []}
+    if len({r[1] for r in fit_rows}) < 2:
+        artifact['warnings'].append('Fit targets contain only one outcome; constant map cannot demonstrate probability reliability.')
+    select_rows, support = [], 0
+    for _, target, found in aligned(select_gold, select_predictions, artifact):
+        support += len(target)
+        select_rows.extend((score, int(k in target)) for k, (_, score) in found.items())
+    if not select_rows or not support:
+        raise ValueError('Threshold selection requires predicted spans and gold entities')
+    select_rows.sort(reverse=True)
+    curve = []
+    tp = predicted = 0
+    i = 0
+    while i < len(select_rows):
+        threshold = select_rows[i][0]
+        while i < len(select_rows) and select_rows[i][0] == threshold:
+            tp += select_rows[i][1]
+            predicted += 1
+            i += 1
+        curve.append({'threshold': threshold, **_prf(tp, predicted, support)})
+    best = max(curve, key=lambda row: (row['f1'], row['recall'], row['threshold']))
+    artifact.update(threshold=best['threshold'], threshold_objective='Max exact typed-span F1 on calibration-select; ties favor recall, then higher threshold',
+                    select_operating_point=best, threshold_curve=curve)
+    return artifact

reproduction/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "datafog-pii"
+version = "0.0.0"
+description = "Local environment for the DataFog PII prototype"
+requires-python = ">=3.12,<3.13"
+dependencies = [
+    "torch",
+    "transformers",
+    "datasets",
+    "accelerate",
+    "safetensors",
+    "scikit-learn",
+    "onnx",
+    "onnxscript",
+    "onnxruntime",
+]
+
+[tool.uv]
+package = false

reproduction/scripts/benchmark_native.py

@@ -0,0 +1,48 @@
+"""Observe native CPU load and end-to-end request costs at actual input lengths."""
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import time
+import psutil
+
+def main():
+    root=Path(__file__).resolve().parents[1]
+    bundle=root/"artifacts/bundle"
+    binary=root/"rust/target/release/datafog-pii"
+    records=[json.loads(line) for line in (root/"data/prepared/validation.jsonl").read_text().splitlines()]
+    records.sort(key=lambda r:len(r["text"].encode()))
+    # Observed minimum, median, p90 and maximum validation lengths, not limits.
+    chosen=[records[i] for i in sorted({0,len(records)//2,int((len(records)-1)*0.9),len(records)-1})]
+    start=time.perf_counter()
+    process=subprocess.Popen([str(binary),str(bundle)],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,bufsize=1)
+    # Empty request includes runtime initialization plus minimal inference: upper bound on load.
+    process.stdin.write(json.dumps({"text":""})+"\n");process.stdin.flush()
+    first=json.loads(process.stdout.readline())
+    if "error" in first:raise RuntimeError(first)
+    cold=time.perf_counter()-start
+    monitor=psutil.Process(process.pid);peak=monitor.memory_info().rss
+    observations=[]
+    for row in chosen:
+        start=time.perf_counter()
+        process.stdin.write(json.dumps({"text":row["text"]})+"\n");process.stdin.flush()
+        answer=json.loads(process.stdout.readline())
+        elapsed=time.perf_counter()-start
+        if "error" in answer:raise RuntimeError(answer)
+        peak=max(peak,monitor.memory_info().rss)
+        observations.append({"id":row["id"],"bytes":len(row["text"].encode()),"latency_seconds":elapsed,"findings":len(answer["findings"])})
+    process.stdin.close();process.wait()
+    if process.returncode:raise RuntimeError(process.stderr.read())
+    shutil.copy2(binary,bundle/"datafog-pii")
+    sums={}
+    for path in sorted(bundle.rglob("*")):
+        if path.is_file() and path.name!="checksums.json":
+            sums[str(path.relative_to(bundle))]=hashlib.sha256(path.read_bytes()).hexdigest()
+    (bundle/"checksums.json").write_text(json.dumps(sums,indent=2)+"\n")
+    evidence={"backend":"native Rust + ONNX Runtime CPU, one inference thread","bundle_bytes":sum(p.stat().st_size for p in bundle.rglob("*") if p.is_file()),"cold_load_seconds":cold,"cold_load_definition":"process launch through first empty-input response; includes initialization and minimal inference","warm_requests":observations,"peak_rss_bytes":peak,"memory_definition":"maximum RSS sampled after responses, not allocator high-water mark","warm_observed_requests_per_second":len(observations)/sum(r["latency_seconds"] for r in observations),"sampling":"one request each at observed min/median/p90/max validation byte lengths; descriptive observations, no latency guarantee"}
+    (root/"artifacts/native-benchmark.json").write_text(json.dumps(evidence,indent=2)+"\n")
+    print(json.dumps(evidence,indent=2))
+
+if __name__=="__main__":main()

reproduction/scripts/calibrate.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from pii.evaluation import fit_calibration, read_jsonl, write_json
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description='Fit pooled isotonic calibration and independent experimental F1 threshold')
+    for name in ('fit-gold', 'fit-predictions', 'select-gold', 'select-predictions', 'output'):
+        p.add_argument('--' + name, required=True)
+    args = p.parse_args()
+    artifact = fit_calibration(*(read_jsonl(getattr(args, key)) for key in ('fit_gold', 'fit_predictions', 'select_gold', 'select_predictions')))
+    write_json(args.output, artifact)
+    print(f"Saved calibration: {args.output}; experimental threshold={artifact['threshold']}")

reproduction/scripts/check_chunk_boundaries.py

@@ -0,0 +1,63 @@
+"""Oracle BIO logits prove window mechanics independently of trained predictions."""
+import json
+import sys
+from pathlib import Path
+import numpy as np
+from transformers import AutoTokenizer
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+from pii.core import encode_windows, LABELS, byte_positions, decode_window, result
+
+root=Path(__file__).resolve().parents[1]
+tokenizer=AutoTokenizer.from_pretrained(root/'artifacts/backbone',local_files_only=True)
+max_length,stride=512,42
+cases=[]
+
+def make_case(name,prefix,entities,suffix):
+    text=prefix; spans=[]
+    for label,value in entities:
+        start=len(text);text+=value;spans.append({'label':label,'start':start,'end':len(text)});text+=' '
+    text+=suffix
+    bp=byte_positions(text)
+    expected=[{'label':s['label'],'start':bp[s['start']],'end':bp[s['end']]} for s in spans]
+    encoded=encode_windows(tokenizer,text,max_length,stride)
+    raw=tokenizer(text,add_special_tokens=False,truncation=False,return_offsets_mapping=True,verbose=False)
+    covered=[]
+    for wi,ids in enumerate(encoded['input_ids']):covered.extend(ids[1 if wi==0 else stride+1:-1])
+    assert covered==raw['input_ids'], 'manual windows lost or duplicated tokens outside overlap'
+    windows=[]; findings=[]; covers=[0]*len(spans); clips=0
+    for wi,offsets in enumerate(encoded['offset_mapping']):
+        content=[(a,b) for a,b in offsets if a!=b]
+        complete=[content[0][0]<=s['start'] and content[-1][1]>=s['end'] for s in spans]
+        covers[:]=[v+int(c) for v,c in zip(covers,complete)]
+        logits=np.full((len(offsets),15),-8,dtype=np.float32)
+        for i,(a,b) in enumerate(offsets):
+            label_id=0; peak=12.
+            if a!=b:
+                for si,s in enumerate(spans):
+                    if a<s['end'] and s['start']<b:
+                        assert s['start']<=a and b<=s['end'], 'fixture boundary unrepresentable'
+                        label_id=1+2*LABELS.index(s['label'])+int(a!=s['start'])
+                        peak=5. if complete[si] else 12.
+                        break
+            logits[i,label_id]=peak
+        decoded=decode_window(logits,offsets,text,max_length,wi>0,wi+1<len(encoded['input_ids']))
+        clips+=sum(f.get('_boundary_fragment',False) for f in decoded)
+        findings.extend(decoded)
+        windows.append({'input_ids':encoded['input_ids'][wi],'offsets':[[bp[a],bp[b]] for a,b in offsets], 'logits':logits.tolist()})
+    actual=result(text,findings)
+    assert [{k:f[k] for k in ('label','start','end')} for f in actual['findings']]==expected,(name,actual,expected)
+    assert all(covers), 'no complete window for entity'
+    # Demonstrate the old greedy rule really loses exact spans on boundary cases.
+    old=result(text,[{k:v for k,v in f.items() if k!='_boundary_fragment'} for f in findings])
+    old_wrong=[{k:f[k] for k in ('label','start','end')} for f in old['findings']]!=expected
+    if name!='adjacent-repeated-unicode':assert clips and old_wrong
+    cases.append({'name':name,'text':text,'expected':expected,'redacted':actual['redacted'],'windows':windows,'complete_window_counts':covers,'edge_predictions':clips,'old_greedy_would_fail':old_wrong,'full_token_coverage':True,'raw_tokens':len(raw['input_ids'])})
+
+make_case('boundary-email','🙂 '+'hello '*505,[('email','jose.garcia@example.com')],'end '*30)
+make_case('max43-token-boundary-entity','🙂 '+'hello '*490,[('street_address',' '.join(['alpha']*43))],'end '*30)
+make_case('adjacent-repeated-unicode','🙂 ',[('first_name','José'),('last_name','García'),('first_name','José'),('last_name','García')],'done')
+fixture={'max_length':max_length,'stride':stride,'labels':LABELS,'cases':cases}
+(root/'rust/tests/boundary-oracle.json').write_text(json.dumps(fixture,separators=(',',':'))+'\n')
+report={'max_length':max_length,'stride':stride,'policy':'Artificial-edge fragments suppressed only when a strictly containing same-label prediction exists. Remaining candidates use original greedy score ordering.', 'proof':'Oracle true BIO labels; partial-window probabilities intentionally exceed complete-window probabilities.', 'cases':[{k:v for k,v in c.items() if k not in ('text','windows','redacted')} for c in cases]}
+(root/'artifacts/boundary-proof.json').write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps(report,indent=2))

reproduction/scripts/check_environment.py

@@ -0,0 +1,40 @@
+"""Small environment smoke check; does not download data or train a PII model."""
+import importlib.metadata
+import json
+import platform
+from pathlib import Path
+
+import onnxruntime as ort
+import torch
+
+result = {
+    "platform": platform.platform(),
+    "python": platform.python_version(),
+    "versions": {name: importlib.metadata.version(name) for name in (
+        "torch", "transformers", "datasets", "accelerate", "safetensors",
+        "scikit-learn", "onnx", "onnxscript", "onnxruntime",
+    )},
+    "mps_built": torch.backends.mps.is_built(),
+    "mps_available": torch.backends.mps.is_available(),
+    "onnxruntime_providers": ort.get_available_providers(),
+}
+if result["mps_available"]:
+    torch.manual_seed(0)
+    model = torch.nn.Linear(8, 2).to("mps")
+    optimizer = torch.optim.AdamW(model.parameters())
+    before = model.weight.detach().clone()
+    loss = model(torch.ones(4, 8, device="mps")).square().mean()
+    loss.backward()
+    finite_gradients = all(torch.isfinite(p.grad).all().item() for p in model.parameters())
+    optimizer.step()
+    torch.mps.synchronize()
+    result["mps_smoke_check"] = {
+        "finite_loss": torch.isfinite(loss).item(),
+        "finite_gradients": finite_gradients,
+        "weights_changed": not torch.equal(before, model.weight),
+    }
+    assert all(result["mps_smoke_check"].values()), result
+out = Path(__file__).resolve().parents[1] / "artifacts" / "environment-check.json"
+out.parent.mkdir(exist_ok=True)
+out.write_text(json.dumps(result, indent=2) + "\n")
+print(json.dumps(result, indent=2))

reproduction/scripts/check_rust_parity.py

@@ -0,0 +1,69 @@
+"""Check native Rust IDs, masks, offsets, logits, decoder/redaction and explicit errors."""
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+import numpy as np
+import onnxruntime as ort
+from transformers import AutoTokenizer
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from pii.core import encode_windows, byte_positions, decode_window, result, Predictor
+
+def main():
+    p=argparse.ArgumentParser()
+    p.add_argument('bundle',type=Path)
+    p.add_argument('--binary',type=Path,default=Path('rust/target/release/datafog-pii'))
+    p.add_argument('--output',type=Path)
+    args=p.parse_args()
+    config=json.loads((args.bundle/'config.json').read_text())
+    tokenizer=AutoTokenizer.from_pretrained(args.bundle,local_files_only=True)
+    opts=ort.SessionOptions(); opts.intra_op_num_threads=1
+    session=ort.InferenceSession(str(args.bundle/'model.onnx'),sess_options=opts,providers=['CPUExecutionProvider'])
+    predictor=Predictor(args.bundle)
+    calibration=json.loads((args.bundle/config['calibration_file']).read_text()) if config.get('calibration_file') else None
+    texts=['','Alice Smith','🙂 José García: jose@example.com\nAccount: ABC123','Email marketing is useful.','Ann Ann, phone 415-555-0123.','Customer message. '*550+'\nJosé García <jose@example.com>','Order delay. '*170+'é Smith\n'+'More detail. '*180]
+    payload=''.join(json.dumps({'text':text})+'\n' for text in texts)+'{"wrong":1}\n'+json.dumps({'text':'Recovery after invalid input'})+'\n'
+    start=time.perf_counter()
+    proc=subprocess.run([str(args.binary.resolve()),str(args.bundle.resolve()),'--debug'],input=payload,text=True,capture_output=True,check=True)
+    elapsed=time.perf_counter()-start
+    outputs=[json.loads(line) for line in proc.stdout.splitlines()]
+    assert len(outputs)==len(texts)+2
+    assert 'error' in outputs[-2] and 'contains_pii' not in outputs[-2],outputs[-2]
+    assert 'error' not in outputs[-1],outputs[-1]
+    max_diff=0.; checks=[]
+    for text,out in zip(texts,outputs):
+        assert 'error' not in out,out
+        encoded=encode_windows(tokenizer,text,config['max_length'],config['stride'])
+        assert len(encoded['input_ids'])==len(out['windows'])
+        findings=[]
+        for i,window in enumerate(out.pop('windows')):
+            assert window['input_ids']==encoded['input_ids'][i]
+            assert window['attention_mask']==encoded['attention_mask'][i]
+            bp=byte_positions(text)
+            assert window['offsets']==[[bp[a],bp[b]] for a,b in encoded['offset_mapping'][i]]
+            logits=session.run(None,{k:np.array([encoded[k][i]],dtype=np.int64) for k in ('input_ids','attention_mask')})[0][0]
+            rust_logits=np.array(window['logits'],dtype=np.float32)
+            diff=float(np.abs(logits-rust_logits).max()); max_diff=max(max_diff,diff)
+            # Same CPU runtime and one-thread settings: require exact f32 values.
+            np.testing.assert_array_equal(logits,rust_logits)
+            findings.extend(decode_window(logits,encoded['offset_mapping'][i],text,config['max_length'], i > 0, i + 1 < len(encoded['input_ids'])))
+        expected=result(text,findings,calibration,calibration.get('threshold') if calibration else None)
+        public=predictor.predict(text)
+        assert public==expected, 'actual Predictor differs from shared reference'
+        assert out['contains_pii']==expected['contains_pii']
+        assert out['redacted']==expected['redacted']
+        assert len(out['findings'])==len(expected['findings'])
+        for actual,want in zip(out['findings'],expected['findings']):
+            for key in ('start','end','label'):assert actual[key]==want[key]
+            # f64 scalar summation vs numpy reduction differ at rounding scale.
+            assert abs(actual['raw_score']-want['raw_score'])<=64*np.finfo(np.float64).eps
+            if want['confidence'] is None: assert actual['confidence'] is None
+            else: assert abs(actual['confidence']-want['confidence'])<=1e-10
+        checks.append({'bytes':len(text.encode()),'windows':len(encoded['input_ids']),'findings':len(out['findings'])})
+    report={'bundle':str(args.bundle),'cases':checks,'max_abs_logit_difference':max_diff,'logits_bit_identical':True,'token_ids_masks_global_byte_offsets_parity':True,'findings_redaction_parity':True,'actual_python_predictor_parity':True,'malformed_request_explicit_error_and_recovery':True,'cold_process_debug_batch_seconds':elapsed,'note':'Parity fixtures prove implementation consistency, not detection quality.'}
+    serialized=json.dumps(report,indent=2)+'\n'
+    if args.output:args.output.write_text(serialized)
+    print(serialized)
+if __name__=='__main__':main()

reproduction/scripts/check_training.py

@@ -0,0 +1,67 @@
+"""Diagnostic learning and exact optimizer/RNG resume proof; not held-out quality."""
+import sys,json,time,random,hashlib,gc
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+sys.path.insert(0,str(Path(__file__).resolve().parent))
+import numpy as np
+import torch
+from transformers import AutoTokenizer,AutoModelForTokenClassification,DataCollatorForTokenClassification
+from pii.core import LABELS,BIO,decode_window,byte_positions
+from train import prepare
+
+out=Path('work/training-diagnostic');out.mkdir(parents=True,exist_ok=True)
+random.seed(42);np.random.seed(42);torch.manual_seed(42)
+tok=AutoTokenizer.from_pretrained('artifacts/backbone',local_files_only=True)
+# Shortest complete documents providing every target label keeps diagnostic bounded.
+rows=[json.loads(x) for x in open('data/prepared/train.jsonl')]
+rows.sort(key=lambda r:(len(r['text']),r['id']))
+selected=[];missing=set(LABELS)
+for r in rows:
+    labels={s['label'] for s in r['spans']}
+    if labels & missing:selected.append(r);missing-=labels
+    if not missing:break
+chunks,stats=prepare(selected,tok,512,42)
+collate=DataCollatorForTokenClassification(tok,return_tensors='pt')
+batch={k:v.to('mps') for k,v in collate(chunks).items() if k in ('input_ids','attention_mask','labels')}
+model=AutoModelForTokenClassification.from_pretrained('artifacts/backbone',num_labels=len(BIO),id2label=dict(enumerate(BIO)),label2id={v:k for k,v in enumerate(BIO)},attn_implementation='eager',local_files_only=True).to('mps')
+opt=torch.optim.AdamW(model.parameters(),lr=2e-5,weight_decay=.01)
+offsets=[tok(r['text'],return_offsets_mapping=True)['offset_mapping'] for r in selected]
+gold=[]
+for r in selected:
+    bp=byte_positions(r['text']);gold.append({(s['label'],bp[s['start']],bp[s['end']]) for s in r['spans']})
+def assess():
+    model.eval()
+    with torch.no_grad():o=model(**batch)
+    found=0;pred=0
+    for r,off,logits,g in zip(selected,offsets,o.logits.cpu().numpy(),gold):
+        p={(f['label'],f['start'],f['end']) for f in decode_window(logits[:len(off)],off,r['text'],512)}
+        found+=len(p&g);pred+=len(p)
+    return {'loss':o.loss.item(),'exact_entity_true_positives':found,'gold_entities':sum(map(len,gold)),'predicted_entities':pred,'recall':found/sum(map(len,gold))}
+def step():
+    model.train();opt.zero_grad(set_to_none=True);loss=model(**batch).loss
+    assert torch.isfinite(loss);loss.backward();torch.nn.utils.clip_grad_norm_(model.parameters(),1.,error_if_nonfinite=True);opt.step();return loss.item()
+def fingerprint():
+    h=hashlib.sha256()
+    for n,p in model.named_parameters():h.update(n.encode());h.update(p.detach().cpu().numpy().tobytes())
+    return h.hexdigest()
+start=time.perf_counter();before=assess();history=[]
+while True:
+    loss=step();a=assess();history.append(a)
+    if a['loss']<before['loss'] and a['recall']>before['recall']:break
+    if time.perf_counter()-start>180:raise RuntimeError('Diagnostic 3-minute coordination budget exhausted; no learning claim')
+model.save_pretrained(out/'model')
+state={'optimizer':opt.state_dict(),'random':random.getstate(),'numpy':np.random.get_state(),'torch':torch.get_rng_state(),'mps':torch.mps.get_rng_state()}
+torch.save(state,out/'state.pt');del state
+uninterrupted_loss=step();uninterrupted_hash=fingerprint();uninterrupted=assess();torch.save({k:v.detach().cpu() for k,v in model.state_dict().items()},out/'uninterrupted.pt')
+del model,opt;gc.collect();torch.mps.empty_cache()
+model=AutoModelForTokenClassification.from_pretrained(out/'model',local_files_only=True,attn_implementation='eager').to('mps')
+opt=torch.optim.AdamW(model.parameters(),lr=2e-5,weight_decay=.01)
+state=torch.load(out/'state.pt',map_location='cpu',weights_only=False);opt.load_state_dict(state['optimizer'])
+random.setstate(state['random']);np.random.set_state(state['numpy']);torch.set_rng_state(state['torch']);torch.mps.set_rng_state(state['mps']);del state
+resumed_loss=step();resumed_hash=fingerprint();resumed=assess()
+reference=torch.load(out/'uninterrupted.pt',weights_only=True)
+max_parameter_difference=max((v.detach().cpu()-reference[k]).abs().max().item() for k,v in model.state_dict().items())
+del reference
+report={'purpose':'Training functionality diagnostic only; same examples fitted and scored, no benchmark','device':'mps','seed':42,'recipe':{'lr':2e-5,'weight_decay':.01,'gradient_clip':1.},'selected_ids':[r['id'] for r in selected],'labels_covered':LABELS,'chunk_stats':stats,'steps_to_first_learning_evidence':len(history),'stopping_rule':'First evaluation with both lower token loss and greater exact-entity recall than random-head baseline','before':before,'after':history[-1],'resume':{'checkpoint':'work/training-diagnostic','uninterrupted_next_update_loss':uninterrupted_loss,'resumed_next_update_loss':resumed_loss,'uninterrupted_parameters_sha256':uninterrupted_hash,'resumed_parameters_sha256':resumed_hash,'all_model_parameters_bitwise_equal':uninterrupted_hash==resumed_hash,'max_parameter_abs_difference':max_parameter_difference,'resume_status':'Same next-update loss and evaluation; MPS parameter differences reported without a bitwise guarantee','uninterrupted_metrics':uninterrupted,'resumed_metrics':resumed},'elapsed_seconds':time.perf_counter()-start}
+Path('artifacts/training-check.json').write_text(json.dumps(report,indent=2));print(json.dumps(report,indent=2),flush=True)
+assert uninterrupted_loss==resumed_loss and uninterrupted==resumed,'Resumed loss or outputs differ'

reproduction/scripts/evaluate.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from pii.evaluation import evaluate, read_jsonl, write_json
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description='Evaluate complete saved span predictions against character-offset gold JSONL')
+    for name in ('gold', 'predictions', 'output'):
+        p.add_argument('--' + name, required=True)
+    p.add_argument('--calibration')
+    p.add_argument('--threshold', type=float)
+    args = p.parse_args()
+    calibration = json.loads(Path(args.calibration).read_text()) if args.calibration else None
+    result = evaluate(read_jsonl(args.gold), read_jsonl(args.predictions), args.threshold, calibration)
+    write_json(args.output, result)
+    print(json.dumps({'overall': result['overall'], 'documents': result['documents']}))

reproduction/scripts/export_model.py

@@ -0,0 +1,52 @@
+"""Export selected local checkpoint into a native inference bundle."""
+import hashlib
+import json
+import shutil
+import sys
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+import numpy as np
+import onnxruntime as ort
+import torch
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+from pii.core import LABELS, MODEL_REVISION
+
+class Export(torch.nn.Module):
+    def __init__(self,model):
+        super().__init__();self.model=model
+    def forward(self,input_ids,attention_mask):
+        return self.model(input_ids=input_ids,attention_mask=attention_mask).logits
+
+def main():
+    out=Path("artifacts/bundle");out.mkdir(parents=True,exist_ok=True)
+    training=json.loads(Path("artifacts/training/run-config.json").read_text())
+    source=Path("artifacts/training/best")
+    tokenizer=AutoTokenizer.from_pretrained(source,local_files_only=True)
+    model=AutoModelForTokenClassification.from_pretrained(source,local_files_only=True,attn_implementation="eager").eval()
+    sample=tokenizer("🙂 José Smith: jose@example.com",return_tensors="pt")
+    tokenizer.save_pretrained(out)
+    config={"labels":LABELS,"max_length":training["max_length"],"stride":training["stride"],"model_revision":MODEL_REVISION,"dataset_revision":"b70ffaf5ff39e079776134c5bf4381f00a9fd1ed","offsets":"half-open UTF-8 bytes","raw_score":"minimum predicted-class token softmax within span","status":"experimental prototype, uncalibrated until calibration_file supplied"}
+    (out/"config.json").write_text(json.dumps(config,indent=2)+"\n")
+    wrapper=Export(model).eval()
+    torch.onnx.export(wrapper, (sample["input_ids"],sample["attention_mask"]),str(out/"model.onnx"),input_names=["input_ids","attention_mask"],output_names=["logits"],dynamic_axes={"input_ids":{0:"batch",1:"sequence"},"attention_mask":{0:"batch",1:"sequence"},"logits":{0:"batch",1:"sequence"}},opset_version=17,dynamo=False)
+    assert not model.training, "Export must preserve evaluation mode for parity checks"
+    dylibs=list((Path(ort.__file__).parent/"capi").glob("libonnxruntime.*.dylib"))
+    if len(dylibs)!=1:raise RuntimeError(f"Ambiguous native runtime paths: {dylibs}")
+    shutil.copy2(dylibs[0],out/"libonnxruntime.dylib")
+    shutil.copytree("rust/licenses",out/"licenses",dirs_exist_ok=True)
+    shutil.copy2("artifacts/backbone/README.md",out/"licenses/backbone-model-card.md")
+    (out/"licenses/dataset-attribution.txt").write_text("Nemotron-PII by NVIDIA, CC BY 4.0. Source: https://huggingface.co/datasets/nvidia/Nemotron-PII\nRevision b70ffaf5ff39e079776134c5bf4381f00a9fd1ed. Training filters and preparation modifications: see artifacts/data-manifest.json and docs/data.md.\n")
+    options=ort.SessionOptions();options.intra_op_num_threads=1
+    session=ort.InferenceSession(str(out/"model.onnx"),sess_options=options,providers=["CPUExecutionProvider"])
+    checks=[]
+    for texts in ([""],["Alice Smith", "🙂 José García: jose@example.com"],["Long ticket. "*170]):
+        x=tokenizer(texts,return_tensors="pt",padding=True,truncation=True,max_length=training["max_length"])
+        x={k:v for k,v in x.items() if k in ("input_ids","attention_mask")}
+        with torch.no_grad():expected=model(**x).logits.numpy()
+        actual=session.run(None,{k:v.numpy() for k,v in x.items()})[0]
+        checks.append({"batch":len(texts),"sequence":x["input_ids"].shape[1],"max_abs_difference":float(np.abs(expected-actual).max()),"argmax_identical":bool(np.array_equal(expected.argmax(-1),actual.argmax(-1)))})
+        if not checks[-1]["argmax_identical"]:raise RuntimeError("Export changed probe token predictions")
+    (out/"export-evidence.json").write_text(json.dumps(checks,indent=2)+"\n")
+    print(json.dumps(checks,indent=2),flush=True)
+
+if __name__=="__main__":main()

reproduction/scripts/final_audit.py

@@ -0,0 +1,37 @@
+import hashlib,json,subprocess
+from pathlib import Path
+root=Path.cwd(); bundle=root/'artifacts/bundle'
+def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
+freeze=json.loads((root/'artifacts/pipeline/evaluation-freeze.json').read_text())['sha256']
+assert all(sha(root/p)==v for p,v in freeze.items()),'freeze changed'
+checks=json.loads((bundle/'checksums.json').read_text())
+assert all(sha(bundle/p)==v for p,v in checks.items()),'bundle checksum mismatch'
+cal=json.loads((bundle/'calibration.json').read_text())
+import bisect
+def confidence(x):
+ xs,ys=cal['xs'],cal['ys']
+ if x<=xs[0]:return ys[0]
+ if x>=xs[-1]:return ys[-1]
+ i=bisect.bisect_right(xs,x)
+ return ys[i-1]+(x-xs[i-1])*(ys[i]-ys[i-1])/(xs[i]-xs[i-1])
+n=tp=fp=fn=0
+with open('data/prepared/test.jsonl') as gold,open('artifacts/predictions/test.jsonl') as pred:
+ import itertools
+ for g,p in itertools.zip_longest(gold,pred):
+  assert g and p
+  g,p=json.loads(g),json.loads(p)
+  assert g['id']==p['id'] and g['text']==p['text']
+  text=g['text']; offsets=[0]
+  for c in text:offsets.append(offsets[-1]+len(c.encode()))
+  targets={(s['label'],offsets[s['start']],offsets[s['end']]) for s in g['spans']}
+  findings={(s['label'],s['start'],s['end']) for s in p['findings'] if confidence(s['raw_score'])>=cal['threshold']}
+  tp+=len(targets&findings);fp+=len(findings-targets);fn+=len(targets-findings);n+=1
+expected=json.loads(Path('artifacts/evaluation/test.json').read_text())
+assert [tp,fp,fn]==[expected['overall'][k] for k in ('tp','fp','fn')]
+assert n==90018
+proc=subprocess.run([str(bundle/'datafog-pii'),str(bundle)],input=json.dumps({'text':'🙂 Email me at alice@example.com.'})+'\n'+json.dumps({'text':'The screen is blank.'})+'\n',text=True,capture_output=True,check=True,env={'PATH':'/usr/bin:/bin'})
+outputs=[json.loads(x) for x in proc.stdout.splitlines()]
+assert len(outputs)==2 and all('error' not in x for x in outputs)
+report={'frozen_inputs_unchanged':True,'bundle_checksums_verified':len(checks),'independent_metric_recount':{'documents':n,'tp':tp,'fp':fp,'fn':fn},'packaged_native_smoke_without_python_path':outputs}
+Path('artifacts/final-audit.json').write_text(json.dumps(report,indent=2)+'\n')
+print(json.dumps(report,indent=2))

reproduction/scripts/predict.py

@@ -0,0 +1,66 @@
+"""Save unfiltered predictions from the deployed ONNX model for evaluation."""
+import argparse
+from collections import Counter
+import json
+import sys
+import time
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+import numpy as np
+import onnxruntime as ort
+from transformers import AutoTokenizer
+from pii.core import decode_window, result, encode_windows
+
+def main():
+    p=argparse.ArgumentParser()
+    p.add_argument("--bundle",type=Path,default=Path("artifacts/bundle"))
+    p.add_argument("--input",type=Path,required=True)
+    p.add_argument("--output",type=Path,required=True)
+    p.add_argument("--batch-size",type=int,default=16,help="Measured training recipe batch; dynamically padded ONNX batches")
+    args=p.parse_args()
+    config=json.loads((args.bundle/"config.json").read_text())
+    tokenizer=AutoTokenizer.from_pretrained(args.bundle,local_files_only=True)
+    options=ort.SessionOptions();options.intra_op_num_threads=1
+    session=ort.InferenceSession(str(args.bundle/"model.onnx"),sess_options=options,providers=["CPUExecutionProvider"])
+    args.output.parent.mkdir(parents=True,exist_ok=True)
+    started=time.perf_counter();documents=windows=0
+    def batch_predict(rows,handle):
+        nonlocal documents,windows
+        encoded={"input_ids":[],"attention_mask":[],"offset_mapping":[]}
+        mapping=[]
+        for i,row in enumerate(rows):
+            windows_for_row=encode_windows(tokenizer,row["text"],config["max_length"],config["stride"])
+            for key in encoded:encoded[key].extend(windows_for_row[key])
+            mapping.extend([i]*len(windows_for_row["input_ids"]))
+        counts=Counter(mapping);seen=Counter();findings=[[] for _ in rows]
+        for begin in range(0,len(mapping),args.batch_size):
+            end=min(len(mapping),begin+args.batch_size)
+            length=max(len(x) for x in encoded["input_ids"][begin:end])
+            ids=np.full((end-begin,length),tokenizer.pad_token_id,dtype=np.int64)
+            masks=np.zeros_like(ids)
+            for j,index in enumerate(range(begin,end)):
+                n=len(encoded["input_ids"][index]);ids[j,:n]=encoded["input_ids"][index];masks[j,:n]=encoded["attention_mask"][index]
+            logits=session.run(None,{"input_ids":ids,"attention_mask":masks})[0]
+            for j,index in enumerate(range(begin,end)):
+                sample=mapping[index];offsets=encoded["offset_mapping"][index]
+                findings[sample].extend(decode_window(logits[j,:len(offsets)],offsets,rows[sample]["text"],config["max_length"],has_previous=seen[sample]>0,has_next=seen[sample]+1<counts[sample]))
+                seen[sample]+=1
+        for row,found in zip(rows,findings):
+            prediction={"id":row["id"],"text":row["text"],**result(row["text"],found)}
+            handle.write(json.dumps(prediction,ensure_ascii=False)+"\n")
+        documents+=len(rows);windows+=len(mapping)
+        args.output.with_suffix(".progress.json").write_text(json.dumps({"documents":documents,"windows":windows,"seconds":time.perf_counter()-started})+"\n")
+    temporary=args.output.with_suffix(".partial.jsonl")
+    with args.input.open() as source,temporary.open("w") as target:
+        rows=[]
+        for line in source:
+            if not line.strip():continue
+            rows.append(json.loads(line))
+            if len(rows)==args.batch_size:batch_predict(rows,target);rows=[]
+        if rows:batch_predict(rows,target)
+    temporary.replace(args.output)
+    evidence={"documents":documents,"windows":windows,"seconds":time.perf_counter()-started,"backend":"ONNX Runtime CPU","batch_size":args.batch_size,"raw_unfiltered":True}
+    args.output.with_suffix(".evidence.json").write_text(json.dumps(evidence,indent=2)+"\n")
+    print(json.dumps(evidence),flush=True)
+
+if __name__=="__main__":main()

reproduction/scripts/prepare_data.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Run from project root: .venv/bin/python scripts/prepare_data.py."""
+import sys
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+import json, math, hashlib, bisect
+from collections import Counter, defaultdict
+import numpy as np
+import pyarrow.parquet as pq
+from huggingface_hub import hf_hub_download
+from transformers import AutoTokenizer
+from pii.data import LABELS, DATA_REVISION, MODEL_REVISION, validate, digest, family_key, support
+
+ROOT=Path(__file__).resolve().parents[1]
+RAW=ROOT/'data/raw'; OUT=ROOT/'data/prepared'; ART=ROOT/'artifacts'
+OUT.mkdir(parents=True,exist_ok=True); ART.mkdir(exist_ok=True)
+manifest={'source':'nvidia/Nemotron-PII','revision':DATA_REVISION,'license':'CC-BY-4.0','attribution':'NVIDIA; Amy Steier, Andre Manoel, Alexa Haushalter, Maarten Van Segbroeck, Nemotron-PII (2025). https://huggingface.co/datasets/nvidia/Nemotron-PII','source_files':{},'source_stats':{},'exclusions':{}}
+files=['README.md']+[f'data/{s}-00000-of-00001.parquet' for s in ('train','test')]
+for f in files:
+    p=Path(hf_hub_download(manifest['source'],f,repo_type='dataset',revision=DATA_REVISION,local_dir=RAW))
+    manifest['source_files'][f]={'sha256':hashlib.sha256(p.read_bytes()).hexdigest(),'bytes':p.stat().st_size}
+rows=[]; excluded=[]; parent=[]; key_owner={}
+def find(i):
+    while parent[i]!=i:
+        parent[i]=parent[parent[i]]; i=parent[i]
+    return i
+def union(i,j):
+    a,b=find(i),find(j)
+    if a!=b: parent[max(a,b)]=min(a,b)
+for source in ('train','test'):
+    table=pq.read_table(RAW/f'data/{source}-00000-of-00001.parquet')
+    stats={'rows':len(table),'labels':Counter(),'invalid':Counter(),'empty_all_spans':0,'semantic_flags':Counter()}
+    seen_uids=Counter(); seen_texts=Counter()
+    for n,r in enumerate(table.to_pylist()):
+        seen_uids[r['uid']]+=1; seen_texts[digest(r['text'])]+=1
+        spans,reason=validate(r)
+        if reason:
+            stats['invalid'][reason]+=1; excluded.append({'source':source,'index':n,'uid':r['uid'],'reason':reason}); continue
+        stats['labels'].update(s['label'] for s in spans); stats['empty_all_spans']+=not spans
+        for s in spans:
+            if s['label']=='email' and '@' not in s['text']:
+                stats['semantic_flags']['email_without_at']+=1
+        row={'id':f'{source}:{n}:{r["uid"]}','uid':r['uid'],'source':source,'source_index':n,'text':r['text'],'spans':[{k:s[k] for k in ('label','start','end')} for s in spans if s['label'] in LABELS], 'domain':r['domain'],'document_type':r['document_type'],'family':family_key(r['text'],spans),'text_sha256':digest(r['text'])}
+        i=len(rows); parent.append(i)
+        for key in ('uid:'+row['uid'],'text:'+row['text_sha256'],'family:'+row['family']):
+            if key in key_owner: union(i,key_owner[key])
+            else:key_owner[key]=i
+        rows.append(row)
+    stats['duplicate_uid_rows']=sum(v-1 for v in seen_uids.values()); stats['duplicate_exact_text_rows']=sum(v-1 for v in seen_texts.values())
+    manifest['source_stats'][source]=stats
+    print(source,json.dumps(stats),flush=True)
+groups=defaultdict(list)
+for i,row in enumerate(rows): groups[find(i)].append(row)
+splits={s:[] for s in ('train','validation','calibration_fit','calibration_select','test')}
+train_groups=[]; overlaps=0
+for group in groups.values():
+    group_id=min(r['id'] for r in group)
+    for r in group:r['group']=digest(group_id)
+    test=[r for r in group if r['source']=='test']
+    if test:
+        splits['test'].extend(test)
+        for r in group:
+            if r['source']=='train':
+                excluded.append({'source':'train','index':r['source_index'],'uid':r['uid'],'reason':'test_family_overlap'}); overlaps+=1
+    else:train_groups.append(group)
+# Stable pseudorandom family order. Allocate heldouts jointly, without model predictions.
+train_groups.sort(key=lambda g:digest('datafog-pii-split-v1:'+g[0]['group']))
+# Owner-approved transparent volume-derived heuristic; no quality guarantee.
+all_support=Counter()
+for group in train_groups:
+    for r in group: all_support.update(set(s['label'] for s in r['spans']) or ['__negative__'])
+quota=math.ceil(math.sqrt(min(all_support[l] for l in LABELS)))
+heldouts=('validation','calibration_fit','calibration_select'); counts={s:Counter() for s in heldouts}
+for group in train_groups:
+    coverage=Counter()
+    for r in group:
+        labels=set(s['label'] for s in r['spans']); coverage.update(labels or ['__negative__'])
+    deficits={s:sum(min(v,max(0,quota-counts[s][l])) for l,v in coverage.items()) for s in heldouts}
+    chosen=max(heldouts,key=lambda s:(deficits[s],-len(splits[s])))
+    if deficits[chosen]==0:chosen='train'
+    splits[chosen].extend(group)
+    if chosen in counts:counts[chosen].update(coverage)
+manifest['split_rule']={'quota_positive_documents_per_label_and_negative_documents':quota,'derivation':'ceil(sqrt(minimum observed eligible positive-document support across target labels)); equal support in three heldouts. Transparent prototype allocation heuristic balancing training retention with heldout growth; no statistical precision or quality guarantee.','eligible_train_document_support':dict(all_support),'grouping':'Connected components of UID, exact original text SHA256, and whitespace-collapsed casefolded all-label-masked text SHA256. All train groups touching test excluded. Source test retained in full except structurally invalid rows.','allocation':'Stable SHA256 family order; assign each family to heldout with greatest remaining document-support deficit (tie: smaller partition), otherwise train.','seed':'datafog-pii-split-v1','test_overlap_train_rows':overlaps,'heldout_achieved_support':counts}
+manifest['splits']={}
+for name,records in splits.items():
+    records.sort(key=lambda r:r['id'])
+    p=OUT/f'{name}.jsonl'
+    with p.open('w') as f:
+        for r in records:f.write(json.dumps(r,ensure_ascii=False)+'\n')
+    manifest['splits'][name]={**support(records),'sha256':hashlib.sha256(p.read_bytes()).hexdigest()}
+    print(name,json.dumps(manifest['splits'][name]),flush=True)
+sets={name:set(r['group'] for r in records) for name,records in splits.items()}
+for a in sets:
+    for b in sets:
+        if a<b:assert sets[a].isdisjoint(sets[b]),(a,b)
+manifest['pairwise_group_disjoint']=True
+manifest['exclusions']=dict(Counter(r['reason'] for r in excluded))
+with (OUT/'excluded.jsonl').open('w') as f:
+    for r in excluded:f.write(json.dumps(r)+'\n')
+(ART/'data-manifest.json').write_text(json.dumps(manifest,indent=2))
+print('Tokenizing full eligible training-side data for length/alignment statistics',flush=True)
+tokenizer=AutoTokenizer.from_pretrained('distilbert/distilbert-base-cased',revision=MODEL_REVISION)
+tokenizer.save_pretrained(ROOT/'artifacts/tokenizer')
+length_stats={}; entity_lengths=defaultdict(list); boundary_bad=Counter(); boundary_examples=[]
+for name in ('train','validation','calibration_fit','calibration_select'):
+    lengths=[]
+    for base in range(0,len(splits[name]),256):
+        batch=splits[name][base:base+256]
+        encoded=tokenizer([r['text'] for r in batch],add_special_tokens=False,return_offsets_mapping=True,truncation=False,verbose=False)
+        for r,ids,offsets in zip(batch,encoded['input_ids'],encoded['offset_mapping']):
+            lengths.append(len(ids)); starts=[a for a,b in offsets]; ends=[b for a,b in offsets]
+            for s in r['spans']:
+                i=bisect.bisect_right(ends,s['start']); j=bisect.bisect_left(starts,s['end'])
+                entity_lengths[s['label']].append(j-i)
+                if i>=j or starts[i]!=s['start'] or ends[j-1]!=s['end']:
+                    boundary_bad[s['label']]+=1
+                    if len(boundary_examples)<30:boundary_examples.append({'id':r['id'],'span':s,'text':r['text'][s['start']:s['end']],'covering_offsets':offsets[i:j]})
+    length_stats[name]={'documents':len(lengths),'tokens':sum(lengths),'max':max(lengths,default=0),'quantiles':dict(zip(['p50','p90','p95','p99'],np.quantile(lengths,[.5,.9,.95,.99]).tolist())),'over_510_tokens':sum(x>510 for x in lengths),'characters':sum(len(r['text']) for r in splits[name])}
+    print(name,length_stats[name],flush=True)
+manifest['tokenizer']={'model':'distilbert/distilbert-base-cased','revision':MODEL_REVISION,'model_max_length':tokenizer.model_max_length,'lengths':length_stats,'entity_tokens_by_label':{k:{'count':len(v),'max':max(v),'p99':float(np.quantile(v,.99))} for k,v in entity_lengths.items()},'unrepresentable_boundaries_by_label':dict(boundary_bad),'unrepresentable_boundary_examples':boundary_examples}
+(ART/'data-manifest.json').write_text(json.dumps(manifest,indent=2))
+print('DONE',json.dumps(manifest['tokenizer']),flush=True)

reproduction/scripts/probe_model.py

@@ -0,0 +1,83 @@
+"""Prove MPS update, checkpoint reload, dynamic ONNX export before full training."""
+import json
+import sys
+import time
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import numpy as np
+import onnxruntime as ort
+import torch
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+from pii.core import BIO, LABELS, MODEL_REVISION
+
+torch.manual_seed(42)  # Transformers TrainingArguments default seed.
+out = Path("artifacts/probe")
+out.mkdir(parents=True, exist_ok=True)
+tokenizer = AutoTokenizer.from_pretrained("artifacts/backbone", local_files_only=True)
+model = AutoModelForTokenClassification.from_pretrained("artifacts/backbone", num_labels=len(BIO), id2label=dict(enumerate(BIO)), label2id={v:k for k,v in enumerate(BIO)}, attn_implementation="eager", local_files_only=True).to("mps")
+optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5, weight_decay=0.01)
+report = {"model_revision": MODEL_REVISION, "parameters": sum(p.numel() for p in model.parameters()), "max_length": model.config.max_position_embeddings, "precision":"float32", "device":"mps", "benchmarks":[]}
+before = model.classifier.weight.detach().clone()
+# Official HF recipe batch=16; measure both short and maximum-length inputs.
+for length in (32, model.config.max_position_embeddings):
+    x = torch.full((16, length), 100, dtype=torch.long, device="mps")
+    x[:,0], x[:,-1] = tokenizer.cls_token_id, tokenizer.sep_token_id
+    y = torch.zeros_like(x)
+    y[:,0], y[:,-1] = -100, -100
+    y[:,1] = 1
+    times = []
+    for stage in ("warmup", "measured"):
+        optimizer.zero_grad(set_to_none=True)
+        torch.mps.synchronize()
+        start = time.perf_counter()
+        loss = model(input_ids=x, attention_mask=torch.ones_like(x), labels=y).loss
+        loss.backward()
+        finite = all(torch.isfinite(p.grad).all().item() for p in model.parameters() if p.grad is not None)
+        optimizer.step()
+        torch.mps.synchronize()
+        times.append(time.perf_counter()-start)
+        assert torch.isfinite(loss).item() and finite
+    report["benchmarks"].append({"batch_size":16,"sequence_length":length,"step_seconds":times,"loss":loss.item(),"allocated_bytes":torch.mps.current_allocated_memory(),"driver_bytes":torch.mps.driver_allocated_memory()})
+assert not torch.equal(before, model.classifier.weight)
+model.eval()
+sample = tokenizer("🙂 José Smith: jose@example.com", return_tensors="pt")
+sample = {k:v.to("mps") for k,v in sample.items() if k in ("input_ids","attention_mask")}
+with torch.no_grad():
+    reference = model(**sample).logits.cpu()
+start = time.perf_counter()
+model.save_pretrained(out / "checkpoint")
+torch.save({"optimizer":optimizer.state_dict(),"rng":torch.get_rng_state(),"mps_rng":torch.mps.get_rng_state()}, out / "resume.pt")
+report["checkpoint_seconds"] = time.perf_counter()-start
+del optimizer
+model = model.to("cpu")
+reloaded = AutoModelForTokenClassification.from_pretrained(out / "checkpoint", local_files_only=True, attn_implementation="eager").to("mps").eval()
+with torch.no_grad():
+    actual = reloaded(**sample).logits.cpu()
+report["reload_max_abs_difference"] = (actual-reference).abs().max().item()
+assert torch.equal(actual,reference)
+del reloaded
+torch.mps.empty_cache()
+tokenizer.save_pretrained(out)
+(out / "config.json").write_text(json.dumps({"labels":LABELS,"max_length":model.config.max_position_embeddings,"stride":0,"model_revision":MODEL_REVISION},indent=2))
+
+class Export(torch.nn.Module):
+    def __init__(self, model):
+        super().__init__()
+        self.model=model
+    def forward(self, input_ids, attention_mask):
+        return self.model(input_ids=input_ids,attention_mask=attention_mask).logits
+
+sample = {k:v.cpu() for k,v in sample.items()}
+torch.onnx.export(Export(model).eval(), (sample["input_ids"],sample["attention_mask"]), str(out / "model.onnx"), input_names=["input_ids","attention_mask"], output_names=["logits"], dynamic_axes={"input_ids":{0:"batch",1:"sequence"},"attention_mask":{0:"batch",1:"sequence"},"logits":{0:"batch",1:"sequence"}}, opset_version=17, dynamo=False)
+session=ort.InferenceSession(str(out / "model.onnx"),providers=["CPUExecutionProvider"])
+report["export_checks"]=[]
+for text in ("", "Alice Smith", "🙂 José Smith: jose@example.com", "Long ticket. " * 150):
+    encoded=tokenizer(text,return_tensors="pt",truncation=True,max_length=model.config.max_position_embeddings)
+    encoded={k:v for k,v in encoded.items() if k in ("input_ids","attention_mask")}
+    with torch.no_grad():
+        logits=model(**encoded).logits.numpy()
+    actual=session.run(None,{k:v.numpy() for k,v in encoded.items()})[0]
+    report["export_checks"].append({"tokens":encoded["input_ids"].shape[1],"max_abs_logit_difference":float(np.max(np.abs(logits-actual))),"argmax_identical":bool(np.array_equal(logits.argmax(-1),actual.argmax(-1)))})
+    assert np.array_equal(logits.argmax(-1),actual.argmax(-1))
+(out / "evidence.json").write_text(json.dumps(report,indent=2)+"\n")
+print(json.dumps(report,indent=2),flush=True)

reproduction/scripts/regex_baseline.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Pinned official DataFog regex baseline on shared email/phone scope."""
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import sys
+import time
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from pii.evaluation import evaluate, read_jsonl, write_json
+
+VERSION = '4.8.1'
+MAPPING = {'EMAIL': 'email', 'PHONE': 'phone_number'}
+
+
+def scoped_gold(records):
+    return [{**r, 'spans':[s for s in r['spans'] if s['label'] in MAPPING.values()]} for r in records]
+
+
+def predict(records):
+    import datafog
+    if importlib.metadata.version('datafog') != VERSION:
+        raise RuntimeError(f'Baseline requires datafog=={VERSION}')
+    for r in records:
+        result = datafog.scan(r['text'], engine='regex', entity_types=list(MAPPING))
+        findings = []
+        byte_offsets = [0]
+        for ch in r['text']:
+            byte_offsets.append(byte_offsets[-1] + len(ch.encode('utf-8')))
+        for e in result.entities:
+            if r['text'][e.start:e.end] != e.text:
+                raise ValueError('Upstream span/source mismatch')
+            findings.append(dict(label=MAPPING[e.type], start=byte_offsets[e.start], end=byte_offsets[e.end], raw_score=1.0, confidence=None))
+        yield dict(id=r['id'], text=r['text'], findings=findings, contains_pii=bool(findings), score_semantics='Constant match indicator from DataFog regex, not calibrated confidence')
+
+
+def write_jsonl(path, records):
+    with open(path, 'w') as f:
+        for record in records:
+            f.write(json.dumps(record, ensure_ascii=False)+'\n')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--gold', required=True)
+    parser.add_argument('--output-dir', required=True)
+    parser.add_argument('--model-predictions')
+    parser.add_argument('--model-calibration')
+    args = parser.parse_args()
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    gold = scoped_gold(read_jsonl(args.gold))
+    started = time.perf_counter()
+    predictions = list(predict(gold))
+    elapsed = time.perf_counter()-started
+    write_jsonl(output/'gold-email-phone.jsonl', gold)
+    write_jsonl(output/'regex-predictions.jsonl', predictions)
+    metrics = evaluate(gold, predictions)
+    # A regex match flag is not a probability; do not imply a calibration comparison.
+    for key in ('raw_reliability','confidence_reliability','reliability_population','confidence_semantics'):
+        metrics.pop(key, None)
+    metrics['score_semantics'] = 'Regex match indicator, no probability calibration evaluated'
+    metrics['comparison_scope'] = ['email','phone_number']
+    metrics['unsupported_labels'] = ['first_name','last_name','street_address','customer_id','account_number']
+    write_json(output/'regex-metrics.json', metrics)
+    provenance = dict(package='datafog', version=VERSION, official_repository='https://github.com/DataFog/datafog-python',
+                      release='https://pypi.org/project/datafog/4.8.1/', input_sha256=hashlib.sha256(Path(args.gold).read_bytes()).hexdigest(),
+                      documents=len(gold), elapsed_seconds=elapsed, API="datafog.scan(text, engine='regex', entity_types=['EMAIL','PHONE'])",
+                      offset_conversion='Python character offsets converted to original UTF-8 byte offsets', dependencies={d.metadata['Name']:d.version for d in importlib.metadata.distributions()})
+    write_json(output/'regex-provenance.json', provenance)
+    if args.model_predictions:
+        model = read_jsonl(args.model_predictions)
+        model = [{**r, 'findings':[s for s in r['findings'] if s['label'] in MAPPING.values()]} for r in model]
+        calibration = json.loads(Path(args.model_calibration).read_text()) if args.model_calibration else None
+        model_metrics = evaluate(gold, model, calibration=calibration)
+        model_metrics['comparison_scope'] = ['email','phone_number']
+        write_json(output/'model-email-phone-metrics.json', model_metrics)
+    print(json.dumps({'documents':len(gold), 'seconds':elapsed, 'output_dir':str(output)}))

reproduction/scripts/report.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Render final measured evidence as Markdown; absent evidence stays a blocker."""
+import argparse
+import json
+from pathlib import Path
+
+
+def load(root, name):
+    path = root / name
+    return json.loads(path.read_text()) if path.exists() else None
+
+
+def number(value, decimals=4):
+    if value is None:
+        return 'Unavailable'
+    return f'{value:,.{decimals}f}' if isinstance(value, (int, float)) else str(value)
+
+
+def table(headers, rows):
+    safe = lambda v: str(v).replace('|', '\\|').replace('\n', ' ')
+    return '\n'.join(['| '+' | '.join(headers)+' |', '| '+' | '.join(['---']*len(headers))+' |'] +
+                     ['| '+' | '.join(safe(v) for v in row)+' |' for row in rows])
+
+
+def render(root):
+    paths = {
+        'test':'artifacts/evaluation/test.json', 'fixtures':'artifacts/evaluation/support-fixtures.json',
+        'regex':'artifacts/regex-test/regex-metrics.json', 'model_shared':'artifacts/regex-test/model-email-phone-metrics.json',
+        'calibration':'artifacts/bundle/calibration.json', 'training':'artifacts/training/complete.json',
+        'data':'artifacts/data-manifest.json', 'parity':'artifacts/bundle/rust-parity.json',
+        'benchmark':'artifacts/native-benchmark.json', 'training_check':'artifacts/training-check.json',
+        'boundary':'artifacts/boundary-proof.json'}
+    a = {k:load(root,p) for k,p in paths.items()}
+    history_path = root/'artifacts/training/history.jsonl'
+    history = [json.loads(line) for line in history_path.read_text().splitlines() if line.strip()] if history_path.exists() else []
+    if not history and a['training']:
+        history = a['training'].get('history', [])
+    evidence = lambda key: f'[{paths[key]}]({paths[key]})'
+    lines = ['# DataFog PII prototype evaluation', '',
+             'This is an evaluated prototype, not a production-readiness claim. All numbers below come from saved artifacts; absent evidence is explicitly marked unavailable.', '']
+    required = ('test','fixtures','regex','model_shared','calibration','training','data','parity','benchmark')
+    missing = [paths[k] for k in required if a[k] is None]
+    lines += [('Required evaluation artifacts are present.' if not missing else '**Incomplete evidence:** '+', '.join(missing)), '']
+    criteria = []
+    def criterion(title, passed, keys, detail):
+        criteria.append([title, 'Evidence present' if passed else 'Unresolved / missing evidence', ', '.join(evidence(k) for k in keys), detail])
+    criterion('Pinned data and disjoint document families', bool(a['data'] and a['data'].get('pairwise_group_disjoint')), ['data'], 'Structural validation and split rules are saved; annotation semantics remain imperfect.')
+    criterion('Trained checkpoint and resumability', bool(a['training'] and a['training_check']), ['training','training_check'], 'Completion plus separate optimizer/RNG resume diagnostic; inspect reported MPS parameter differences.')
+    p=a['parity'] or {}
+    criterion('Native Rust and Python parity', all(p.get(k) for k in ('token_ids_masks_global_byte_offsets_parity','findings_redaction_parity','malformed_request_explicit_error_and_recovery')), ['parity'], 'Checks tokenizer/global byte offsets, outputs/redaction, and explicit error recovery; parity does not prove quality.')
+    criterion('Complete-input boundary behavior', bool(a['boundary']), ['boundary'], 'Manual global-offset window construction; finite fixture evidence and representability limits apply.')
+    criterion('Independent calibration and experimental threshold', bool(a['calibration'] and a['test']), ['calibration','test'], 'Fit/select partitions are separate; final reliability is reported on held-out predicted spans.')
+    criterion('Held-out quality and support mechanics', bool(a['test'] and a['fixtures']), ['test','fixtures'], 'Synthetic population only; no reviewed private tickets.')
+    criterion('Existing regex comparison', bool(a['regex'] and a['model_shared']), ['regex','model_shared'], 'Official published DataFog 4.8.1; email and phone only.')
+    criterion('Native resource measurements', bool(a['benchmark']), ['benchmark'], 'Local observed load, warm request latency, memory, and bundle size; no performance target assumed.')
+    lines += [table(['Contract criterion','Status','Evidence','Interpretation / limitation'],criteria),'']
+    data=a['data'] or {}
+    lines += ['## Data and execution', '', f"Dataset: `{data.get('source','Unavailable')}` revision `{data.get('revision','Unavailable')}`. License: {data.get('license','Unavailable')}. {data.get('attribution','')}", '']
+    if data.get('splits'):
+        lines += [table(['Partition','Documents','Target-clean documents'],[(k,v.get('documents'),v.get('negative_documents')) for k,v in data['splits'].items()]),'']
+    if data.get('exclusions'):
+        lines += ['Structural exclusions: '+', '.join(f'{k}: {v:,}' for k,v in data['exclusions'].items())+'.','']
+    if history:
+        lines += [table(['Epoch','Train mean batch loss','Validation token loss','Selected at epoch','Elapsed seconds'],
+                        [(r.get('epoch'),number(r.get('train_mean_batch_loss')),number(r.get('validation_token_loss')),r.get('selected'),number(r.get('elapsed_seconds'),1)) for r in history]),'']
+    else:
+        lines += ['Training history unavailable.','']
+    if a['training']:
+        lines += [f"Training elapsed: {number(a['training'].get('elapsed_seconds'),1)} seconds. Best validation token loss: {number(a['training'].get('best_validation_loss'))}.",'']
+    lines += ['Corrective iterations: two of the three-round fuse used. Round 1 fixed tokenizer overflow by manually constructing complete-input windows; round 2 kept the ONNX export wrapper and Python reference in evaluation mode. Neither correction tuned against final test outcomes. Further corrective work must remain within the agreed fuse.','']
+    test=a['test']
+    lines += ['## Held-out detection', '']
+    if test:
+        metrics=[('overall',test.get('overall',{})), *test.get('per_label',{}).items()]
+        lines += [table(['Label','Gold entities','Predicted','TP','FP','FN','Precision','Recall','F1'],
+                        [(label,m.get('support'),m.get('predicted'),m.get('tp'),m.get('fp'),m.get('fn'),number(m.get('precision')),number(m.get('recall')),number(m.get('f1'))) for label,m in metrics]),'']
+        cov=test.get('redaction_byte_coverage',{})
+        lines += [f"UTF-8 byte-union redaction precision {number(cov.get('precision'))}, recall {number(cov.get('recall'))}, F1 {number(cov.get('f1'))}. Missed gold bytes: {number(cov.get('fn'),0)}; unnecessary redaction bytes: {number(cov.get('fp'),0)}. Coverage ignores labels and weights multibyte characters by byte length.",'']
+        docs=test.get('documents',{})
+        lines += [table(['Document measure','Count'],[(k,number(v,0)) for k,v in docs.items()]),'',
+                  'Document false negatives mean no accepted span on a document containing target PII; partial misses are counted separately in `with_any_missed_entity`. False positives mean at least one accepted span on a target-clean document.','']
+    else:
+        lines += ['Held-out detection results unavailable.','']
+    lines += ['## Confidence and operating point','']
+    c=a['calibration']
+    if c:
+        lines += [f"Method: {c.get('method','Unavailable')}, pooled across labels. Experimental acceptance threshold: {number(c.get('threshold'),8)}. Objective: {c.get('threshold_objective','Unavailable')}.",'',
+                  f"Calibration fit support: {number(c.get('fit_documents'),0)} documents, {number(c.get('fit_predicted_spans'),0)} predicted spans, {number(c.get('fit_correct_spans'),0)} exact matches. Selection support: {number(c.get('select_documents'),0)} documents.",'']
+        lines += [f"Calibration warning: {warning}" for warning in c.get('warnings',[])]
+    else:
+        lines += ['Calibration artifact unavailable.','']
+    if test:
+        raw=test.get('raw_reliability',{}); calibrated=test.get('confidence_reliability',{})
+        lines += [table(['Score','Predicted-span support','Brier','ECE'],[(name,r.get('support'),number(r.get('brier'),6),number(r.get('ece'),6)) for name,r in [('Raw token-derived span score',raw),('Calibrated exact-span score',calibrated)]]),'']
+        rows=[]
+        for i in range(10):
+            rb=raw.get('bins',[])[i] if len(raw.get('bins',[]))>i else {}
+            cb=calibrated.get('bins',[])[i] if len(calibrated.get('bins',[]))>i else {}
+            rows.append([f'[{i/10:.1f}, {(i+1)/10:.1f}'+(']' if i==9 else ')'),rb.get('support','Unavailable'),number(rb.get('mean_score')),number(rb.get('accuracy')),cb.get('support','Unavailable'),number(cb.get('mean_score')),number(cb.get('accuracy'))])
+        lines += [table(['Confidence bin','Raw n','Raw mean','Raw accuracy','Calibrated n','Calibrated mean','Calibrated accuracy'],rows),'']
+    lines += ['Reliability uses all predicted spans before thresholding and exact label/boundary correctness. Missed entities are absent from this table: recall must be read separately. Ten equal-width bins make ECE bin-dependent. Calibration is conditional on this synthetic evaluation population and annotation quality; the threshold is not a production default.','']
+    lines += ['## Existing regex comparison','']
+    if a['regex'] and a['model_shared']:
+        rows=[]
+        for name,key in [('DataFog 4.8.1 regex','regex'),('Trained model, experimental threshold','model_shared')]:
+            for label in ['overall','email','phone_number']:
+                m=a[key].get('overall',{}) if label=='overall' else a[key].get('per_label',{}).get(label,{})
+                rows.append([name,label,m.get('support'),number(m.get('precision')),number(m.get('recall')),number(m.get('f1'))])
+        lines += [table(['System','Scope','Gold entities','Precision','Recall','F1'],rows),'']
+    else:
+        lines += ['Complete same-population, same-label comparison unavailable.','']
+    lines += ['Only email and phone are supported by this existing regex baseline within the seven-label contract. First name, last name, street address, customer ID, and account number are excluded from both sides of this head-to-head comparison. All documents remain in the comparison. Constant regex match indicators are not calibrated probabilities. See [baseline provenance](docs/regex-baseline.md).','']
+    lines += ['## Synthetic support fixtures','']
+    if a['fixtures']:
+        f=a['fixtures']; m=f.get('overall',{})
+        lines += [f"{number(f.get('documents',{}).get('total'),0)} fixture documents; exact-span precision {number(m.get('precision'))}, recall {number(m.get('recall'))}, F1 {number(m.get('f1'))}. Gold entity support: {number(m.get('support'),0)}.",'']
+    else:
+        lines += ['Fixture predictions unavailable.','']
+    lines += ['These hand-authored cases include Unicode signatures, pasted fields, multiline addresses, repeated entities, clean numeric text, and long inputs. They exercise mechanics and expose case-specific failures; they do not establish real customer-support quality.','']
+    lines += ['## Native resource measurements','']
+    b=a['benchmark']
+    if b:
+        lines += [table(['Measurement','Observed value'],[(k,number(b.get(k),3 if k=='cold_load_seconds' else 0)) for k in ('bundle_bytes','cold_load_seconds','peak_rss_bytes')]),'']
+        warm=b.get('warm',b.get('requests',b.get('warm_requests',[])))
+        if isinstance(warm,dict): warm=warm.get('requests',[])
+        if isinstance(warm,list) and warm:
+            lines += [table(['Input UTF-8 bytes','Warm end-to-end seconds'],[(r.get('bytes','Unavailable'),number(r.get('latency_seconds'),6)) for r in warm]),'']
+        if b.get('note'): lines += [str(b['note']),'']
+    else:
+        lines += ['Native resource measurements unavailable.','']
+    if a['parity']:
+        lines += [f"Maximum absolute Python/Rust logit difference on parity cases: {number(p.get('max_abs_logit_difference'),8)}. Token IDs, masks and global byte offsets agree: {p.get('token_ids_masks_global_byte_offsets_parity','Unavailable')}; findings/redaction agree: {p.get('findings_redaction_parity','Unavailable')}.",'']
+    unrep=data.get('tokenizer',{}).get('unrepresentable_boundaries_by_label',{})
+    lines += ['## Remaining limitations','',
+              '- Nemotron is broad synthetic data with semantic annotation errors. Structural filtering cannot prove that labels are semantically correct or complete.',
+              '- Split allocation uses documented label-support heuristics and grouped families. It does not guarantee statistical precision or eliminate every possible near-duplicate.',
+              '- No private customer tickets were supplied or uploaded; real-world English customer-support generalization remains unproven.',
+              '- Token boundaries cannot represent every character boundary. Observed training-source unrepresentable boundaries: '+(', '.join(f'{k}={v}' for k,v in unrep.items()) if unrep else 'unavailable')+'.',
+              '- Native inference requires the packaged runtime and pinned tokenizer/configuration. A successful fixture parity check is not a universal numerical guarantee.',
+              '- `contains_pii` means at least one accepted finding among the seven selected labels; it is not an independently calibrated probability or assurance that a document contains no other sensitive information.',
+              '- No existing DataFog checkout was modified, no Git repository was created, and no model was published as part of this standalone prototype.', '',
+              'Reproduction commands and artifact locations are in [README.md](README.md).']
+    return '\n'.join(lines)+'\n'
+
+
+if __name__ == '__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--root',type=Path,default=Path(__file__).resolve().parents[1])
+    parser.add_argument('--output',type=Path)
+    args=parser.parse_args()
+    output=args.output or args.root/'REPORT.md'
+    output.parent.mkdir(parents=True,exist_ok=True)
+    output.write_text(render(args.root))
+    print(output)

reproduction/scripts/run_pipeline.py

@@ -0,0 +1,65 @@
+"""Persistent sequential prototype run; records failures without hiding them."""
+import datetime
+import hashlib
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT=Path(__file__).resolve().parents[1]
+PY=str(ROOT/".venv/bin/python")
+STATE=ROOT/"artifacts/pipeline"
+
+def now():return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+def digest(path):
+    h=hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda:f.read(1024*1024),b""):h.update(block)
+    return h.hexdigest()
+
+def stage(name,command):
+    marker=STATE/f"{name}.done.json"
+    if marker.exists():return
+    event={"stage":name,"status":"running","started_at":now(),"command":command}
+    (STATE/"status.json").write_text(json.dumps(event,indent=2)+"\n")
+    print(json.dumps(event),flush=True)
+    with (STATE/f"{name}.log").open("a") as log:
+        completed=subprocess.run(command,cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
+    event.update(finished_at=now(),returncode=completed.returncode,status="complete" if completed.returncode==0 else "failed")
+    (STATE/"status.json").write_text(json.dumps(event,indent=2)+"\n")
+    if completed.returncode:raise RuntimeError(f"{name} failed; inspect {STATE/name}.log")
+    marker.write_text(json.dumps(event,indent=2)+"\n")
+
+def main():
+    STATE.mkdir(parents=True,exist_ok=True)
+    training_command=[PY,"scripts/train.py","--stride","42"]
+    checkpoints=sorted((ROOT/"artifacts/training").glob("epoch-*/state.pt"),key=lambda p:int(p.parent.name.split("-")[-1]))
+    if checkpoints:training_command.extend(["--resume",str(checkpoints[-1].parent)])
+    stage("train",training_command)
+    stage("export",[PY,"scripts/export_model.py"])
+    for split in ("calibration_fit","calibration_select"):
+        stage("predict-"+split,[PY,"scripts/predict.py","--input",f"data/prepared/{split}.jsonl","--output",f"artifacts/predictions/{split}.jsonl"])
+    stage("calibrate",[PY,"scripts/calibrate.py","--fit-gold","data/prepared/calibration_fit.jsonl","--fit-predictions","artifacts/predictions/calibration_fit.jsonl","--select-gold","data/prepared/calibration_select.jsonl","--select-predictions","artifacts/predictions/calibration_select.jsonl","--output","artifacts/bundle/calibration.json"])
+    bundle=ROOT/"artifacts/bundle"
+    config=json.loads((bundle/"config.json").read_text())
+    config.update(calibration_file="calibration.json",status="experimental prototype; demonstration threshold only")
+    (bundle/"config.json").write_text(json.dumps(config,indent=2)+"\n")
+    stage("final-parity",[PY,"scripts/check_rust_parity.py","artifacts/bundle","--output","artifacts/bundle/rust-parity.json"])
+    freeze=STATE/"evaluation-freeze.json"
+    files=[p for p in bundle.rglob("*") if p.is_file()]+[ROOT/"pii/core.py",ROOT/"pii/evaluation.py",ROOT/"scripts/predict.py",ROOT/"rust/target/release/datafog-pii"]
+    hashes={str(p.relative_to(ROOT)):digest(p) for p in files}
+    if freeze.exists():
+        original=json.loads(freeze.read_text())
+        if any(hashes.get(path)!=value for path,value in original["sha256"].items()):raise RuntimeError("Frozen evaluation inputs changed; owner-visible diagnostic review required")
+    else:freeze.write_text(json.dumps({"frozen_at":now(),"sha256":hashes,"policy":"No tuning against test results"},indent=2)+"\n")
+    for split,path in (("support-fixtures","data/support-fixtures.jsonl"),("test","data/prepared/test.jsonl")):
+        prediction=f"artifacts/predictions/{split}.jsonl"
+        stage("predict-"+split,[PY,"scripts/predict.py","--input",path,"--output",prediction])
+        stage("evaluate-"+split,[PY,"scripts/evaluate.py","--gold",path,"--predictions",prediction,"--calibration","artifacts/bundle/calibration.json","--output",f"artifacts/evaluation/{split}.json"])
+    stage("regex-test",[str(ROOT/"work/regex-env/bin/python"),"scripts/regex_baseline.py","--gold","data/prepared/test.jsonl","--output-dir","artifacts/regex-test","--model-predictions","artifacts/predictions/test.jsonl","--model-calibration","artifacts/bundle/calibration.json"])
+    stage("benchmark",[PY,"scripts/benchmark_native.py"])
+    stage("report",[PY,"scripts/report.py"])
+    (STATE/"status.json").write_text(json.dumps({"status":"complete","finished_at":now(),"next":"Review evidence and final report against contract"},indent=2)+"\n")
+
+if __name__=="__main__":main()

reproduction/scripts/train.py

@@ -0,0 +1,127 @@
+"""Full supervised training using the pinned HF token-classification recipe."""
+import argparse
+import json
+import math
+import random
+import sys
+import time
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import numpy as np
+import torch
+from transformers import AutoModelForTokenClassification, AutoTokenizer, DataCollatorForTokenClassification
+from pii.core import BIO, MODEL_REVISION, load_jsonl, encode_windows
+
+def prepare(records, tokenizer, max_length, stride):
+    chunks=[]
+    stats={"documents":len(records),"chunks":0,"tokens":0,"ignored_boundary_tokens":0,"partial_window_entity_tokens":0}
+    for record in records:
+        encoded=encode_windows(tokenizer,record["text"],max_length,stride)
+        for ids, mask, offsets in zip(encoded["input_ids"],encoded["attention_mask"],encoded["offset_mapping"]):
+            content=[(a,b) for a,b in offsets if a!=b]
+            left,right=content[0][0] if content else 0,content[-1][1] if content else 0
+            labels=[]
+            for a,b in offsets:
+                label=0
+                if a==b:
+                    labels.append(-100)
+                    continue
+                for span in record["spans"]:
+                    s,e=span["start"],span["end"]
+                    if a<e and s<b:
+                        if not (s<=a and b<=e):
+                            label=-100
+                            stats["ignored_boundary_tokens"]+=1
+                        elif s<left or e>right:
+                            label=-100
+                            stats["partial_window_entity_tokens"]+=1
+                        else:
+                            # First representable token of the entity gets B.
+                            earlier=any(s<=aa<a and bb<=e and aa!=bb for aa,bb in offsets)
+                            label=BIO.index(("I-" if earlier else "B-")+span["label"])
+                        break
+                labels.append(label)
+            chunks.append({"input_ids":ids,"attention_mask":mask,"labels":labels})
+            stats["tokens"]+=len(ids)
+    stats["chunks"]=len(chunks)
+    return chunks,stats
+
+def main():
+    p=argparse.ArgumentParser()
+    p.add_argument("--stride",type=int,required=True)
+    p.add_argument("--device",default="mps")
+    p.add_argument("--resume",type=Path)
+    p.add_argument("--prepare-only",action="store_true")
+    args=p.parse_args()
+    out=Path("artifacts/training");out.mkdir(parents=True,exist_ok=True)
+    seed=42
+    random.seed(seed);np.random.seed(seed);torch.manual_seed(seed)
+    tokenizer=AutoTokenizer.from_pretrained("artifacts/backbone",local_files_only=True)
+    config={"seed":seed,"batch_size":16,"epochs":2,"learning_rate":2e-5,"weight_decay":0.01,"max_length":512,"stride":args.stride,"device":args.device,"precision":"float32","model_revision":MODEL_REVISION,"recipe":"https://huggingface.co/docs/transformers/tasks/token_classification","stopping_rule":"Two epochs from official recipe; select lowest validation token loss. This is a prototype recipe hypothesis, not an optimum.","checkpoint_cadence":"Each epoch per official recipe; measured probe checkpoint cost 0.733s.","selection_metric":"validation token cross entropy"}
+    cache=out/f"chunks-manual-stride-{args.stride}.pt"
+    if cache.exists():
+        prepared=torch.load(cache,weights_only=False)
+    else:
+        prepared={}
+        for name in ("train","validation"):
+            prepared[name],prepared[name+"_stats"]=prepare(load_jsonl(f"data/prepared/{name}.jsonl"),tokenizer,512,args.stride)
+            print(json.dumps({name:prepared[name+"_stats"]}),flush=True)
+        torch.save(prepared,cache)
+    config["data_stats"]={name:prepared[name+"_stats"] for name in ("train","validation")}
+    config["estimated_training_seconds_max_length_probe"]=math.ceil(len(prepared["train"])/16)*2*0.558059167
+    (out/"run-config.json").write_text(json.dumps(config,indent=2)+"\n")
+    if args.prepare_only:
+        print(json.dumps(config,indent=2),flush=True);return
+    if config["estimated_training_seconds_max_length_probe"]>24*3600:
+        raise RuntimeError("Estimated run exceeds user-approved 24h limit; owner decision required")
+    model=AutoModelForTokenClassification.from_pretrained(args.resume / "model" if args.resume else "artifacts/backbone",num_labels=len(BIO),id2label=dict(enumerate(BIO)),label2id={v:k for k,v in enumerate(BIO)},local_files_only=True,attn_implementation="eager").to(args.device)
+    optimizer=torch.optim.AdamW(model.parameters(),lr=config["learning_rate"],weight_decay=config["weight_decay"])
+    steps_per_epoch=math.ceil(len(prepared["train"])/16)
+    scheduler=torch.optim.lr_scheduler.LambdaLR(optimizer,lambda step:max(0.0,1-step/(steps_per_epoch*config["epochs"])))
+    collate=DataCollatorForTokenClassification(tokenizer,padding=True,return_tensors="pt")
+    start_epoch,best_loss,global_step=0,float("inf"),0
+    if args.resume:
+        state=torch.load(args.resume/"state.pt",map_location="cpu",weights_only=False)
+        optimizer.load_state_dict(state["optimizer"]);scheduler.load_state_dict(state["scheduler"])
+        random.setstate(state["random"]);np.random.set_state(state["numpy"]);torch.set_rng_state(state["torch"])
+        if args.device=="mps":torch.mps.set_rng_state(state["mps"])
+        start_epoch,best_loss,global_step=state["epoch"],state["best_loss"],state["step"]
+    history=[]
+    started=time.perf_counter()
+    for epoch in range(start_epoch,config["epochs"]):
+        model.train()
+        order=list(range(len(prepared["train"])));random.shuffle(order)
+        running=0.0
+        for step,begin in enumerate(range(0,len(order),16)):
+            batch=collate([prepared["train"][i] for i in order[begin:begin+16]])
+            batch={k:v.to(args.device) for k,v in batch.items()}
+            optimizer.zero_grad(set_to_none=True)
+            loss=model(**batch).loss
+            if not torch.isfinite(loss):raise RuntimeError("nonfinite loss")
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(),1.0,error_if_nonfinite=True) # Trainer default.
+            optimizer.step();scheduler.step()
+            running+=loss.item();global_step+=1
+            # Persist current progress every batch; no logging interval assumption.
+            (out/"progress.json").write_text(json.dumps({"epoch":epoch+1,"step_in_epoch":step+1,"steps_per_epoch":steps_per_epoch,"global_step":global_step,"loss":loss.item(),"elapsed_seconds":time.perf_counter()-started})+"\n")
+        model.eval();total_loss,total_tokens=0.0,0
+        with torch.no_grad():
+            for begin in range(0,len(prepared["validation"]),16):
+                batch=collate(prepared["validation"][begin:begin+16]);batch={k:v.to(args.device) for k,v in batch.items()}
+                count=(batch["labels"]!=-100).sum().item()
+                total_loss+=model(**batch).loss.item()*count;total_tokens+=count
+        val_loss=total_loss/total_tokens
+        selected=val_loss<best_loss
+        if selected:
+            best_loss=val_loss
+            model.save_pretrained(out/"best");tokenizer.save_pretrained(out/"best")
+        checkpoint=out/f"epoch-{epoch+1}"
+        model.save_pretrained(checkpoint/"model")
+        torch.save({"optimizer":optimizer.state_dict(),"scheduler":scheduler.state_dict(),"random":random.getstate(),"numpy":np.random.get_state(),"torch":torch.get_rng_state(),"mps":torch.mps.get_rng_state() if args.device=="mps" else None,"epoch":epoch+1,"best_loss":best_loss,"step":global_step},checkpoint/"state.pt")
+        event={"epoch":epoch+1,"train_mean_batch_loss":running/steps_per_epoch,"validation_token_loss":val_loss,"selected":selected,"elapsed_seconds":time.perf_counter()-started}
+        history.append(event)
+        with (out/"history.jsonl").open("a") as handle:handle.write(json.dumps(event)+"\n")
+        print(json.dumps(event),flush=True)
+    (out/"complete.json").write_text(json.dumps({"history":history,"best_validation_loss":best_loss,"elapsed_seconds":time.perf_counter()-started},indent=2)+"\n")
+
+if __name__=="__main__":main()

reproduction/tests/test_data.py

@@ -0,0 +1,20 @@
+import unittest
+from pii.data import validate, family_key
+
+class DataValidation(unittest.TestCase):
+    def test_python_literal_and_unicode(self):
+        text='😊 José'
+        spans,error=validate({'text':text,'spans':"[{'start':2,'end':6,'text':'José','label':'first_name'}]"})
+        self.assertIsNone(error); self.assertEqual(text[spans[0]['start']:spans[0]['end']],'José')
+    def test_invalid_offset_excluded(self):
+        self.assertEqual(validate({'text':'José','spans':"[{'start':0,'end':3,'text':'José','label':'first_name'}]"})[1],'source_slice_mismatch')
+    def test_overlap_excluded(self):
+        spans=[{'start':0,'end':4,'text':'John','label':'first_name'},{'start':0,'end':4,'text':'John','label':'last_name'}]
+        self.assertEqual(validate({'text':'John','spans':repr(spans)})[1],'duplicate_or_overlap')
+    def test_no_eval(self):
+        self.assertEqual(validate({'text':'x','spans':"__import__('os').system('false')"})[1],'parse')
+    def test_family_hides_values(self):
+        a=[{'start':6,'end':10,'label':'first_name'}]; b=[{'start':6,'end':10,'label':'first_name'}]
+        self.assertEqual(family_key('Name: John',a),family_key('Name: Jane',b))
+        self.assertNotEqual(family_key('Name: John',a),family_key('User: John',a))
+if __name__=='__main__':unittest.main()

reproduction/tests/test_evaluation.py

@@ -0,0 +1,56 @@
+import unittest
+from pii.evaluation import evaluate, apply_calibration, fit_calibration, read_jsonl
+
+
+def pred(gold, score=.8):
+    return {'id': gold['id'], 'text': gold['text'], 'findings': [dict(label=s['label'], start=len(gold['text'][:s['start']].encode()), end=len(gold['text'][:s['end']].encode()), raw_score=score, confidence=None) for s in gold['spans']]}
+
+
+class EvaluationTests(unittest.TestCase):
+    def test_unicode_and_fixtures(self):
+        gold = read_jsonl('data/support-fixtures.jsonl')
+        result = evaluate(gold, [pred(g) for g in gold])
+        self.assertEqual(result['overall']['f1'], 1)
+        self.assertEqual(result['redaction_byte_coverage']['f1'], 1)
+        self.assertGreater(result['documents']['clean_support'], 0)
+        self.assertEqual(result['documents']['false_positive'], 0)
+        self.assertEqual(result['documents']['false_negative'], 0)
+
+    def test_partial_wrong_label_and_clean_doc(self):
+        gold = [{'id': 'a', 'text': 'é Alice', 'spans': [{'label':'first_name', 'start':2, 'end':7}]}, {'id':'b', 'text':'version 123', 'spans':[]}]
+        predictions = [{'id':'a', 'text':'é Alice', 'findings':[{'label':'last_name', 'start':3, 'end':6, 'raw_score':.7}]}, {'id':'b', 'text':'version 123', 'findings':[{'label':'customer_id', 'start':8, 'end':11, 'raw_score':.9}]}]
+        result = evaluate(gold, predictions)
+        self.assertEqual(result['overall']['tp'], 0)
+        self.assertEqual(result['redaction_byte_coverage']['tp'], 3)
+        self.assertEqual(result['redaction_byte_coverage']['fn'], 2)
+        self.assertEqual(result['redaction_byte_coverage']['fp'], 3)
+        self.assertEqual(result['documents']['false_positive'], 1)
+        self.assertEqual(result['documents']['with_any_missed_entity'], 1)
+        self.assertEqual(result['documents']['false_negative'], 0)
+        self.assertAlmostEqual(result['raw_reliability']['brier'], .65)
+        self.assertEqual(evaluate(gold, predictions, threshold=.8)['documents']['false_negative'], 1)
+
+    def test_fail_closed_on_missing_and_bad_boundaries(self):
+        gold = [{'id':'a', 'text':'é', 'spans':[]}]
+        with self.assertRaises(ValueError):
+            evaluate(gold, [])
+        with self.assertRaises(ValueError):
+            evaluate(gold, [{'id':'a', 'text':'é', 'findings':[{'label':'first_name','start':1,'end':2,'raw_score':.5}]}])
+
+    def test_independent_calibration_and_interpolation(self):
+        fg = [{'id':'fit', 'text':'Alice Bob', 'spans':[{'label':'first_name','start':0,'end':5}]}]
+        fp = [{'id':'fit','text':'Alice Bob','findings':[{'label':'first_name','start':0,'end':5,'raw_score':.9}, {'label':'last_name','start':6,'end':9,'raw_score':.1}]}]
+        sg = [{'id':'select','text':'Carol Dave', 'spans':[{'label':'first_name','start':0,'end':5}]}]
+        sp = [{'id':'select','text':'Carol Dave','findings':[{'label':'first_name','start':0,'end':5,'raw_score':.8}, {'label':'last_name','start':6,'end':10,'raw_score':.2}]}]
+        artifact = fit_calibration(fg, fp, sg, sp)
+        self.assertEqual(artifact['select_operating_point']['f1'], 1)
+        self.assertAlmostEqual(apply_calibration(.5, artifact), .5)
+        self.assertEqual(apply_calibration(0, artifact), 0)
+        self.assertEqual(apply_calibration(1, artifact), 1)
+        with self.assertRaises(ValueError):
+            fit_calibration(fg, fp, fg, fp)
+        self.assertEqual(evaluate(sg, sp, calibration=artifact)['overall']['f1'], 1)
+
+
+if __name__ == '__main__':
+    unittest.main()

runtime/Cargo.lock

@@ -0,0 +1,877 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "castaway"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "cc"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "compact_str"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "daachorse"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d"
+
+[[package]]
+name = "darling"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "dary_heap"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "datafog-pii"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "ort",
+ "serde",
+ "serde_json",
+ "tokenizers",
+]
+
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "either"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+
+[[package]]
+name = "esaxx-rs"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "libloading"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "log"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
+
+[[package]]
+name = "macro_rules_attribute"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "pastey",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
+
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "monostate"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
+dependencies = [
+ "monostate-impl",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "monostate-impl"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "onig"
+version = "6.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "ort"
+version = "2.0.0-rc.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b"
+dependencies = [
+ "libloading",
+ "ndarray",
+ "ort-sys",
+ "smallvec",
+ "tracing",
+]
+
+[[package]]
+name = "ort-sys"
+version = "2.0.0-rc.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872"
+
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
+[[package]]
+name = "pastey"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
+dependencies = [
+ "portable-atomic",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "rand"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-cond"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
+dependencies = [
+ "either",
+ "itertools",
+ "rayon",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "ryu"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.6",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "smallvec"
+version = "1.16.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e"
+
+[[package]]
+name = "spm_precompiled"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
+dependencies = [
+ "base64",
+ "nom",
+ "serde",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.6",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc"
+dependencies = [
+ "ahash",
+ "compact_str",
+ "daachorse",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom",
+ "itertools",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
+dependencies = [
+ "pin-project-lite",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
+
+[[package]]
+name = "unicode-normalization-alignments"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
+dependencies = [
+ "smallvec",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

runtime/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "datafog-pii"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+tokenizers = { version = "=0.23.2", default-features = false, features = ["onig"] }
+ort = { version = "=2.0.0-rc.13", default-features = false, features = ["std", "load-dynamic", "api-27"] }

runtime/licenses/onnxruntime-LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

runtime/licenses/ort-LICENSE-APACHE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

runtime/licenses/ort-LICENSE-MIT

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2023-2026 pyke.io
+Copyright (c) 2020 Nicolas Bigaouette
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

runtime/licenses/tokenizers-LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

runtime/src/main.rs

@@ -0,0 +1,431 @@
+use anyhow::{Context, Result, bail, ensure};
+use ort::{session::Session, value::Tensor};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use std::{
+    io::{self, BufRead, Write},
+    path::Path,
+};
+use tokenizers::{Encoding, Tokenizer};
+
+#[derive(Deserialize)]
+struct Config {
+    labels: Vec<String>,
+    max_length: usize,
+    stride: usize,
+    #[serde(default)]
+    calibration: Option<Calibration>,
+    #[serde(default)]
+    calibration_file: Option<String>,
+}
+#[derive(Deserialize)]
+struct Calibration {
+    #[serde(alias = "xs")]
+    x: Vec<f64>,
+    #[serde(alias = "ys")]
+    y: Vec<f64>,
+    threshold: Option<f64>,
+}
+impl Calibration {
+    fn apply(&self, v: f64) -> f64 {
+        let i = self.x.partition_point(|x| *x < v);
+        if i == 0 {
+            self.y[0]
+        } else if i == self.x.len() {
+            *self.y.last().unwrap()
+        } else {
+            self.y[i - 1]
+                + (v - self.x[i - 1]) / (self.x[i] - self.x[i - 1]) * (self.y[i] - self.y[i - 1])
+        }
+    }
+}
+#[derive(Clone, Debug, Serialize)]
+struct Finding {
+    label: String,
+    start: usize,
+    end: usize,
+    raw_score: f64,
+    confidence: Option<f64>,
+    #[serde(skip)]
+    boundary_fragment: bool,
+}
+fn encode_windows(
+    tokenizer: &Tokenizer,
+    text: &str,
+    max_length: usize,
+    stride: usize,
+) -> Result<Vec<Encoding>> {
+    let full = tokenizer
+        .encode(text, false)
+        .map_err(|e| anyhow::anyhow!("{e}"))?;
+    let capacity = max_length - 2;
+    ensure!(stride < capacity, "invalid stride");
+    let cls = tokenizer
+        .token_to_id("[CLS]")
+        .context("missing CLS token")?;
+    let sep = tokenizer
+        .token_to_id("[SEP]")
+        .context("missing SEP token")?;
+    let mut windows = Vec::new();
+    let mut start = 0;
+    loop {
+        let end = (start + capacity).min(full.len());
+        let n = end - start + 2;
+        let mut ids = vec![cls];
+        ids.extend_from_slice(&full.get_ids()[start..end]);
+        ids.push(sep);
+        let mut offsets = vec![(0, 0)];
+        offsets.extend_from_slice(&full.get_offsets()[start..end]);
+        offsets.push((0, 0));
+        let mut tokens = vec!["[CLS]".to_string()];
+        tokens.extend_from_slice(&full.get_tokens()[start..end]);
+        tokens.push("[SEP]".to_string());
+        let mut special = vec![0; n];
+        special[0] = 1;
+        special[n - 1] = 1;
+        windows.push(Encoding::new(
+            ids,
+            vec![0; n],
+            tokens,
+            vec![None; n],
+            offsets,
+            special,
+            vec![1; n],
+            vec![],
+            Default::default(),
+        ));
+        if end == full.len() {
+            break;
+        }
+        start += capacity - stride;
+    }
+    Ok(windows)
+}
+fn decode(enc: &tokenizers::Encoding, logits: &[f32], labels: &[String]) -> Vec<Finding> {
+    let k = 1 + 2 * labels.len();
+    let mut out = Vec::new();
+    let mut current: Option<Finding> = None;
+    for (i, row) in logits.chunks_exact(k).enumerate() {
+        let (start, end) = enc.get_offsets()[i];
+        if enc.get_special_tokens_mask()[i] != 0 || enc.get_attention_mask()[i] == 0 || start == end
+        {
+            if let Some(f) = current.take() {
+                out.push(f);
+            }
+            continue;
+        }
+        let mut id = 0;
+        for j in 1..k {
+            if row[j] > row[id] {
+                id = j;
+            }
+        }
+        if id == 0 {
+            if let Some(f) = current.take() {
+                out.push(f);
+            }
+            continue;
+        }
+        let label = &labels[(id - 1) / 2];
+        let max = row[id] as f64;
+        let prob = 1.0 / row.iter().map(|v| (*v as f64 - max).exp()).sum::<f64>();
+        if id % 2 == 0 && current.as_ref().is_some_and(|f| f.label == *label) {
+            let f = current.as_mut().unwrap();
+            f.end = end;
+            f.raw_score = f.raw_score.min(prob);
+        } else {
+            if let Some(f) = current.take() {
+                out.push(f);
+            }
+            current = Some(Finding {
+                label: label.clone(),
+                start,
+                end,
+                raw_score: prob,
+                confidence: None,
+                boundary_fragment: false,
+            });
+        }
+    }
+    if let Some(f) = current {
+        out.push(f);
+    }
+    out
+}
+fn mark_boundaries(
+    enc: &tokenizers::Encoding,
+    findings: &mut [Finding],
+    has_previous: bool,
+    has_next: bool,
+) {
+    let content: Vec<_> = enc.get_offsets().iter().filter(|(a, b)| a != b).collect();
+    if let (Some(first), Some(last)) = (content.first(), content.last()) {
+        for f in findings {
+            f.boundary_fragment =
+                (has_previous && f.start == first.0) || (has_next && f.end == last.1);
+        }
+    }
+}
+fn resolve(findings: Vec<Finding>) -> Vec<Finding> {
+    let mut findings: Vec<Finding> = findings
+        .iter()
+        .filter(|f| {
+            !(f.boundary_fragment
+                && findings.iter().any(|g| {
+                    g.label == f.label
+                        && g.start <= f.start
+                        && g.end >= f.end
+                        && (g.start < f.start || g.end > f.end)
+                }))
+        })
+        .cloned()
+        .collect();
+    findings.sort_by(|a, b| {
+        b.raw_score
+            .total_cmp(&a.raw_score)
+            .then(a.start.cmp(&b.start))
+            .then(a.end.cmp(&b.end))
+            .then(a.label.cmp(&b.label))
+    });
+    let mut accepted: Vec<Finding> = Vec::new();
+    for f in findings {
+        if !accepted.iter().any(|a| f.start < a.end && a.start < f.end) {
+            accepted.push(f);
+        }
+    }
+    accepted.sort_by_key(|f| (f.start, f.end));
+    accepted
+}
+fn redact(text: &str, findings: &[Finding]) -> Result<String> {
+    let mut s = String::new();
+    let mut pos = 0;
+    for f in findings {
+        ensure!(
+            f.start >= pos
+                && f.end <= text.len()
+                && text.is_char_boundary(f.start)
+                && text.is_char_boundary(f.end),
+            "invalid UTF-8 finding boundary"
+        );
+        s.push_str(&text[pos..f.start]);
+        s.push_str("[REDACTED]");
+        pos = f.end;
+    }
+    s.push_str(&text[pos..]);
+    Ok(s)
+}
+struct Detector {
+    config: Config,
+    tokenizer: Tokenizer,
+    session: Session,
+}
+impl Detector {
+    fn load(path: &Path) -> Result<Self> {
+        let mut config: Config = serde_json::from_slice(&std::fs::read(path.join("config.json"))?)?;
+        if let Some(file) = &config.calibration_file {
+            config.calibration = Some(serde_json::from_slice(&std::fs::read(path.join(file))?)?);
+        }
+        ensure!(config.labels.len() == 7, "expected seven labels");
+        ensure!(
+            config.max_length > 2 && config.stride < config.max_length - 2,
+            "invalid chunk settings"
+        );
+        if let Some(c) = &config.calibration {
+            ensure!(
+                !c.x.is_empty() && c.x.len() == c.y.len(),
+                "invalid calibration lengths"
+            );
+            ensure!(
+                c.x.iter().all(|x| x.is_finite())
+                    && c.y.iter().all(|y| y.is_finite() && (0.0..=1.0).contains(y))
+                    && c.x.windows(2).all(|v| v[0] < v[1])
+                    && c.y.windows(2).all(|v| v[0] <= v[1]),
+                "invalid calibration values"
+            );
+        }
+        let mut tokenizer = Tokenizer::from_file(path.join("tokenizer.json"))
+            .map_err(|e| anyhow::anyhow!("{e}"))?;
+        tokenizer.with_padding(None);
+        tokenizer
+            .with_truncation(None)
+            .map_err(|e| anyhow::anyhow!("{e}"))?;
+        let _ = ort::init_from(path.join(if cfg!(target_os = "windows") { "onnxruntime.dll" } else if cfg!(target_os = "macos") { "libonnxruntime.dylib" } else { "libonnxruntime.so" }))?.commit();
+        let session = Session::builder()?
+            .with_intra_threads(1)
+            .map_err(|e| anyhow::anyhow!("{e}"))?
+            .commit_from_file(path.join("model.onnx"))?;
+        Ok(Self {
+            config,
+            tokenizer,
+            session,
+        })
+    }
+    fn predict(&mut self, text: &str, debug: bool) -> Result<serde_json::Value> {
+        let encodings = encode_windows(
+            &self.tokenizer,
+            text,
+            self.config.max_length,
+            self.config.stride,
+        )?;
+        let mut findings = Vec::new();
+        let mut windows = Vec::new();
+        for (index, e) in encodings.iter().enumerate() {
+            let n = e.len();
+            let ids: Vec<i64> = e.get_ids().iter().map(|x| *x as i64).collect();
+            let mask: Vec<i64> = e.get_attention_mask().iter().map(|x| *x as i64).collect();
+            let outputs=self.session.run(ort::inputs!["input_ids"=>Tensor::from_array(([1,n],ids.clone()))?,"attention_mask"=>Tensor::from_array(([1,n],mask.clone()))?])?;
+            let (shape, logits) = outputs["logits"].try_extract_tensor::<f32>()?;
+            ensure!(
+                shape.as_ref() == [1, n as i64, 15],
+                "unexpected logits shape: {shape:?}"
+            );
+            ensure!(logits.iter().all(|x| x.is_finite()), "non-finite logits");
+            let mut decoded = decode(e, logits, &self.config.labels);
+            mark_boundaries(e, &mut decoded, index > 0, index + 1 < encodings.len());
+            if n == self.config.max_length {
+                let content: Vec<_> = e.get_offsets().iter().filter(|(a, b)| a != b).collect();
+                if let (Some(first), Some(last)) = (content.first(), content.last()) {
+                    ensure!(
+                        !decoded
+                            .iter()
+                            .any(|f| f.start == first.0 && f.end == last.1),
+                        "entity spans a full window; complete entity boundary is unsupported"
+                    );
+                }
+            }
+            findings.extend(decoded);
+            if debug {
+                windows.push(json!({"input_ids":ids,"attention_mask":mask,"offsets":e.get_offsets(),"special_tokens_mask":e.get_special_tokens_mask(),"logits":logits.chunks_exact(15).collect::<Vec<_>>()}));
+            }
+        }
+        let mut findings = resolve(findings);
+        if let Some(c) = &self.config.calibration {
+            for f in &mut findings {
+                f.confidence = Some(c.apply(f.raw_score));
+            }
+            if let Some(t) = c.threshold {
+                findings.retain(|f| f.confidence.unwrap() >= t);
+            }
+        }
+        let redacted = redact(text, &findings)?;
+        let mut result =
+            json!({"findings":findings,"contains_pii":!findings.is_empty(),"redacted":redacted});
+        if debug {
+            result["windows"] = json!(windows);
+        }
+        Ok(result)
+    }
+}
+fn main() -> Result<()> {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    if args.is_empty() {
+        bail!("usage: datafog-pii BUNDLE [--debug]; stdin JSONL objects with text:string");
+    }
+    ensure!(
+        args.len() == 1 || (args.len() == 2 && args[1] == "--debug"),
+        "unknown arguments"
+    );
+    let mut detector = Detector::load(Path::new(&args[0])).context("loading detector")?;
+    let debug = args.len() == 2;
+    let stdin = io::stdin();
+    let mut stdout = io::BufWriter::new(io::stdout().lock());
+    for line in stdin.lock().lines() {
+        let line = line?;
+        let result = (|| -> Result<_> {
+            let input: serde_json::Value = serde_json::from_str(&line)?;
+            let text = input
+                .get("text")
+                .and_then(|t| t.as_str())
+                .context("expected text:string")?;
+            detector.predict(text, debug)
+        })();
+        let output = match result {
+            Ok(v) => v,
+            Err(e) => json!({"error":format!("{e:#}")}),
+        };
+        writeln!(stdout, "{}", serde_json::to_string(&output)?)?;
+        stdout.flush()?;
+    }
+    Ok(())
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn unicode_overlap_and_adjacent() {
+        let f = |start, end, score| Finding {
+            label: "first_name".into(),
+            start,
+            end,
+            raw_score: score,
+            confidence: None,
+            boundary_fragment: false,
+        };
+        let spans = resolve(vec![f(5, 8, 0.6), f(5, 6, 0.9), f(6, 8, 0.8), f(5, 6, 0.7)]);
+        assert_eq!(spans.len(), 2);
+        assert_eq!(
+            redact("🙂 José", &spans).unwrap(),
+            "🙂 [REDACTED][REDACTED]é"
+        );
+        assert!(redact("🙂", &[f(1, 3, 1.)]).is_err());
+    }
+    #[test]
+    fn oracle_boundaries_and_complete_token_coverage() {
+        let fixture: serde_json::Value =
+            serde_json::from_str(include_str!("../tests/boundary-oracle.json")).unwrap();
+        let mut tokenizer = Tokenizer::from_file(
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../bundle/tokenizer.json"),
+        )
+        .unwrap();
+        tokenizer.with_truncation(None).unwrap();
+        tokenizer.with_padding(None);
+        let labels: Vec<String> = serde_json::from_value(fixture["labels"].clone()).unwrap();
+        for case in fixture["cases"].as_array().unwrap() {
+            let text = case["text"].as_str().unwrap();
+            let windows = encode_windows(&tokenizer, text, 512, 42).unwrap();
+            assert_eq!(windows.len(), case["windows"].as_array().unwrap().len());
+            let mut findings = Vec::new();
+            let mut covered = Vec::new();
+            for (i, e) in windows.iter().enumerate() {
+                let oracle = &case["windows"][i];
+                assert_eq!(
+                    serde_json::to_value(e.get_ids()).unwrap(),
+                    oracle["input_ids"]
+                );
+                assert_eq!(
+                    serde_json::to_value(e.get_offsets()).unwrap(),
+                    oracle["offsets"]
+                );
+                covered.extend_from_slice(&e.get_ids()[if i == 0 { 1 } else { 43 }..e.len() - 1]);
+                let logits: Vec<Vec<f32>> =
+                    serde_json::from_value(oracle["logits"].clone()).unwrap();
+                let mut f = decode(e, &logits.concat(), &labels);
+                mark_boundaries(e, &mut f, i > 0, i + 1 < windows.len());
+                findings.extend(f);
+            }
+            let full = tokenizer.encode(text, false).unwrap();
+            assert_eq!(covered, full.get_ids());
+            let actual = resolve(findings);
+            let spans: Vec<_> = actual
+                .iter()
+                .map(|f| json!({"label":f.label,"start":f.start,"end":f.end}))
+                .collect();
+            assert_eq!(json!(spans), case["expected"]);
+            assert_eq!(
+                redact(text, &actual).unwrap(),
+                case["redacted"].as_str().unwrap()
+            );
+        }
+    }
+    #[test]
+    fn interpolation() {
+        let c = Calibration {
+            x: vec![0.2, 0.8],
+            y: vec![0.1, 0.9],
+            threshold: None,
+        };
+        assert!((c.apply(0.5) - 0.5).abs() < 1e-12);
+        assert_eq!(c.apply(0.), 0.1);
+        assert_eq!(c.apply(1.), 0.9);
+    }
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "is_local": true,
+  "local_files_only": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

tools/install_bundle.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Install a verified, already-downloaded snapshot; inference stays offline."""
+import argparse
+import hashlib
+import json
+import platform
+import shutil
+import tarfile
+import tempfile
+from pathlib import Path
+
+
+def digest(path):
+    with path.open('rb') as stream:
+        return hashlib.file_digest(stream, 'sha256').hexdigest()
+
+
+def verify(root):
+    manifest = json.loads((root / 'manifest.json').read_text())
+    for name, metadata in manifest['files'].items():
+        path = root / name
+        if Path(name).is_absolute() or '..' in Path(name).parts:
+            raise ValueError('Unsafe manifest path')
+        if not path.is_file() or path.stat().st_size != metadata['bytes'] or digest(path) != metadata['sha256']:
+            raise ValueError(f'Integrity check failed: {name}')
+    return manifest
+
+
+def install(snapshot, destination):
+    verify(snapshot)
+    if platform.system() != 'Darwin' or platform.machine() not in ('arm64', 'aarch64'):
+        raise ValueError('This release archive is qualified only for macOS ARM64; see PLATFORMS.md')
+    version = platform.mac_ver()[0]
+    if version and tuple(map(int, version.split('.')[:2])) < (14, 0):
+        raise ValueError('The preserved dylib requires macOS 14.0 or newer')
+    if destination.exists():
+        raise ValueError('Destination already exists; select a new directory')
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(dir=destination.parent) as temporary:
+        stage = Path(temporary) / 'bundle'
+        stage.mkdir()
+        with tarfile.open(snapshot / 'archives/datafog-pii-v0.1.0-macos-arm64.tar.gz') as archive:
+            for member in archive.getmembers():
+                if member.name.startswith('/') or '..' in Path(member.name).parts or not (member.isfile() or member.isdir()):
+                    raise ValueError('Unsafe archive member')
+            archive.extractall(stage, filter='data')
+        (stage / 'datafog-pii').chmod(0o755)
+        # Validate extracted content independently, including executable and notices.
+        checksums = json.loads((stage / 'checksums.json').read_text())
+        for name, expected in checksums.items():
+            if digest(stage / name) != expected:
+                raise ValueError(f'Extracted bundle checksum failed: {name}')
+        stage.rename(destination)
+    return destination
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--snapshot', required=True, type=Path)
+    parser.add_argument('--destination', required=True, type=Path)
+    args = parser.parse_args()
+    print(install(args.snapshot.resolve(), args.destination.resolve()))