reproduction/scripts/benchmark_native.py

"""Observe native CPU load and end-to-end request costs at actual input lengths."""
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import time
import psutil

def main():
    root=Path(__file__).resolve().parents[1]
    bundle=root/"artifacts/bundle"
    binary=root/"rust/target/release/datafog-pii"
    records=[json.loads(line) for line in (root/"data/prepared/validation.jsonl").read_text().splitlines()]
    records.sort(key=lambda r:len(r["text"].encode()))
    # Observed minimum, median, p90 and maximum validation lengths, not limits.
    chosen=[records[i] for i in sorted({0,len(records)//2,int((len(records)-1)*0.9),len(records)-1})]
    start=time.perf_counter()
    process=subprocess.Popen([str(binary),str(bundle)],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,bufsize=1)
    # Empty request includes runtime initialization plus minimal inference: upper bound on load.
    process.stdin.write(json.dumps({"text":""})+"\n");process.stdin.flush()
    first=json.loads(process.stdout.readline())
    if "error" in first:raise RuntimeError(first)
    cold=time.perf_counter()-start
    monitor=psutil.Process(process.pid);peak=monitor.memory_info().rss
    observations=[]
    for row in chosen:
        start=time.perf_counter()
        process.stdin.write(json.dumps({"text":row["text"]})+"\n");process.stdin.flush()
        answer=json.loads(process.stdout.readline())
        elapsed=time.perf_counter()-start
        if "error" in answer:raise RuntimeError(answer)
        peak=max(peak,monitor.memory_info().rss)
        observations.append({"id":row["id"],"bytes":len(row["text"].encode()),"latency_seconds":elapsed,"findings":len(answer["findings"])})
    process.stdin.close();process.wait()
    if process.returncode:raise RuntimeError(process.stderr.read())
    shutil.copy2(binary,bundle/"datafog-pii")
    sums={}
    for path in sorted(bundle.rglob("*")):
        if path.is_file() and path.name!="checksums.json":
            sums[str(path.relative_to(bundle))]=hashlib.sha256(path.read_bytes()).hexdigest()
    (bundle/"checksums.json").write_text(json.dumps(sums,indent=2)+"\n")
    evidence={"backend":"native Rust + ONNX Runtime CPU, one inference thread","bundle_bytes":sum(p.stat().st_size for p in bundle.rglob("*") if p.is_file()),"cold_load_seconds":cold,"cold_load_definition":"process launch through first empty-input response; includes initialization and minimal inference","warm_requests":observations,"peak_rss_bytes":peak,"memory_definition":"maximum RSS sampled after responses, not allocator high-water mark","warm_observed_requests_per_second":len(observations)/sum(r["latency_seconds"] for r in observations),"sampling":"one request each at observed min/median/p90/max validation byte lengths; descriptive observations, no latency guarantee"}
    (root/"artifacts/native-benchmark.json").write_text(json.dumps(evidence,indent=2)+"\n")
    print(json.dumps(evidence,indent=2))

if __name__=="__main__":main()