#!/usr/bin/env python3 """Render final measured evidence as Markdown; absent evidence stays a blocker.""" import argparse import json from pathlib import Path def load(root, name): path = root / name return json.loads(path.read_text()) if path.exists() else None def number(value, decimals=4): if value is None: return 'Unavailable' return f'{value:,.{decimals}f}' if isinstance(value, (int, float)) else str(value) def table(headers, rows): safe = lambda v: str(v).replace('|', '\\|').replace('\n', ' ') return '\n'.join(['| '+' | '.join(headers)+' |', '| '+' | '.join(['---']*len(headers))+' |'] + ['| '+' | '.join(safe(v) for v in row)+' |' for row in rows]) def render(root): paths = { 'test':'artifacts/evaluation/test.json', 'fixtures':'artifacts/evaluation/support-fixtures.json', 'regex':'artifacts/regex-test/regex-metrics.json', 'model_shared':'artifacts/regex-test/model-email-phone-metrics.json', 'calibration':'artifacts/bundle/calibration.json', 'training':'artifacts/training/complete.json', 'data':'artifacts/data-manifest.json', 'parity':'artifacts/bundle/rust-parity.json', 'benchmark':'artifacts/native-benchmark.json', 'training_check':'artifacts/training-check.json', 'boundary':'artifacts/boundary-proof.json'} a = {k:load(root,p) for k,p in paths.items()} history_path = root/'artifacts/training/history.jsonl' history = [json.loads(line) for line in history_path.read_text().splitlines() if line.strip()] if history_path.exists() else [] if not history and a['training']: history = a['training'].get('history', []) evidence = lambda key: f'[{paths[key]}]({paths[key]})' lines = ['# DataFog PII prototype evaluation', '', 'This is an evaluated prototype, not a production-readiness claim. All numbers below come from saved artifacts; absent evidence is explicitly marked unavailable.', ''] required = ('test','fixtures','regex','model_shared','calibration','training','data','parity','benchmark') missing = [paths[k] for k in required if a[k] is None] lines += [('Required evaluation artifacts are present.' if not missing else '**Incomplete evidence:** '+', '.join(missing)), ''] criteria = [] def criterion(title, passed, keys, detail): criteria.append([title, 'Evidence present' if passed else 'Unresolved / missing evidence', ', '.join(evidence(k) for k in keys), detail]) criterion('Pinned data and disjoint document families', bool(a['data'] and a['data'].get('pairwise_group_disjoint')), ['data'], 'Structural validation and split rules are saved; annotation semantics remain imperfect.') criterion('Trained checkpoint and resumability', bool(a['training'] and a['training_check']), ['training','training_check'], 'Completion plus separate optimizer/RNG resume diagnostic; inspect reported MPS parameter differences.') p=a['parity'] or {} criterion('Native Rust and Python parity', all(p.get(k) for k in ('token_ids_masks_global_byte_offsets_parity','findings_redaction_parity','malformed_request_explicit_error_and_recovery')), ['parity'], 'Checks tokenizer/global byte offsets, outputs/redaction, and explicit error recovery; parity does not prove quality.') criterion('Complete-input boundary behavior', bool(a['boundary']), ['boundary'], 'Manual global-offset window construction; finite fixture evidence and representability limits apply.') criterion('Independent calibration and experimental threshold', bool(a['calibration'] and a['test']), ['calibration','test'], 'Fit/select partitions are separate; final reliability is reported on held-out predicted spans.') criterion('Held-out quality and support mechanics', bool(a['test'] and a['fixtures']), ['test','fixtures'], 'Synthetic population only; no reviewed private tickets.') criterion('Existing regex comparison', bool(a['regex'] and a['model_shared']), ['regex','model_shared'], 'Official published DataFog 4.8.1; email and phone only.') criterion('Native resource measurements', bool(a['benchmark']), ['benchmark'], 'Local observed load, warm request latency, memory, and bundle size; no performance target assumed.') lines += [table(['Contract criterion','Status','Evidence','Interpretation / limitation'],criteria),''] data=a['data'] or {} lines += ['## Data and execution', '', f"Dataset: `{data.get('source','Unavailable')}` revision `{data.get('revision','Unavailable')}`. License: {data.get('license','Unavailable')}. {data.get('attribution','')}", ''] if data.get('splits'): lines += [table(['Partition','Documents','Target-clean documents'],[(k,v.get('documents'),v.get('negative_documents')) for k,v in data['splits'].items()]),''] if data.get('exclusions'): lines += ['Structural exclusions: '+', '.join(f'{k}: {v:,}' for k,v in data['exclusions'].items())+'.',''] if history: lines += [table(['Epoch','Train mean batch loss','Validation token loss','Selected at epoch','Elapsed seconds'], [(r.get('epoch'),number(r.get('train_mean_batch_loss')),number(r.get('validation_token_loss')),r.get('selected'),number(r.get('elapsed_seconds'),1)) for r in history]),''] else: lines += ['Training history unavailable.',''] if a['training']: lines += [f"Training elapsed: {number(a['training'].get('elapsed_seconds'),1)} seconds. Best validation token loss: {number(a['training'].get('best_validation_loss'))}.",''] lines += ['Corrective iterations: two of the three-round fuse used. Round 1 fixed tokenizer overflow by manually constructing complete-input windows; round 2 kept the ONNX export wrapper and Python reference in evaluation mode. Neither correction tuned against final test outcomes. Further corrective work must remain within the agreed fuse.',''] test=a['test'] lines += ['## Held-out detection', ''] if test: metrics=[('overall',test.get('overall',{})), *test.get('per_label',{}).items()] lines += [table(['Label','Gold entities','Predicted','TP','FP','FN','Precision','Recall','F1'], [(label,m.get('support'),m.get('predicted'),m.get('tp'),m.get('fp'),m.get('fn'),number(m.get('precision')),number(m.get('recall')),number(m.get('f1'))) for label,m in metrics]),''] cov=test.get('redaction_byte_coverage',{}) lines += [f"UTF-8 byte-union redaction precision {number(cov.get('precision'))}, recall {number(cov.get('recall'))}, F1 {number(cov.get('f1'))}. Missed gold bytes: {number(cov.get('fn'),0)}; unnecessary redaction bytes: {number(cov.get('fp'),0)}. Coverage ignores labels and weights multibyte characters by byte length.",''] docs=test.get('documents',{}) lines += [table(['Document measure','Count'],[(k,number(v,0)) for k,v in docs.items()]),'', 'Document false negatives mean no accepted span on a document containing target PII; partial misses are counted separately in `with_any_missed_entity`. False positives mean at least one accepted span on a target-clean document.',''] else: lines += ['Held-out detection results unavailable.',''] lines += ['## Confidence and operating point',''] c=a['calibration'] if c: lines += [f"Method: {c.get('method','Unavailable')}, pooled across labels. Experimental acceptance threshold: {number(c.get('threshold'),8)}. Objective: {c.get('threshold_objective','Unavailable')}.",'', f"Calibration fit support: {number(c.get('fit_documents'),0)} documents, {number(c.get('fit_predicted_spans'),0)} predicted spans, {number(c.get('fit_correct_spans'),0)} exact matches. Selection support: {number(c.get('select_documents'),0)} documents.",''] lines += [f"Calibration warning: {warning}" for warning in c.get('warnings',[])] else: lines += ['Calibration artifact unavailable.',''] if test: raw=test.get('raw_reliability',{}); calibrated=test.get('confidence_reliability',{}) lines += [table(['Score','Predicted-span support','Brier','ECE'],[(name,r.get('support'),number(r.get('brier'),6),number(r.get('ece'),6)) for name,r in [('Raw token-derived span score',raw),('Calibrated exact-span score',calibrated)]]),''] rows=[] for i in range(10): rb=raw.get('bins',[])[i] if len(raw.get('bins',[]))>i else {} cb=calibrated.get('bins',[])[i] if len(calibrated.get('bins',[]))>i else {} rows.append([f'[{i/10:.1f}, {(i+1)/10:.1f}'+(']' if i==9 else ')'),rb.get('support','Unavailable'),number(rb.get('mean_score')),number(rb.get('accuracy')),cb.get('support','Unavailable'),number(cb.get('mean_score')),number(cb.get('accuracy'))]) lines += [table(['Confidence bin','Raw n','Raw mean','Raw accuracy','Calibrated n','Calibrated mean','Calibrated accuracy'],rows),''] lines += ['Reliability uses all predicted spans before thresholding and exact label/boundary correctness. Missed entities are absent from this table: recall must be read separately. Ten equal-width bins make ECE bin-dependent. Calibration is conditional on this synthetic evaluation population and annotation quality; the threshold is not a production default.',''] lines += ['## Existing regex comparison',''] if a['regex'] and a['model_shared']: rows=[] for name,key in [('DataFog 4.8.1 regex','regex'),('Trained model, experimental threshold','model_shared')]: for label in ['overall','email','phone_number']: m=a[key].get('overall',{}) if label=='overall' else a[key].get('per_label',{}).get(label,{}) rows.append([name,label,m.get('support'),number(m.get('precision')),number(m.get('recall')),number(m.get('f1'))]) lines += [table(['System','Scope','Gold entities','Precision','Recall','F1'],rows),''] else: lines += ['Complete same-population, same-label comparison unavailable.',''] lines += ['Only email and phone are supported by this existing regex baseline within the seven-label contract. First name, last name, street address, customer ID, and account number are excluded from both sides of this head-to-head comparison. All documents remain in the comparison. Constant regex match indicators are not calibrated probabilities. See [baseline provenance](docs/regex-baseline.md).',''] lines += ['## Synthetic support fixtures',''] if a['fixtures']: f=a['fixtures']; m=f.get('overall',{}) lines += [f"{number(f.get('documents',{}).get('total'),0)} fixture documents; exact-span precision {number(m.get('precision'))}, recall {number(m.get('recall'))}, F1 {number(m.get('f1'))}. Gold entity support: {number(m.get('support'),0)}.",''] else: lines += ['Fixture predictions unavailable.',''] lines += ['These hand-authored cases include Unicode signatures, pasted fields, multiline addresses, repeated entities, clean numeric text, and long inputs. They exercise mechanics and expose case-specific failures; they do not establish real customer-support quality.',''] lines += ['## Native resource measurements',''] b=a['benchmark'] if b: lines += [table(['Measurement','Observed value'],[(k,number(b.get(k),3 if k=='cold_load_seconds' else 0)) for k in ('bundle_bytes','cold_load_seconds','peak_rss_bytes')]),''] warm=b.get('warm',b.get('requests',b.get('warm_requests',[]))) if isinstance(warm,dict): warm=warm.get('requests',[]) if isinstance(warm,list) and warm: lines += [table(['Input UTF-8 bytes','Warm end-to-end seconds'],[(r.get('bytes','Unavailable'),number(r.get('latency_seconds'),6)) for r in warm]),''] if b.get('note'): lines += [str(b['note']),''] else: lines += ['Native resource measurements unavailable.',''] if a['parity']: lines += [f"Maximum absolute Python/Rust logit difference on parity cases: {number(p.get('max_abs_logit_difference'),8)}. Token IDs, masks and global byte offsets agree: {p.get('token_ids_masks_global_byte_offsets_parity','Unavailable')}; findings/redaction agree: {p.get('findings_redaction_parity','Unavailable')}.",''] unrep=data.get('tokenizer',{}).get('unrepresentable_boundaries_by_label',{}) lines += ['## Remaining limitations','', '- Nemotron is broad synthetic data with semantic annotation errors. Structural filtering cannot prove that labels are semantically correct or complete.', '- Split allocation uses documented label-support heuristics and grouped families. It does not guarantee statistical precision or eliminate every possible near-duplicate.', '- No private customer tickets were supplied or uploaded; real-world English customer-support generalization remains unproven.', '- Token boundaries cannot represent every character boundary. Observed training-source unrepresentable boundaries: '+(', '.join(f'{k}={v}' for k,v in unrep.items()) if unrep else 'unavailable')+'.', '- Native inference requires the packaged runtime and pinned tokenizer/configuration. A successful fixture parity check is not a universal numerical guarantee.', '- `contains_pii` means at least one accepted finding among the seven selected labels; it is not an independently calibrated probability or assurance that a document contains no other sensitive information.', '- No existing DataFog checkout was modified, no Git repository was created, and no model was published as part of this standalone prototype.', '', 'Reproduction commands and artifact locations are in [README.md](README.md).'] return '\n'.join(lines)+'\n' if __name__ == '__main__': parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--root',type=Path,default=Path(__file__).resolve().parents[1]) parser.add_argument('--output',type=Path) args=parser.parse_args() output=args.output or args.root/'REPORT.md' output.parent.mkdir(parents=True,exist_ok=True) output.write_text(render(args.root)) print(output)