| #!/usr/bin/env python3 | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from pii.evaluation import evaluate, read_jsonl, write_json | |
| if __name__ == '__main__': | |
| p = argparse.ArgumentParser(description='Evaluate complete saved span predictions against character-offset gold JSONL') | |
| for name in ('gold', 'predictions', 'output'): | |
| p.add_argument('--' + name, required=True) | |
| p.add_argument('--calibration') | |
| p.add_argument('--threshold', type=float) | |
| args = p.parse_args() | |
| calibration = json.loads(Path(args.calibration).read_text()) if args.calibration else None | |
| result = evaluate(read_jsonl(args.gold), read_jsonl(args.predictions), args.threshold, calibration) | |
| write_json(args.output, result) | |
| print(json.dumps({'overall': result['overall'], 'documents': result['documents']})) | |