| { |
| "schema_version": 1, |
| "generated_at_utc": "2026-09-22T19:57:53.364394+00:00", |
| "baseline": "torch-fp32", |
| "results_directory": "results/final", |
| "environment_latest_invocation": { |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39", |
| "python": "3.12.11", |
| "packages": { |
| "torch": "2.8.0", |
| "transformers": "4.57.1", |
| "onnx": "1.19.1", |
| "onnxruntime-gpu": "1.23.2", |
| "numpy": "2.5.3", |
| "tokenizers": "0.22.2" |
| }, |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "timing": "perf_counter_ns; synchronous output to CPU; tokenizer + windows + GPU + decoding; batch1", |
| "repeats": 3, |
| "seed": 42, |
| "quality_batch_size": 1, |
| "min_confidence": 0.3, |
| "stride": 128, |
| "max_length": 512, |
| "torch_allow_tf32": false, |
| "onnx_use_tf32": false, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "dataset": { |
| "dataset_id": "redmadrobot-rnd/pii_benchmark", |
| "revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "split": "test", |
| "source_url": "https://huggingface.co/datasets/redmadrobot-rnd/pii_benchmark/resolve/f77ea831274daf980cc45c61a93c226be9d978d6/test.csv", |
| "source_rows": 2841, |
| "evaluated_rows": 2841, |
| "excluded_rows": [], |
| "case_insensitive_alignment_rows": [ |
| { |
| "id": "row_0000", |
| "source_row_zero_based": 0, |
| "reason": "Token WWW differs from original text www; same-length case-insensitive comparison at original offsets" |
| }, |
| { |
| "id": "row_1628", |
| "source_row_zero_based": 1628, |
| "reason": "Token WWW differs from original text www; same-length case-insensitive comparison at original offsets" |
| } |
| ], |
| "strict_alignment_rows": 2839, |
| "gold_entities": 5614, |
| "per_label_support": { |
| "BIRTH_CERTIFICATE": 341, |
| "CITY": 339, |
| "COUNTRY": 242, |
| "CREDIT_CARD": 203, |
| "DISTRICT": 161, |
| "DRIVER_LICENSE": 371, |
| "EMAIL": 221, |
| "FIRST_NAME": 499, |
| "HOUSE": 163, |
| "INN": 263, |
| "IP_ADDRESS": 155, |
| "LAST_NAME": 457, |
| "MIDDLE_NAME": 308, |
| "MILITARY_ID": 278, |
| "OMS": 170, |
| "PASSPORT": 494, |
| "PHONE": 173, |
| "REGION": 176, |
| "SNILS": 223, |
| "STREET": 171, |
| "URL": 206 |
| }, |
| "alignment_policy": "Original text unchanged; exact monotonic offsets; whitespace-only gaps; strict BIO; empty O token allowed; only equal-length case differences at original offsets allowed as explicit fallback", |
| "primary_metric": "Exact span micro-F1 across all 21 fine labels, gold BIO boundaries unchanged", |
| "secondary_metric": "Coarse14 exact (no gold merge), and PERSON+LOCATION overlap after identical whitespace merge on both sides", |
| "published_score_comparability": "Leaderboard-style diagnostic only: the public code differs from the card; this evaluation preserves all 2841 rows, explicitly aligns WWW/www on two rows, and never tunes a threshold on test" |
| }, |
| "evaluated_rows": 2841, |
| "record_ids_count": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_count": 256, |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a" |
| }, |
| "caveats": [ |
| "Quality uses exact character-span micro-F1 over all 21 model entity types; native gold BIO boundaries are preserved.", |
| "This is a paired backend comparison. The PERSON+LOCATION overlap diagnostic does not claim to reproduce the published model-card or leaderboard F1.", |
| "End-to-end batch1 means one document request: tokenization, sliding windows, GPU work, transfers, and entity decoding. Long documents may require multiple windows.", |
| "Fixed-shape model-with-transfers timing includes host-to-device and device-to-host transfer and synchronized CPU output, but excludes tokenization and entity decoding; it is not GPU-kernel-only latency.", |
| "Latency is warm, local, sequential, and excludes model loading, HTTP overhead, queueing, and concurrent clients. Repeated timings of the same texts are not independent unique requests.", |
| "Quality and end-to-end latency both use one window per GPU call. Long-document windows run sequentially; batch16 throughput is measured separately.", |
| "No threshold is tuned on this benchmark; the shared decoder uses a fixed confidence threshold." |
| ], |
| "warnings": [], |
| "backends": [ |
| { |
| "backend": "torch-fp32", |
| "source_file": "torch-fp32.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8093242535358827, |
| "recall": 0.8256145350908444, |
| "f1": 0.8173882373688387, |
| "delta_f1_pp": 0.0, |
| "delta_recall_pp": 0.0, |
| "largest_per_label_changes": [], |
| "largest_per_label_recall_decline": null, |
| "changed_documents": 0, |
| "document_disagreement_fraction": 0.0, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 8.9687816875, |
| "p50_ms": 6.387492, |
| "p95_ms": 18.345667699999993, |
| "p99_ms": 27.723387640000013 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 1.0, |
| "speedup_e2e_p95": 1.0, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 7.274395559999999, |
| "p50_ms": 5.332269500000001, |
| "p95_ms": 16.473245, |
| "p99_ms": 17.009684940000007, |
| "speedup_p50": 1.0 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 8.11055105, |
| "p50_ms": 6.35912, |
| "p95_ms": 17.328865450000002, |
| "p99_ms": 18.138672500000013, |
| "speedup_p50": 1.0 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 9.80815812, |
| "p50_ms": 8.737929000000001, |
| "p95_ms": 18.012525, |
| "p99_ms": 19.19540607, |
| "speedup_p50": 1.0 |
| } |
| }, |
| "throughput_batch16_docs_s": 713.082441394094, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| }, |
| { |
| "backend": "trt-graph", |
| "source_file": "trt-graph.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8087260034904014, |
| "recall": 0.8254364089775561, |
| "f1": 0.8169957686882934, |
| "delta_f1_pp": -0.03924686805453215, |
| "delta_recall_pp": -0.017812611328826833, |
| "largest_per_label_changes": [ |
| { |
| "label": "COUNTRY", |
| "support": 242, |
| "f1": 0.8547008547008547, |
| "recall": 0.8264462809917356, |
| "delta_f1_pp": -0.6112849795933295, |
| "delta_recall_pp": -0.4132231404958664 |
| }, |
| { |
| "label": "INN", |
| "support": 263, |
| "f1": 0.6795366795366795, |
| "recall": 0.6692015209125475, |
| "delta_f1_pp": -0.38610038610038533, |
| "delta_recall_pp": -0.38022813688213253 |
| }, |
| { |
| "label": "FIRST_NAME", |
| "support": 499, |
| "f1": 0.9012096774193549, |
| "recall": 0.8957915831663327, |
| "delta_f1_pp": 0.20161290322581182, |
| "delta_recall_pp": 0.2004008016032066 |
| } |
| ], |
| "largest_per_label_recall_decline": { |
| "label": "COUNTRY", |
| "support": 242, |
| "f1": 0.8547008547008547, |
| "recall": 0.8264462809917356, |
| "delta_f1_pp": -0.6112849795933295, |
| "delta_recall_pp": -0.4132231404958664 |
| }, |
| "changed_documents": 10, |
| "document_disagreement_fraction": 0.0035198873636043126, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 1.4303671210937499, |
| "p50_ms": 1.0887105, |
| "p95_ms": 2.9440187499999975, |
| "p99_ms": 6.8875136800000005 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 5.867025256025363, |
| "speedup_e2e_p95": 6.231505047309909, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 0.8334529, |
| "p50_ms": 0.709606, |
| "p95_ms": 1.3865699999999999, |
| "p99_ms": 1.5624917000000011, |
| "speedup_p50": 7.514408700039178 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 0.8977010100000001, |
| "p50_ms": 0.7679715, |
| "p95_ms": 1.5429889, |
| "p99_ms": 1.8931186800000013, |
| "speedup_p50": 8.280411447560228 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 2.1506562, |
| "p50_ms": 2.0664255000000002, |
| "p95_ms": 3.159513, |
| "p99_ms": 3.36587041, |
| "speedup_p50": 4.228523602713962 |
| } |
| }, |
| "throughput_batch16_docs_s": 1673.8645348271762, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| }, |
| { |
| "backend": "trt-fp16", |
| "source_file": "trt-fp16.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8087260034904014, |
| "recall": 0.8254364089775561, |
| "f1": 0.8169957686882934, |
| "delta_f1_pp": -0.03924686805453215, |
| "delta_recall_pp": -0.017812611328826833, |
| "largest_per_label_changes": [ |
| { |
| "label": "COUNTRY", |
| "support": 242, |
| "f1": 0.8547008547008547, |
| "recall": 0.8264462809917356, |
| "delta_f1_pp": -0.6112849795933295, |
| "delta_recall_pp": -0.4132231404958664 |
| }, |
| { |
| "label": "INN", |
| "support": 263, |
| "f1": 0.6795366795366795, |
| "recall": 0.6692015209125475, |
| "delta_f1_pp": -0.38610038610038533, |
| "delta_recall_pp": -0.38022813688213253 |
| }, |
| { |
| "label": "FIRST_NAME", |
| "support": 499, |
| "f1": 0.9012096774193549, |
| "recall": 0.8957915831663327, |
| "delta_f1_pp": 0.20161290322581182, |
| "delta_recall_pp": 0.2004008016032066 |
| } |
| ], |
| "largest_per_label_recall_decline": { |
| "label": "COUNTRY", |
| "support": 242, |
| "f1": 0.8547008547008547, |
| "recall": 0.8264462809917356, |
| "delta_f1_pp": -0.6112849795933295, |
| "delta_recall_pp": -0.4132231404958664 |
| }, |
| "changed_documents": 8, |
| "document_disagreement_fraction": 0.002815909890883539, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 2.8120914114583333, |
| "p50_ms": 2.1979795, |
| "p95_ms": 5.564566999999999, |
| "p99_ms": 9.163146490000027 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 2.9060744197113757, |
| "speedup_e2e_p95": 3.296872461055819, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 1.7901495699999999, |
| "p50_ms": 1.5656425, |
| "p95_ms": 3.172446, |
| "p99_ms": 4.487881570000003, |
| "speedup_p50": 3.4058027295503286 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 1.5199791799999998, |
| "p50_ms": 1.5550324999999998, |
| "p95_ms": 2.2867704999999994, |
| "p99_ms": 2.5846311800000006, |
| "speedup_p50": 4.089380768569146 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 2.8324707500000006, |
| "p50_ms": 2.6145329999999998, |
| "p95_ms": 4.54732995, |
| "p99_ms": 5.16478469, |
| "speedup_p50": 3.342061087008656 |
| } |
| }, |
| "throughput_batch16_docs_s": 1808.154612064009, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| }, |
| { |
| "backend": "onnx-fp16", |
| "source_file": "onnx-fp16.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8091829608938548, |
| "recall": 0.8256145350908444, |
| "f1": 0.8173161699876565, |
| "delta_f1_pp": -0.007206738118215039, |
| "delta_recall_pp": 0.0, |
| "largest_per_label_changes": [ |
| { |
| "label": "INN", |
| "support": 263, |
| "f1": 0.6820809248554913, |
| "recall": 0.6730038022813688, |
| "delta_f1_pp": -0.13167585421920425, |
| "delta_recall_pp": 0.0 |
| }, |
| { |
| "label": "DRIVER_LICENSE", |
| "support": 371, |
| "f1": 0.772169167803547, |
| "recall": 0.7628032345013477, |
| "delta_f1_pp": 0.10520015910130409, |
| "delta_recall_pp": 0.0 |
| }, |
| { |
| "label": "PASSPORT", |
| "support": 494, |
| "f1": 0.7907907907907908, |
| "recall": 0.7995951417004049, |
| "delta_f1_pp": -0.07923755418745015, |
| "delta_recall_pp": 0.0 |
| } |
| ], |
| "largest_per_label_recall_decline": null, |
| "changed_documents": 7, |
| "document_disagreement_fraction": 0.0024639211545230966, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 3.075605110677083, |
| "p50_ms": 2.477493, |
| "p95_ms": 6.228689999999999, |
| "p99_ms": 8.676384970000008 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 2.57820788999202, |
| "speedup_e2e_p95": 2.9453492949560816, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 2.35228557, |
| "p50_ms": 2.151414, |
| "p95_ms": 4.234126949999999, |
| "p99_ms": 5.602559630000007, |
| "speedup_p50": 2.4784953058779022 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 3.593195329999999, |
| "p50_ms": 2.628103, |
| "p95_ms": 7.632001399999999, |
| "p99_ms": 8.93119823000002, |
| "speedup_p50": 2.419661634266237 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 2.83443601, |
| "p50_ms": 2.620768, |
| "p95_ms": 4.398550349999998, |
| "p99_ms": 5.487695390000002, |
| "speedup_p50": 3.3341100776566264 |
| } |
| }, |
| "throughput_batch16_docs_s": 1752.4336323184366, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| }, |
| { |
| "backend": "onnx-fp32", |
| "source_file": "onnx-fp32.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8093242535358827, |
| "recall": 0.8256145350908444, |
| "f1": 0.8173882373688387, |
| "delta_f1_pp": 0.0, |
| "delta_recall_pp": 0.0, |
| "largest_per_label_changes": [], |
| "largest_per_label_recall_decline": null, |
| "changed_documents": 0, |
| "document_disagreement_fraction": 0.0, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 4.338136834635416, |
| "p50_ms": 3.283219, |
| "p95_ms": 8.534234999999999, |
| "p99_ms": 16.62495859000002 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 1.9454967822737381, |
| "speedup_e2e_p95": 2.149655792229766, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 2.5534402, |
| "p50_ms": 2.245244, |
| "p95_ms": 4.1106772, |
| "p99_ms": 6.435467550000005, |
| "speedup_p50": 2.37491760361012 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 3.9541245400000005, |
| "p50_ms": 3.243965, |
| "p95_ms": 7.9958354, |
| "p99_ms": 9.127662770000002, |
| "speedup_p50": 1.9602924199243825 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 7.6860065099999995, |
| "p50_ms": 7.606868, |
| "p95_ms": 9.7087102, |
| "p99_ms": 10.119910030000007, |
| "speedup_p50": 1.148689447483511 |
| } |
| }, |
| "throughput_batch16_docs_s": 768.269900900315, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| }, |
| { |
| "backend": "torch-fp16", |
| "source_file": "torch-fp16.json", |
| "examples": 2841, |
| "gold_entities": 5614, |
| "full_dataset": true, |
| "quality_comparable_to_baseline": true, |
| "precision": 0.8094988650253186, |
| "recall": 0.8257926612041325, |
| "f1": 0.817564588660612, |
| "delta_f1_pp": 0.017635129177329123, |
| "delta_recall_pp": 0.01781261132881573, |
| "largest_per_label_changes": [ |
| { |
| "label": "FIRST_NAME", |
| "support": 499, |
| "f1": 0.9012096774193549, |
| "recall": 0.8957915831663327, |
| "delta_f1_pp": 0.20161290322581182, |
| "delta_recall_pp": 0.2004008016032066 |
| } |
| ], |
| "largest_per_label_recall_decline": null, |
| "changed_documents": 3, |
| "document_disagreement_fraction": 0.001055966209081327, |
| "latency_e2e_batch1": { |
| "count": 768, |
| "mean_ms": 8.734926875000001, |
| "p50_ms": 6.832012499999999, |
| "p95_ms": 18.264540849999996, |
| "p99_ms": 22.715963760000022 |
| }, |
| "timing_unique_texts": 256, |
| "timing_repeats": 3, |
| "speedup_e2e_p50": 0.9349356430480771, |
| "speedup_e2e_p95": 1.0044417678312454, |
| "latency_model_with_transfers_batch1": { |
| "32": { |
| "count": 100, |
| "mean_ms": 8.92398116, |
| "p50_ms": 7.36248, |
| "p95_ms": 17.618035449999997, |
| "p99_ms": 19.597754260000006, |
| "speedup_p50": 0.7242490981299781 |
| }, |
| "128": { |
| "count": 100, |
| "mean_ms": 8.07579839, |
| "p50_ms": 6.37584, |
| "p95_ms": 16.722866449999998, |
| "p99_ms": 17.580922550000007, |
| "speedup_p50": 0.9973776004416672 |
| }, |
| "512": { |
| "count": 100, |
| "mean_ms": 6.3415938800000005, |
| "p50_ms": 5.629647, |
| "p95_ms": 10.786636199999998, |
| "p99_ms": 12.509507950000003, |
| "speedup_p50": 1.5521273358702599 |
| } |
| }, |
| "throughput_batch16_docs_s": 1076.637310592768, |
| "per_backend_provenance_present": true, |
| "identity": { |
| "model_revision": "c802e8cd26f85d1cf920973ea6f83965a0618d63", |
| "dataset_revision": "f77ea831274daf980cc45c61a93c226be9d978d6", |
| "dataset_sha256": "6bf544a380a3ee5bec94b946124bea3afaecce49e734679ad0f0c0e7c12977bb", |
| "min_confidence": 0.3, |
| "evaluated_rows": 2841, |
| "record_ids_sha256": "7964ca1d5f775e2e8193c37656ac007ad3b251f6a3c4053fc750b90795c6175c", |
| "timing_sample_ids_sha256": "300b590ad0589cb5cd611ebc5f7cd343008fbe49b269dbe9ba35ac7ce813195a", |
| "timing_unique_texts": 256, |
| "repeats": 3, |
| "quality_batch_size": 1, |
| "stride": 128, |
| "max_length": 512, |
| "source_sha256": { |
| "inference.py": "e1405c61c77765a3d7252ce4fbc597604821a13ca2dc6b922bc8cacb7b61b2ae", |
| "evaluation.py": "02600d8abd80d582ea154f570f102f9051c51d5e14b9cdf09b439f6e264ce015", |
| "trt_backend.py": "eafc144a9b698e94dfe155074c7a3ab1063ee7817ab1a285b69639db1c07b424", |
| "benchmark.py": "5a74707e3e2e9f23b5c4f7ebc2e0d296e9e6ff716f5417d798aa08d7961ee97b" |
| }, |
| "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER", |
| "cuda": "12.8", |
| "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.39" |
| }, |
| "warnings": [] |
| } |
| ] |
| } |
|
|
