| { |
| "reports": [ |
| { |
| "source": "artifacts/benchmarks/loudink_v1/daily_mac/loudink_v1_daily_mac_measurement.json", |
| "label": "loudink-v1 v0.4 daily-mac product", |
| "writing": { |
| "shipped": { |
| "total": 22, |
| "passed": 22, |
| "accuracy": 1.0 |
| }, |
| "neural": { |
| "total": 22, |
| "passed": 19, |
| "accuracy": 0.8636363636363636, |
| "missing": 0 |
| }, |
| "polish_lift": 0.13636363636363635, |
| "latency": { |
| "n": 22, |
| "mean": 174.0530966088853, |
| "p50": 179.1810840368271, |
| "p95": 332.44862500578165, |
| "max": 358.969375025481, |
| "model_invoked_p50": 185.30124984681606, |
| "model_invoked_p95": 332.44862500578165, |
| "model_invoked_n": 18, |
| "fast_path_rate": 0.18181818181818177 |
| } |
| }, |
| "computer": { |
| "shipped": { |
| "total": 34, |
| "passed": 34, |
| "accuracy": 1.0 |
| }, |
| "neural": { |
| "total": 34, |
| "passed": 1, |
| "accuracy": 0.029411764705882353, |
| "missing": 33 |
| }, |
| "latency": { |
| "n": 34, |
| "mean": 13.59642523723271, |
| "p50": 0.05, |
| "p95": 0.05, |
| "max": 460.6284580659121, |
| "model_invoked_p50": 460.6284580659121, |
| "model_invoked_p95": 460.6284580659121, |
| "model_invoked_n": 1, |
| "fast_path_rate": 0.9705882352941176 |
| } |
| }, |
| "multi_step": { |
| "shipped": { |
| "total": 21, |
| "passed": 21, |
| "accuracy": 1.0 |
| }, |
| "latency": { |
| "n": 21, |
| "mean": 21.982307526948198, |
| "p50": 0.05, |
| "p95": 0.05, |
| "max": 460.6284580659121, |
| "model_invoked_p50": 460.6284580659121, |
| "model_invoked_p95": 460.6284580659121, |
| "model_invoked_n": 1, |
| "fast_path_rate": 0.9523809523809523 |
| }, |
| "n": 21 |
| }, |
| "overall": { |
| "shipped": { |
| "total": 56, |
| "passed": 56, |
| "accuracy": 1.0 |
| }, |
| "latency": { |
| "n": 56, |
| "mean": 76.63297470466766, |
| "p50": 0.05, |
| "p95": 314.540958032012, |
| "max": 460.6284580659121, |
| "model_invoked_p50": 194.78566688485444, |
| "model_invoked_p95": 358.969375025481, |
| "model_invoked_n": 19, |
| "fast_path_rate": 0.6607142857142857 |
| } |
| }, |
| "finish_reasons": { |
| "transcript_ir_fast_path": 33, |
| "loudink_v1_structural_polish": 18, |
| "transcript_polish_fast_path": 4, |
| "ir_model_lean": 1 |
| } |
| } |
| ] |
| } |
|
|
