| { |
| "model": "build/checkpoint-v2", |
| "rows": 3899, |
| "compute_units": "all", |
| "bucket_counts": { |
| "128": 2259, |
| "256": 1151, |
| "512": 489 |
| }, |
| "macro": 0.7274937343358396, |
| "micro": 0.7289048473967684, |
| "coreml_p50_ms_python": 1.0283339652232826, |
| "coreml_p95_ms_python": 8.220041985623538, |
| "argmax_disagreements": 18, |
| "disagreement_examples": [ |
| { |
| "row": 42, |
| "suite": "massive_intent.en", |
| "bucket": 128, |
| "coreml": 9, |
| "pytorch": 19 |
| }, |
| { |
| "row": 237, |
| "suite": "massive_intent.en", |
| "bucket": 128, |
| "coreml": 17, |
| "pytorch": 10 |
| }, |
| { |
| "row": 928, |
| "suite": "jev.emotion", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 2 |
| }, |
| { |
| "row": 965, |
| "suite": "jev.emotion", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 2 |
| }, |
| { |
| "row": 989, |
| "suite": "jev.emotion", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 2 |
| }, |
| { |
| "row": 1132, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 9, |
| "pytorch": 7 |
| }, |
| { |
| "row": 1133, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 3, |
| "pytorch": 9 |
| }, |
| { |
| "row": 1135, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 7, |
| "pytorch": 9 |
| }, |
| { |
| "row": 1173, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 3, |
| "pytorch": 0 |
| }, |
| { |
| "row": 1196, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 0, |
| "pytorch": 9 |
| }, |
| { |
| "row": 1433, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 0, |
| "pytorch": 7 |
| }, |
| { |
| "row": 1456, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 8, |
| "pytorch": 9 |
| }, |
| { |
| "row": 1488, |
| "suite": "app.support_triage", |
| "bucket": 256, |
| "coreml": 1, |
| "pytorch": 9 |
| }, |
| { |
| "row": 2567, |
| "suite": "app.guardrails_jailbreak", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 0 |
| }, |
| { |
| "row": 2746, |
| "suite": "app.moderation_toxicity", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 0 |
| }, |
| { |
| "row": 2886, |
| "suite": "app.moderation_toxicity", |
| "bucket": 128, |
| "coreml": 1, |
| "pytorch": 0 |
| }, |
| { |
| "row": 3147, |
| "suite": "app.rag_relevance", |
| "bucket": 256, |
| "coreml": 0, |
| "pytorch": 1 |
| }, |
| { |
| "row": 3360, |
| "suite": "app.rag_relevance", |
| "bucket": 256, |
| "coreml": 1, |
| "pytorch": 0 |
| } |
| ], |
| "max_logit_absolute_error": 0.2099146842956543, |
| "mean_logit_absolute_error": 0.016456271121731418, |
| "suites": { |
| "jev.ag_news": { |
| "n": 400, |
| "ok": 343, |
| "accuracy": 0.8575 |
| }, |
| "jev.emotion": { |
| "n": 400, |
| "ok": 222, |
| "accuracy": 0.555 |
| }, |
| "massive_intent.en": { |
| "n": 300, |
| "ok": 201, |
| "accuracy": 0.67 |
| }, |
| "app.support_triage": { |
| "n": 400, |
| "ok": 104, |
| "accuracy": 0.26 |
| }, |
| "app.email_spam": { |
| "n": 400, |
| "ok": 371, |
| "accuracy": 0.9275 |
| }, |
| "app.phishing": { |
| "n": 400, |
| "ok": 377, |
| "accuracy": 0.9425 |
| }, |
| "app.guardrails_jailbreak": { |
| "n": 400, |
| "ok": 349, |
| "accuracy": 0.8725 |
| }, |
| "app.moderation_toxicity": { |
| "n": 400, |
| "ok": 288, |
| "accuracy": 0.72 |
| }, |
| "app.rag_relevance": { |
| "n": 400, |
| "ok": 198, |
| "accuracy": 0.495 |
| }, |
| "app.model_routing_domain": { |
| "n": 399, |
| "ok": 389, |
| "accuracy": 0.974937343358396 |
| } |
| } |
| } |
|
|
