{
  "reference": "build/coreml/gliclass_edge_apps_fp16_L128_options25.mlpackage",
  "candidate": "build/coreml/gliclass_edge_apps_lut8_kmeans_per_tensor_L128_options25.mlpackage",
  "total_rows": 3899,
  "supported_rows": 3899,
  "unsupported_rows": 0,
  "argmax_agreement": 0.971018209797384,
  "reference_micro_accuracy": 0.7194152346755578,
  "candidate_micro_accuracy": 0.7217235188509874,
  "mean_logit_absolute_error": 0.14741103855128193,
  "max_logit_absolute_error": 1.887939453125,
  "candidate_median_ms_python": 0.7965000113472342,
  "candidate_p95_ms_python": 1.0225839796476066,
  "suites": {
    "massive_intent.en": {
      "rows": 300,
      "reference_correct": 201,
      "candidate_correct": 199,
      "agreement": 283,
      "reference_accuracy": 0.67,
      "candidate_accuracy": 0.6633333333333333
    },
    "jev.ag_news": {
      "rows": 400,
      "reference_correct": 341,
      "candidate_correct": 343,
      "agreement": 395,
      "reference_accuracy": 0.8525,
      "candidate_accuracy": 0.8575
    },
    "jev.emotion": {
      "rows": 400,
      "reference_correct": 222,
      "candidate_correct": 220,
      "agreement": 384,
      "reference_accuracy": 0.555,
      "candidate_accuracy": 0.55
    },
    "app.support_triage": {
      "rows": 400,
      "reference_correct": 92,
      "candidate_correct": 95,
      "agreement": 359,
      "reference_accuracy": 0.23,
      "candidate_accuracy": 0.2375
    },
    "app.email_spam": {
      "rows": 400,
      "reference_correct": 357,
      "candidate_correct": 353,
      "agreement": 392,
      "reference_accuracy": 0.8925,
      "candidate_accuracy": 0.8825
    },
    "app.phishing": {
      "rows": 400,
      "reference_correct": 374,
      "candidate_correct": 372,
      "agreement": 398,
      "reference_accuracy": 0.935,
      "candidate_accuracy": 0.93
    },
    "app.guardrails_jailbreak": {
      "rows": 400,
      "reference_correct": 339,
      "candidate_correct": 346,
      "agreement": 393,
      "reference_accuracy": 0.8475,
      "candidate_accuracy": 0.865
    },
    "app.moderation_toxicity": {
      "rows": 400,
      "reference_correct": 283,
      "candidate_correct": 285,
      "agreement": 394,
      "reference_accuracy": 0.7075,
      "candidate_accuracy": 0.7125
    },
    "app.rag_relevance": {
      "rows": 400,
      "reference_correct": 202,
      "candidate_correct": 206,
      "agreement": 392,
      "reference_accuracy": 0.505,
      "candidate_accuracy": 0.515
    },
    "app.model_routing_domain": {
      "rows": 399,
      "reference_correct": 394,
      "candidate_correct": 395,
      "agreement": 396,
      "reference_accuracy": 0.9874686716791979,
      "candidate_accuracy": 0.9899749373433584
    }
  }
}