{
  "model": "model.pt",
  "name": "byrne-jev",
  "device": "cuda",
  "benchmark": "LocalLLaMA/typed-decisions",
  "metrics": {
    "accuracy": 0.63,
    "soft_accuracy": 0.5092,
    "brier_score": 0.134,
    "ece": 0.0452,
    "score_mae": 0.4277,
    "within_1_level": 0.915,
    "kl_divergence": 0.2283,
    "total_variation": 0.2196,
    "latency_p50_ms": 110.8594,
    "latency_p95_ms": 157.5138,
    "n_decisions": 2000
  },
  "per_workflow": {
    "agent_trace_observability": {
      "n_decisions": 500,
      "accuracy": 0.704
    },
    "customer_service": {
      "n_decisions": 500,
      "accuracy": 0.61
    },
    "invoice_processing": {
      "n_decisions": 500,
      "accuracy": 0.552
    },
    "security_incidents": {
      "n_decisions": 500,
      "accuracy": 0.654
    }
  },
  "comparison": [
    {
      "Model": "TypeSafe Jev 1.13.0",
      "Kind": "general",
      "Accuracy": 0.727,
      "Soft Acc": 0.58,
      "Brier": 0.148,
      "ECE": 0.144,
      "Score MAE": 0.391,
      "Within 1 Level": 0.952,
      "ms/case": 710
    },
    {
      "Model": "Laya typed-decisions (421M)",
      "Kind": "fine-tuned",
      "Accuracy": 0.766,
      "Soft Acc": 0.471,
      "Brier": null,
      "ECE": null,
      "Score MAE": null,
      "Within 1 Level": null,
      "ms/case": null
    },
    {
      "Model": "ModernBERT-base (149M)",
      "Kind": "specialist",
      "Accuracy": 0.646,
      "Soft Acc": 0.542,
      "Brier": 0.119,
      "ECE": 0.179,
      "Score MAE": 0.444,
      "Within 1 Level": 0.931,
      "ms/case": 349
    },
    {
      "Model": "Teacher Self-Agreement",
      "Kind": "ceiling",
      "Accuracy": 0.735
    }
  ],
  "benchmarks": {
    "ag_news": {
      "n": 200,
      "accuracy": 0.87,
      "ece": 0.09
    },
    "banking77": {
      "n": 200,
      "accuracy": 0.835,
      "ece": 0.0581
    },
    "boolq": {
      "n": 200,
      "accuracy": 0.585,
      "ece": 0.1211
    },
    "emotion": {
      "n": 200,
      "accuracy": 0.89,
      "ece": 0.0619
    },
    "hate": {
      "n": 200,
      "accuracy": 0.63,
      "ece": 0.187
    },
    "imdb": {
      "n": 200,
      "accuracy": 0.99,
      "ece": 0.0149
    },
    "massive": {
      "n": 200,
      "accuracy": 0.785,
      "ece": 0.1021
    },
    "mnli": {
      "n": 200,
      "accuracy": 0.5,
      "ece": 0.0774
    },
    "sst2": {
      "n": 200,
      "accuracy": 0.775,
      "ece": 0.1357
    },
    "yelp": {
      "n": 200,
      "accuracy": 0.505,
      "ece": 0.1396
    }
  }
}