{
  "overall": {
    "count": 764,
    "accuracy": 0.8416230366492147,
    "brier": 0.23366786332576134,
    "brier_soft": 0.23366786332576134,
    "nll": 0.45999593150161044,
    "nll_soft": 0.45999593150161044,
    "ece": 0.06523326879131716,
    "mean_confidence": 0.9013523079735777,
    "confident_error_rate": 0.04712041884816754,
    "score_mae": 0.06568647111125461
  },
  "by_type": {
    "bool": {
      "count": 280,
      "accuracy": 0.85,
      "brier": 0.21610131477479474,
      "brier_soft": 0.21610131477479474,
      "nll": 0.3515884562588345,
      "nll_soft": 0.3515884562588345,
      "ece": 0.08110998832325675,
      "mean_confidence": 0.9250817518080554,
      "confident_error_rate": 0.05714285714285714
    },
    "choice": {
      "count": 444,
      "accuracy": 0.8265765765765766,
      "brier": 0.2619705373448098,
      "brier_soft": 0.2619705373448098,
      "nll": 0.5623854204187092,
      "nll_soft": 0.5623854204187092,
      "ece": 0.07234439122776523,
      "mean_confidence": 0.8829180473388407,
      "confident_error_rate": 0.04504504504504504
    },
    "score": {
      "count": 40,
      "accuracy": 0.95,
      "brier": 0.04247402157108923,
      "brier_soft": 0.04247402157108923,
      "nll": 0.08232493122124444,
      "nll_soft": 0.08232493122124444,
      "ece": 0.06295530532303786,
      "mean_confidence": 0.9398664941778165,
      "confident_error_rate": 0.0,
      "score_mae": 0.06568647111125461
    }
  },
  "by_source": {
    "composition_held_and_or": {
      "count": 32,
      "accuracy": 1.0,
      "brier": 0.0010265396372586473,
      "brier_soft": 0.0010265396372586473,
      "nll": 0.00744750174425,
      "nll_soft": 0.00744750174425,
      "ece": 0.007177238579851596,
      "mean_confidence": 0.9928227614201484,
      "confident_error_rate": 0.0
    },
    "composition_held_conditional": {
      "count": 32,
      "accuracy": 0.8125,
      "brier": 0.3609123301012935,
      "brier_soft": 0.3609123301012935,
      "nll": 0.750269028687234,
      "nll_soft": 0.750269028687234,
      "ece": 0.17994145932455827,
      "mean_confidence": 0.9882197907318171,
      "confident_error_rate": 0.1875
    },
    "composition_held_or_not": {
      "count": 32,
      "accuracy": 0.90625,
      "brier": 0.17745481693255763,
      "brier_soft": 0.17745481693255763,
      "nll": 0.33929718078098925,
      "nll_soft": 0.33929718078098925,
      "ece": 0.0904297754653719,
      "mean_confidence": 0.9966797754653719,
      "confident_error_rate": 0.09375
    },
    "contrastive_authorization": {
      "count": 40,
      "accuracy": 1.0,
      "brier": 1.301873731350463e-06,
      "brier_soft": 1.301873731350463e-06,
      "nll": 0.0007950641143637151,
      "nll_soft": 0.0007950641143637151,
      "ece": 0.0007947384626161291,
      "mean_confidence": 0.9992052615373839,
      "confident_error_rate": 0.0
    },
    "contrastive_deadline": {
      "count": 40,
      "accuracy": 0.95,
      "brier": 0.04247402157108923,
      "brier_soft": 0.04247402157108923,
      "nll": 0.08232493122124444,
      "nll_soft": 0.08232493122124444,
      "ece": 0.06295530532303786,
      "mean_confidence": 0.9398664941778165,
      "confident_error_rate": 0.0,
      "score_mae": 0.06568647111125461
    },
    "emotion": {
      "count": 116,
      "accuracy": 0.646551724137931,
      "brier": 0.47142674905119664,
      "brier_soft": 0.47142674905119664,
      "nll": 0.9009041198087586,
      "nll_soft": 0.9009041198087586,
      "ece": 0.0875629354087716,
      "mean_confidence": 0.6661347203870895,
      "confident_error_rate": 0.017241379310344827
    },
    "mmlu": {
      "count": 116,
      "accuracy": 0.7844827586206896,
      "brier": 0.34407690962026555,
      "brier_soft": 0.34407690962026555,
      "nll": 0.8865278702283488,
      "nll_soft": 0.8865278702283488,
      "ece": 0.14835926317316953,
      "mean_confidence": 0.917652102352729,
      "confident_error_rate": 0.07758620689655173
    },
    "paws": {
      "count": 80,
      "accuracy": 0.775,
      "brier": 0.34762263713242636,
      "brier_soft": 0.34762263713242636,
      "nll": 0.6147711331747864,
      "nll_soft": 0.6147711331747864,
      "ece": 0.15929430289975277,
      "mean_confidence": 0.9307858966627347,
      "confident_error_rate": 0.15
    },
    "qnli": {
      "count": 80,
      "accuracy": 0.9625,
      "brier": 0.05830143324049297,
      "brier_soft": 0.05830143324049297,
      "nll": 0.10144160569398128,
      "nll_soft": 0.10144160569398128,
      "ece": 0.05166224596877384,
      "mean_confidence": 0.9363518495172769,
      "confident_error_rate": 0.0
    },
    "sciq": {
      "count": 116,
      "accuracy": 0.9827586206896551,
      "brier": 0.03841289829077971,
      "brier_soft": 0.03841289829077971,
      "nll": 0.06252221605864929,
      "nll_soft": 0.06252221605864929,
      "ece": 0.03045631037515725,
      "mean_confidence": 0.9742174749730302,
      "confident_error_rate": 0.0
    },
    "tweet_offensive": {
      "count": 80,
      "accuracy": 0.7375,
      "brier": 0.3504298804019966,
      "brier_soft": 0.3504298804019966,
      "nll": 0.5139493259799714,
      "nll_soft": 0.5139493259799714,
      "ece": 0.15698528550077231,
      "mean_confidence": 0.8710457543794903,
      "confident_error_rate": 0.05
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.3-0.4",
      "count": 9,
      "confidence": 0.3614360119116289,
      "accuracy": 0.5555555555555556
    },
    {
      "bin": "0.4-0.5",
      "count": 19,
      "confidence": 0.4628657382665659,
      "accuracy": 0.47368421052631576
    },
    {
      "bin": "0.5-0.6",
      "count": 39,
      "confidence": 0.5494683944332894,
      "accuracy": 0.46153846153846156
    },
    {
      "bin": "0.6-0.7",
      "count": 36,
      "confidence": 0.6379403082210656,
      "accuracy": 0.5555555555555556
    },
    {
      "bin": "0.7-0.8",
      "count": 48,
      "confidence": 0.7524977237611029,
      "accuracy": 0.6041666666666666
    },
    {
      "bin": "0.8-0.9",
      "count": 61,
      "confidence": 0.855432037802958,
      "accuracy": 0.7540983606557377
    },
    {
      "bin": "0.9-1.0",
      "count": 552,
      "confidence": 0.9853069323046632,
      "accuracy": 0.9347826086956522
    }
  ],
  "seconds": 40.56191968917847,
  "model": "runs/assay-27b",
  "data": "data/suites/kev-transfer-v4-dev.jsonl"
}