{
  "overall": {
    "count": 764,
    "accuracy": 0.8416230366492147,
    "brier": 0.228823902196938,
    "brier_soft": 0.228823902196938,
    "nll": 0.4303391753036349,
    "nll_soft": 0.4303391753036349,
    "ece": 0.041427708936302335,
    "mean_confidence": 0.8759641062345186,
    "confident_error_rate": 0.03795811518324607,
    "score_mae": 0.08815105918628001
  },
  "by_type": {
    "bool": {
      "count": 280,
      "accuracy": 0.85,
      "brier": 0.20686426156907256,
      "brier_soft": 0.20686426156907256,
      "nll": 0.32621192520165093,
      "nll_soft": 0.32621192520165093,
      "ece": 0.06418264691914997,
      "mean_confidence": 0.9034955801491882,
      "confident_error_rate": 0.039285714285714285
    },
    "choice": {
      "count": 444,
      "accuracy": 0.8265765765765766,
      "brier": 0.2590440644829713,
      "brier_soft": 0.2590440644829713,
      "nll": 0.5251947192633012,
      "nll_soft": 0.5251947192633012,
      "ece": 0.04839628894718921,
      "mean_confidence": 0.854881037572032,
      "confident_error_rate": 0.04054054054054054
    },
    "score": {
      "count": 40,
      "accuracy": 0.95,
      "brier": 0.04709758521702534,
      "brier_soft": 0.04709758521702534,
      "nll": 0.10633338806522816,
      "nll_soft": 0.10633338806522816,
      "ece": 0.083569772392317,
      "mean_confidence": 0.9172658509854346,
      "confident_error_rate": 0.0,
      "score_mae": 0.08815105918628001
    }
  },
  "by_source": {
    "composition_held_and_or": {
      "count": 32,
      "accuracy": 1.0,
      "brier": 0.0024598475826333784,
      "brier_soft": 0.0024598475826333784,
      "nll": 0.012256508487456694,
      "nll_soft": 0.012256508487456694,
      "ece": 0.011590125121385776,
      "mean_confidence": 0.9884098748786143,
      "confident_error_rate": 0.0
    },
    "composition_held_conditional": {
      "count": 32,
      "accuracy": 0.8125,
      "brier": 0.34715965230721574,
      "brier_soft": 0.34715965230721574,
      "nll": 0.6216016756942785,
      "nll_soft": 0.6216016756942785,
      "ece": 0.17664373260319421,
      "mean_confidence": 0.9769288116292272,
      "confident_error_rate": 0.1875
    },
    "composition_held_or_not": {
      "count": 32,
      "accuracy": 0.90625,
      "brier": 0.168494398454657,
      "brier_soft": 0.168494398454657,
      "nll": 0.27925353580760964,
      "nll_soft": 0.27925353580760964,
      "ece": 0.08669154028980341,
      "mean_confidence": 0.9929415402898034,
      "confident_error_rate": 0.09375
    },
    "contrastive_authorization": {
      "count": 40,
      "accuracy": 1.0,
      "brier": 1.9275568936614584e-05,
      "brier_soft": 1.9275568936614584e-05,
      "nll": 0.0030790094590994214,
      "nll_soft": 0.0030790094590994214,
      "ece": 0.0030741802684094433,
      "mean_confidence": 0.9969258197315906,
      "confident_error_rate": 0.0
    },
    "contrastive_deadline": {
      "count": 40,
      "accuracy": 0.95,
      "brier": 0.04709758521702534,
      "brier_soft": 0.04709758521702534,
      "nll": 0.10633338806522816,
      "nll_soft": 0.10633338806522816,
      "ece": 0.083569772392317,
      "mean_confidence": 0.9172658509854346,
      "confident_error_rate": 0.0,
      "score_mae": 0.08815105918628001
    },
    "emotion": {
      "count": 116,
      "accuracy": 0.646551724137931,
      "brier": 0.47781526614581615,
      "brier_soft": 0.47781526614581615,
      "nll": 0.919468529423018,
      "nll_soft": 0.919468529423018,
      "ece": 0.07703300800806251,
      "mean_confidence": 0.599839918781091,
      "confident_error_rate": 0.008620689655172414
    },
    "mmlu": {
      "count": 116,
      "accuracy": 0.7844827586206896,
      "brier": 0.33259283968426123,
      "brier_soft": 0.33259283968426123,
      "nll": 0.7725269861278372,
      "nll_soft": 0.7725269861278372,
      "ece": 0.11853164362557794,
      "mean_confidence": 0.8894832273685209,
      "confident_error_rate": 0.06896551724137931
    },
    "paws": {
      "count": 80,
      "accuracy": 0.775,
      "brier": 0.3309925042360514,
      "brier_soft": 0.3309925042360514,
      "nll": 0.541684319155395,
      "nll_soft": 0.541684319155395,
      "ece": 0.15122859544283523,
      "mean_confidence": 0.904908261708847,
      "confident_error_rate": 0.1125
    },
    "qnli": {
      "count": 80,
      "accuracy": 0.9625,
      "brier": 0.06201026023343834,
      "brier_soft": 0.06201026023343834,
      "nll": 0.11861526966281896,
      "nll_soft": 0.11861526966281896,
      "ece": 0.0549367479773906,
      "mean_confidence": 0.9167776334313775,
      "confident_error_rate": 0.0
    },
    "sciq": {
      "count": 116,
      "accuracy": 0.9827586206896551,
      "brier": 0.038177410406259445,
      "brier_soft": 0.038177410406259445,
      "nll": 0.06633965956334043,
      "nll_soft": 0.06633965956334043,
      "ece": 0.03355711460065191,
      "mean_confidence": 0.9667304178543323,
      "confident_error_rate": 0.0
    },
    "tweet_offensive": {
      "count": 80,
      "accuracy": 0.7375,
      "brier": 0.331012513237796,
      "brier_soft": 0.331012513237796,
      "nll": 0.4799026446580147,
      "nll_soft": 0.4799026446580147,
      "ece": 0.15083210595826038,
      "mean_confidence": 0.8420857255161387,
      "confident_error_rate": 0.025
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 1,
      "confidence": 0.2863755673514975,
      "accuracy": 0.0
    },
    {
      "bin": "0.3-0.4",
      "count": 12,
      "confidence": 0.3548456743596838,
      "accuracy": 0.5833333333333334
    },
    {
      "bin": "0.4-0.5",
      "count": 30,
      "confidence": 0.4537790778486413,
      "accuracy": 0.4666666666666667
    },
    {
      "bin": "0.5-0.6",
      "count": 53,
      "confidence": 0.5532438982939633,
      "accuracy": 0.5283018867924528
    },
    {
      "bin": "0.6-0.7",
      "count": 40,
      "confidence": 0.6630960879374065,
      "accuracy": 0.575
    },
    {
      "bin": "0.7-0.8",
      "count": 50,
      "confidence": 0.749056549882067,
      "accuracy": 0.7
    },
    {
      "bin": "0.8-0.9",
      "count": 59,
      "confidence": 0.8574375075804922,
      "accuracy": 0.7796610169491526
    },
    {
      "bin": "0.9-1.0",
      "count": 519,
      "confidence": 0.9772471495175656,
      "accuracy": 0.9441233140655106
    }
  ],
  "seconds": 40.363425970077515,
  "model": "runs/assay-27b",
  "data": "data/suites/kev-transfer-v4-dev.jsonl"
}