{
  "overall": {
    "count": 764,
    "accuracy": 0.6361256544502618,
    "brier": 0.49852247078751044,
    "brier_soft": 0.49852247078751044,
    "nll": 0.850876258257955,
    "nll_soft": 0.850876258257955,
    "ece": 0.12407712725654348,
    "mean_confidence": 0.7596816331834112,
    "confident_error_rate": 0.04450261780104712,
    "score_mae": 0.2973544131495921
  },
  "by_type": {
    "bool": {
      "count": 280,
      "accuracy": 0.6642857142857143,
      "brier": 0.49259952159686327,
      "brier_soft": 0.49259952159686327,
      "nll": 0.7325980517166166,
      "nll_soft": 0.7325980517166166,
      "ece": 0.16049088636076536,
      "mean_confidence": 0.8200732964531596,
      "confident_error_rate": 0.07857142857142857
    },
    "choice": {
      "count": 444,
      "accuracy": 0.5923423423423423,
      "brier": 0.5278777983765222,
      "brier_soft": 0.5278777983765222,
      "nll": 0.9644794938837881,
      "nll_soft": 0.9644794938837881,
      "ece": 0.1394493857168115,
      "mean_confidence": 0.72885617509156,
      "confident_error_rate": 0.02702702702702703
    },
    "score": {
      "count": 40,
      "accuracy": 0.925,
      "brier": 0.21413897888400896,
      "brier_soft": 0.21413897888400896,
      "nll": 0.41782778860057623,
      "nll_soft": 0.41782778860057623,
      "ece": 0.268528126028744,
      "mean_confidence": 0.6791025751147204,
      "confident_error_rate": 0.0,
      "score_mae": 0.2973544131495921
    }
  },
  "by_source": {
    "composition_held_and_or": {
      "count": 32,
      "accuracy": 0.5,
      "brier": 0.667279448814107,
      "brier_soft": 0.667279448814107,
      "nll": 0.907676834819235,
      "nll_soft": 0.907676834819235,
      "ece": 0.3314446704480186,
      "mean_confidence": 0.7510849772790604,
      "confident_error_rate": 0.0
    },
    "composition_held_conditional": {
      "count": 32,
      "accuracy": 0.625,
      "brier": 0.6157411019594652,
      "brier_soft": 0.6157411019594652,
      "nll": 0.8572283897434516,
      "nll_soft": 0.8572283897434516,
      "ece": 0.34807412831864337,
      "mean_confidence": 0.7421134787354795,
      "confident_error_rate": 0.03125
    },
    "composition_held_or_not": {
      "count": 32,
      "accuracy": 0.5625,
      "brier": 0.4698421720914693,
      "brier_soft": 0.4698421720914693,
      "nll": 0.6583405822725403,
      "nll_soft": 0.6583405822725403,
      "ece": 0.27331303647467814,
      "mean_confidence": 0.7097878951663201,
      "confident_error_rate": 0.0
    },
    "contrastive_authorization": {
      "count": 40,
      "accuracy": 0.5,
      "brier": 0.836510913803445,
      "brier_soft": 0.836510913803445,
      "nll": 1.2750964423387703,
      "nll_soft": 1.2750964423387703,
      "ece": 0.44958833389484043,
      "mean_confidence": 0.9495883338948404,
      "confident_error_rate": 0.325
    },
    "contrastive_deadline": {
      "count": 40,
      "accuracy": 0.925,
      "brier": 0.21413897888400896,
      "brier_soft": 0.21413897888400896,
      "nll": 0.41782778860057623,
      "nll_soft": 0.41782778860057623,
      "ece": 0.268528126028744,
      "mean_confidence": 0.6791025751147204,
      "confident_error_rate": 0.0,
      "score_mae": 0.2973544131495921
    },
    "emotion": {
      "count": 116,
      "accuracy": 0.45689655172413796,
      "brier": 0.7078152200897051,
      "brier_soft": 0.7078152200897051,
      "nll": 1.376784251217549,
      "nll_soft": 1.376784251217549,
      "ece": 0.2352329817213669,
      "mean_confidence": 0.6614128736301557,
      "confident_error_rate": 0.04310344827586207
    },
    "mmlu": {
      "count": 116,
      "accuracy": 0.45689655172413796,
      "brier": 0.6360155342312662,
      "brier_soft": 0.6360155342312662,
      "nll": 1.2028071667685185,
      "nll_soft": 1.2028071667685185,
      "ece": 0.16795157187610193,
      "mean_confidence": 0.6082772615480655,
      "confident_error_rate": 0.008620689655172414
    },
    "paws": {
      "count": 80,
      "accuracy": 0.5875,
      "brier": 0.5475383162511623,
      "brier_soft": 0.5475383162511623,
      "nll": 0.7795850558186694,
      "nll_soft": 0.7795850558186694,
      "ece": 0.266156270517573,
      "mean_confidence": 0.7797887976199636,
      "confident_error_rate": 0.0375
    },
    "qnli": {
      "count": 80,
      "accuracy": 0.775,
      "brier": 0.31303704237048013,
      "brier_soft": 0.31303704237048013,
      "nll": 0.4808854176032183,
      "nll_soft": 0.4808854176032183,
      "ece": 0.10685060871150737,
      "mean_confidence": 0.7574151574083786,
      "confident_error_rate": 0.0
    },
    "sciq": {
      "count": 116,
      "accuracy": 0.8879310344827587,
      "brier": 0.19311868832984344,
      "brier_soft": 0.19311868832984344,
      "nll": 0.4435553877525076,
      "nll_soft": 0.4435553877525076,
      "ece": 0.06831785607105256,
      "mean_confidence": 0.9123493346740643,
      "confident_error_rate": 0.04310344827586207
    },
    "tweet_offensive": {
      "count": 80,
      "accuracy": 0.7125,
      "brier": 0.4452675100656568,
      "brier_soft": 0.4452675100656568,
      "nll": 0.6660744864168855,
      "nll_soft": 0.6660744864168855,
      "ece": 0.17723213008556796,
      "mean_confidence": 0.858258415610296,
      "confident_error_rate": 0.075
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 5,
      "confidence": 0.2889275957069053,
      "accuracy": 0.0
    },
    {
      "bin": "0.3-0.4",
      "count": 28,
      "confidence": 0.35784894162284964,
      "accuracy": 0.17857142857142858
    },
    {
      "bin": "0.4-0.5",
      "count": 54,
      "confidence": 0.46309241565115256,
      "accuracy": 0.42592592592592593
    },
    {
      "bin": "0.5-0.6",
      "count": 87,
      "confidence": 0.5474811452572101,
      "accuracy": 0.5402298850574713
    },
    {
      "bin": "0.6-0.7",
      "count": 85,
      "confidence": 0.6506189554618936,
      "accuracy": 0.5529411764705883
    },
    {
      "bin": "0.7-0.8",
      "count": 126,
      "confidence": 0.7569211789474528,
      "accuracy": 0.6507936507936508
    },
    {
      "bin": "0.8-0.9",
      "count": 161,
      "confidence": 0.8494863628752396,
      "accuracy": 0.6086956521739131
    },
    {
      "bin": "0.9-1.0",
      "count": 218,
      "confidence": 0.9580391061516451,
      "accuracy": 0.8440366972477065
    }
  ],
  "seconds": 1.7002065181732178,
  "model": "runs/assay-0.6b-v4",
  "data": "data/suites/kev-transfer-v4-dev.jsonl"
}