{
  "overall": {
    "count": 2020,
    "accuracy": 0.8415841584158416,
    "brier": 0.2199661750237191,
    "brier_soft": 0.2199661750237191,
    "nll": 0.39249676153765684,
    "nll_soft": 0.39249676153765684,
    "ece": 0.0305830851696879,
    "mean_confidence": 0.8462516664578095,
    "confident_error_rate": 0.01485148514851485,
    "score_mae": 0.734591401760134
  },
  "by_type": {
    "bool": {
      "count": 800,
      "accuracy": 0.855,
      "brier": 0.20325044236015188,
      "brier_soft": 0.20325044236015188,
      "nll": 0.31629045427560526,
      "nll_soft": 0.31629045427560526,
      "ece": 0.03714657739278206,
      "mean_confidence": 0.8853094374262506,
      "confident_error_rate": 0.0225
    },
    "choice": {
      "count": 1020,
      "accuracy": 0.8676470588235294,
      "brier": 0.18443775278710198,
      "brier_soft": 0.18443775278710198,
      "nll": 0.34555459237851266,
      "nll_soft": 0.34555459237851266,
      "ece": 0.017727219065961402,
      "mean_confidence": 0.8679584693166184,
      "confident_error_rate": 0.011764705882352941
    },
    "score": {
      "count": 200,
      "accuracy": 0.655,
      "brier": 0.4680240590847353,
      "brier_soft": 0.4680240590847353,
      "nll": 0.9367270532974982,
      "nll_soft": 0.9367270532974982,
      "ece": 0.14919123973044776,
      "mean_confidence": 0.5793158880041201,
      "confident_error_rate": 0.0,
      "score_mae": 0.734591401760134
    }
  },
  "by_source": {
    "app_reviews": {
      "count": 200,
      "accuracy": 0.655,
      "brier": 0.4680240590847353,
      "brier_soft": 0.4680240590847353,
      "nll": 0.9367270532974982,
      "nll_soft": 0.9367270532974982,
      "ece": 0.14919123973044776,
      "mean_confidence": 0.5793158880041201,
      "confident_error_rate": 0.0,
      "score_mae": 0.734591401760134
    },
    "bbc_news": {
      "count": 200,
      "accuracy": 0.99,
      "brier": 0.02485312024968503,
      "brier_soft": 0.02485312024968503,
      "nll": 0.05229038920662954,
      "nll_soft": 0.05229038920662954,
      "ece": 0.03289109205025749,
      "mean_confidence": 0.9612872383267218,
      "confident_error_rate": 0.0
    },
    "copa": {
      "count": 100,
      "accuracy": 0.99,
      "brier": 0.014510678299897408,
      "brier_soft": 0.014510678299897408,
      "nll": 0.02477687623355849,
      "nll_soft": 0.02477687623355849,
      "ece": 0.016801298912001265,
      "mean_confidence": 0.9887446982443209,
      "confident_error_rate": 0.0
    },
    "dream": {
      "count": 200,
      "accuracy": 0.985,
      "brier": 0.02921864351778212,
      "brier_soft": 0.02921864351778212,
      "nll": 0.08814026262617411,
      "nll_soft": 0.08814026262617411,
      "ece": 0.01292344301454983,
      "mean_confidence": 0.9882869129162648,
      "confident_error_rate": 0.01
    },
    "ethos": {
      "count": 200,
      "accuracy": 0.785,
      "brier": 0.28935880012627835,
      "brier_soft": 0.28935880012627835,
      "nll": 0.43330864098692673,
      "nll_soft": 0.43330864098692673,
      "ece": 0.0902209003025014,
      "mean_confidence": 0.8620058140147207,
      "confident_error_rate": 0.04
    },
    "hh_rlhf": {
      "count": 200,
      "accuracy": 0.645,
      "brier": 0.44173160731312494,
      "brier_soft": 0.44173160731312494,
      "nll": 0.6311810092807139,
      "nll_soft": 0.6311810092807139,
      "ece": 0.03958897205020696,
      "mean_confidence": 0.6160584099275223,
      "confident_error_rate": 0.0
    },
    "medical_questions_pairs": {
      "count": 200,
      "accuracy": 0.895,
      "brier": 0.13780739465855052,
      "brier_soft": 0.13780739465855052,
      "nll": 0.21259324888263684,
      "nll_soft": 0.21259324888263684,
      "ece": 0.051844561382003836,
      "mean_confidence": 0.9072524506750079,
      "confident_error_rate": 0.005
    },
    "scitail": {
      "count": 200,
      "accuracy": 0.91,
      "brier": 0.13208857504118351,
      "brier_soft": 0.13208857504118351,
      "nll": 0.22125867806726154,
      "nll_soft": 0.22125867806726154,
      "ece": 0.04096846269343061,
      "mean_confidence": 0.9329582954663271,
      "confident_error_rate": 0.025
    },
    "stance_climate": {
      "count": 120,
      "accuracy": 0.8833333333333333,
      "brier": 0.1806405349295063,
      "brier_soft": 0.1806405349295063,
      "nll": 0.35196629973303767,
      "nll_soft": 0.35196629973303767,
      "ece": 0.10556056521146755,
      "mean_confidence": 0.8141962073468899,
      "confident_error_rate": 0.0
    },
    "truthful_qa": {
      "count": 200,
      "accuracy": 0.78,
      "brier": 0.3291895080259756,
      "brier_soft": 0.3291895080259756,
      "nll": 0.7671485420602951,
      "nll_soft": 0.7671485420602951,
      "ece": 0.09806555881395083,
      "mean_confidence": 0.8780655588139509,
      "confident_error_rate": 0.05
    },
    "tweet_irony": {
      "count": 200,
      "accuracy": 0.83,
      "brier": 0.2537469996145951,
      "brier_soft": 0.2537469996145951,
      "nll": 0.39800124916559576,
      "nll_soft": 0.39800124916559576,
      "ece": 0.06288414121493499,
      "mean_confidence": 0.8390211895489462,
      "confident_error_rate": 0.02
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 4,
      "confidence": 0.27240217902249636,
      "accuracy": 0.0
    },
    {
      "bin": "0.3-0.4",
      "count": 32,
      "confidence": 0.3696628771674506,
      "accuracy": 0.46875
    },
    {
      "bin": "0.4-0.5",
      "count": 92,
      "confidence": 0.4695831445360151,
      "accuracy": 0.391304347826087
    },
    {
      "bin": "0.5-0.6",
      "count": 202,
      "confidence": 0.5557999645214033,
      "accuracy": 0.6138613861386139
    },
    {
      "bin": "0.6-0.7",
      "count": 147,
      "confidence": 0.651499808775641,
      "accuracy": 0.6938775510204082
    },
    {
      "bin": "0.7-0.8",
      "count": 176,
      "confidence": 0.749253377021942,
      "accuracy": 0.6647727272727273
    },
    {
      "bin": "0.8-0.9",
      "count": 200,
      "confidence": 0.8559249921875981,
      "accuracy": 0.845
    },
    {
      "bin": "0.9-1.0",
      "count": 1167,
      "confidence": 0.9787594161484913,
      "accuracy": 0.974293059125964
    }
  ],
  "seconds": 123.55612516403198,
  "model": "runs/assay-27b",
  "data": "data/v2/holdout.jsonl"
}