{
  "overall": {
    "count": 2020,
    "accuracy": 0.8415841584158416,
    "brier": 0.220799111831009,
    "brier_soft": 0.220799111831009,
    "nll": 0.39014477942134185,
    "nll_soft": 0.39014477942134185,
    "ece": 0.03993673549801924,
    "mean_confidence": 0.8201206323505023,
    "confident_error_rate": 0.0049504950495049506,
    "score_mae": 0.7803430055728247
  },
  "by_type": {
    "bool": {
      "count": 800,
      "accuracy": 0.855,
      "brier": 0.1998546341034784,
      "brier_soft": 0.1998546341034784,
      "nll": 0.3135695750039896,
      "nll_soft": 0.3135695750039896,
      "ece": 0.041050193204409134,
      "mean_confidence": 0.8587700991805063,
      "confident_error_rate": 0.00875
    },
    "choice": {
      "count": 1020,
      "accuracy": 0.8676470588235294,
      "brier": 0.18523493798833834,
      "brier_soft": 0.18523493798833834,
      "nll": 0.33966287329231043,
      "nll_soft": 0.33966287329231043,
      "ece": 0.03498720217459626,
      "mean_confidence": 0.8462023584334264,
      "confident_error_rate": 0.0029411764705882353
    },
    "score": {
      "count": 200,
      "accuracy": 0.655,
      "brier": 0.4859543093387518,
      "brier_soft": 0.4859543093387518,
      "nll": 0.9539033183488113,
      "nll_soft": 0.9539033183488113,
      "ece": 0.19196724371336882,
      "mean_confidence": 0.5325059620075736,
      "confident_error_rate": 0.0,
      "score_mae": 0.7803430055728247
    }
  },
  "by_source": {
    "app_reviews": {
      "count": 200,
      "accuracy": 0.655,
      "brier": 0.4859543093387518,
      "brier_soft": 0.4859543093387518,
      "nll": 0.9539033183488113,
      "nll_soft": 0.9539033183488113,
      "ece": 0.19196724371336882,
      "mean_confidence": 0.5325059620075736,
      "confident_error_rate": 0.0,
      "score_mae": 0.7803430055728247
    },
    "bbc_news": {
      "count": 200,
      "accuracy": 0.99,
      "brier": 0.028912731212050385,
      "brier_soft": 0.028912731212050385,
      "nll": 0.07044915532346424,
      "nll_soft": 0.07044915532346424,
      "ece": 0.051107039480480426,
      "mean_confidence": 0.9438197171243428,
      "confident_error_rate": 0.0
    },
    "copa": {
      "count": 100,
      "accuracy": 0.99,
      "brier": 0.01453758967734365,
      "brier_soft": 0.01453758967734365,
      "nll": 0.029948603826919724,
      "nll_soft": 0.029948603826919724,
      "ece": 0.022917495301317405,
      "mean_confidence": 0.9817517021787172,
      "confident_error_rate": 0.0
    },
    "dream": {
      "count": 200,
      "accuracy": 0.985,
      "brier": 0.029222745427828708,
      "brier_soft": 0.029222745427828708,
      "nll": 0.08134387315614394,
      "nll_soft": 0.08134387315614394,
      "ece": 0.012822483843521316,
      "mean_confidence": 0.9815427931884518,
      "confident_error_rate": 0.005
    },
    "ethos": {
      "count": 200,
      "accuracy": 0.785,
      "brier": 0.2789473107906072,
      "brier_soft": 0.2789473107906072,
      "nll": 0.4138867544823541,
      "nll_soft": 0.4138867544823541,
      "ece": 0.079336313886892,
      "mean_confidence": 0.8364448087371015,
      "confident_error_rate": 0.02
    },
    "hh_rlhf": {
      "count": 200,
      "accuracy": 0.645,
      "brier": 0.4451497617751254,
      "brier_soft": 0.4451497617751254,
      "nll": 0.6353554945088988,
      "nll_soft": 0.6353554945088988,
      "ece": 0.0625498708907104,
      "mean_confidence": 0.596423357696465,
      "confident_error_rate": 0.0
    },
    "medical_questions_pairs": {
      "count": 200,
      "accuracy": 0.895,
      "brier": 0.13672426710472746,
      "brier_soft": 0.13672426710472746,
      "nll": 0.22289993483247128,
      "nll_soft": 0.22289993483247128,
      "ece": 0.06834734948342853,
      "mean_confidence": 0.8813892348533718,
      "confident_error_rate": 0.0
    },
    "scitail": {
      "count": 200,
      "accuracy": 0.91,
      "brier": 0.1300524297562204,
      "brier_soft": 0.1300524297562204,
      "nll": 0.2165815487842471,
      "nll_soft": 0.2165815487842471,
      "ece": 0.02789948287264498,
      "mean_confidence": 0.9132580009454067,
      "confident_error_rate": 0.015
    },
    "stance_climate": {
      "count": 120,
      "accuracy": 0.8833333333333333,
      "brier": 0.19515038307691712,
      "brier_soft": 0.19515038307691712,
      "nll": 0.3915492223860603,
      "nll_soft": 0.3915492223860603,
      "ece": 0.15516476801357199,
      "mean_confidence": 0.7656094020870176,
      "confident_error_rate": 0.0
    },
    "truthful_qa": {
      "count": 200,
      "accuracy": 0.78,
      "brier": 0.317053920640699,
      "brier_soft": 0.317053920640699,
      "nll": 0.6952282954571797,
      "nll_soft": 0.6952282954571797,
      "ece": 0.09467769715035201,
      "mean_confidence": 0.8436046676596461,
      "confident_error_rate": 0.01
    },
    "tweet_irony": {
      "count": 200,
      "accuracy": 0.83,
      "brier": 0.25369452876235854,
      "brier_soft": 0.25369452876235854,
      "nll": 0.4009100619168858,
      "nll_soft": 0.4009100619168858,
      "ece": 0.05814862800170818,
      "mean_confidence": 0.8039883521861452,
      "confident_error_rate": 0.0
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 7,
      "confidence": 0.2744105925520648,
      "accuracy": 0.14285714285714285
    },
    {
      "bin": "0.3-0.4",
      "count": 43,
      "confidence": 0.35869775450764224,
      "accuracy": 0.4418604651162791
    },
    {
      "bin": "0.4-0.5",
      "count": 104,
      "confidence": 0.4636549146792239,
      "accuracy": 0.4423076923076923
    },
    {
      "bin": "0.5-0.6",
      "count": 236,
      "confidence": 0.5565186483404105,
      "accuracy": 0.6483050847457628
    },
    {
      "bin": "0.6-0.7",
      "count": 172,
      "confidence": 0.6513175851364689,
      "accuracy": 0.6627906976744186
    },
    {
      "bin": "0.7-0.8",
      "count": 181,
      "confidence": 0.7479925616462004,
      "accuracy": 0.7679558011049724
    },
    {
      "bin": "0.8-0.9",
      "count": 261,
      "confidence": 0.8562325391696354,
      "accuracy": 0.8505747126436781
    },
    {
      "bin": "0.9-1.0",
      "count": 1016,
      "confidence": 0.97327787066598,
      "accuracy": 0.9901574803149606
    }
  ],
  "seconds": 123.60580730438232,
  "model": "runs/assay-27b",
  "data": "data/v2/holdout.jsonl"
}