{
  "overall": {
    "count": 2020,
    "accuracy": 0.7039603960396039,
    "brier": 0.40174229812709567,
    "brier_soft": 0.40174229812709567,
    "nll": 0.731639175263139,
    "nll_soft": 0.731639175263139,
    "ece": 0.05181649254555068,
    "mean_confidence": 0.7544755276026817,
    "confident_error_rate": 0.030693069306930693,
    "score_mae": 0.9427933739641703
  },
  "by_type": {
    "bool": {
      "count": 800,
      "accuracy": 0.76875,
      "brier": 0.3384939203700049,
      "brier_soft": 0.3384939203700049,
      "nll": 0.5332047769433746,
      "nll_soft": 0.5332047769433746,
      "ece": 0.06804100590180462,
      "mean_confidence": 0.7969667051236982,
      "confident_error_rate": 0.0525
    },
    "choice": {
      "count": 1020,
      "accuracy": 0.7029411764705882,
      "brier": 0.4057562984937154,
      "brier_soft": 0.4057562984937154,
      "nll": 0.7789359708264965,
      "nll_soft": 0.7789359708264965,
      "ece": 0.06983641662399524,
      "mean_confidence": 0.7703221637894546,
      "confident_error_rate": 0.0196078431372549
    },
    "score": {
      "count": 200,
      "accuracy": 0.45,
      "brier": 0.6342644072856982,
      "brier_soft": 0.6342644072856982,
      "nll": 1.2841631111690737,
      "nll_soft": 1.2841631111690737,
      "ece": 0.18082671237600167,
      "mean_confidence": 0.5036929729660733,
      "confident_error_rate": 0.0,
      "score_mae": 0.9427933739641703
    }
  },
  "by_source": {
    "app_reviews": {
      "count": 200,
      "accuracy": 0.45,
      "brier": 0.6342644072856982,
      "brier_soft": 0.6342644072856982,
      "nll": 1.2841631111690737,
      "nll_soft": 1.2841631111690737,
      "ece": 0.18082671237600167,
      "mean_confidence": 0.5036929729660733,
      "confident_error_rate": 0.0,
      "score_mae": 0.9427933739641703
    },
    "bbc_news": {
      "count": 200,
      "accuracy": 0.97,
      "brier": 0.045893770562849864,
      "brier_soft": 0.045893770562849864,
      "nll": 0.0785250898948866,
      "nll_soft": 0.0785250898948866,
      "ece": 0.028721116555098777,
      "mean_confidence": 0.9721954159172467,
      "confident_error_rate": 0.005
    },
    "copa": {
      "count": 100,
      "accuracy": 0.8,
      "brier": 0.2831331021745002,
      "brier_soft": 0.2831331021745002,
      "nll": 0.44168065160584385,
      "nll_soft": 0.44168065160584385,
      "ece": 0.05239059827716305,
      "mean_confidence": 0.8417148010032206,
      "confident_error_rate": 0.03
    },
    "dream": {
      "count": 200,
      "accuracy": 0.785,
      "brier": 0.3129441102941803,
      "brier_soft": 0.3129441102941803,
      "nll": 0.5791733107872337,
      "nll_soft": 0.5791733107872337,
      "ece": 0.07127968470427938,
      "mean_confidence": 0.8321483632005751,
      "confident_error_rate": 0.035
    },
    "ethos": {
      "count": 200,
      "accuracy": 0.71,
      "brier": 0.45528528822983405,
      "brier_soft": 0.45528528822983405,
      "nll": 0.7399378586910943,
      "nll_soft": 0.7399378586910943,
      "ece": 0.19049043617901848,
      "mean_confidence": 0.8767009685369425,
      "confident_error_rate": 0.145
    },
    "hh_rlhf": {
      "count": 200,
      "accuracy": 0.59,
      "brier": 0.48896179038631715,
      "brier_soft": 0.48896179038631715,
      "nll": 0.6807258118477102,
      "nll_soft": 0.6807258118477102,
      "ece": 0.0844235941409816,
      "mean_confidence": 0.6196837265263665,
      "confident_error_rate": 0.0
    },
    "medical_questions_pairs": {
      "count": 200,
      "accuracy": 0.835,
      "brier": 0.24905794448748267,
      "brier_soft": 0.24905794448748267,
      "nll": 0.39622704921646745,
      "nll_soft": 0.39622704921646745,
      "ece": 0.07978193072545098,
      "mean_confidence": 0.7695873821349596,
      "confident_error_rate": 0.005
    },
    "scitail": {
      "count": 200,
      "accuracy": 0.77,
      "brier": 0.3358466241116217,
      "brier_soft": 0.3358466241116217,
      "nll": 0.5156791331679232,
      "nll_soft": 0.5156791331679232,
      "ece": 0.08103809561037315,
      "mean_confidence": 0.8425311392221119,
      "confident_error_rate": 0.06
    },
    "stance_climate": {
      "count": 120,
      "accuracy": 0.7333333333333333,
      "brier": 0.3524765259996208,
      "brier_soft": 0.3524765259996208,
      "nll": 0.606011712096946,
      "nll_soft": 0.606011712096946,
      "ece": 0.0796837585191846,
      "mean_confidence": 0.7246747508084542,
      "confident_error_rate": 0.0
    },
    "truthful_qa": {
      "count": 200,
      "accuracy": 0.4,
      "brier": 0.8685049843875788,
      "brier_soft": 0.8685049843875788,
      "nll": 2.0497018856242115,
      "nll_soft": 2.0497018856242115,
      "ece": 0.25907005087908236,
      "mean_confidence": 0.6489532786953479,
      "confident_error_rate": 0.045
    },
    "tweet_irony": {
      "count": 200,
      "accuracy": 0.76,
      "brier": 0.3137858246510813,
      "brier_soft": 0.3137858246510813,
      "nll": 0.4809750666980137,
      "nll_soft": 0.4809750666980137,
      "ece": 0.11329888688561211,
      "mean_confidence": 0.6990473306007787,
      "confident_error_rate": 0.0
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 5,
      "confidence": 0.2859458344743007,
      "accuracy": 0.2
    },
    {
      "bin": "0.3-0.4",
      "count": 63,
      "confidence": 0.3676116869076494,
      "accuracy": 0.2857142857142857
    },
    {
      "bin": "0.4-0.5",
      "count": 146,
      "confidence": 0.458595853179923,
      "accuracy": 0.3082191780821918
    },
    {
      "bin": "0.5-0.6",
      "count": 340,
      "confidence": 0.5544766223229719,
      "accuracy": 0.55
    },
    {
      "bin": "0.6-0.7",
      "count": 248,
      "confidence": 0.6495730517363174,
      "accuracy": 0.5967741935483871
    },
    {
      "bin": "0.7-0.8",
      "count": 291,
      "confidence": 0.7486289220577544,
      "accuracy": 0.7353951890034365
    },
    {
      "bin": "0.8-0.9",
      "count": 270,
      "confidence": 0.8542787756118871,
      "accuracy": 0.7925925925925926
    },
    {
      "bin": "0.9-1.0",
      "count": 657,
      "confidence": 0.9655614179469735,
      "accuracy": 0.9056316590563166
    }
  ],
  "seconds": 4.797023773193359,
  "model": "runs/assay-0.6b-v4",
  "data": "data/v4/holdout.jsonl"
}