{
  "overall": {
    "count": 2020,
    "accuracy": 0.7039603960396039,
    "brier": 0.39739394579057974,
    "brier_soft": 0.39739394579057974,
    "nll": 0.7045973172316387,
    "nll_soft": 0.7045973172316387,
    "ece": 0.03718212714244176,
    "mean_confidence": 0.7254018976243364,
    "confident_error_rate": 0.019306930693069307,
    "score_mae": 0.9795367383039922
  },
  "by_type": {
    "bool": {
      "count": 800,
      "accuracy": 0.76875,
      "brier": 0.3369000340326616,
      "brier_soft": 0.3369000340326616,
      "nll": 0.5200392508482914,
      "nll_soft": 0.5200392508482914,
      "ece": 0.06895492798043171,
      "mean_confidence": 0.769696578533584,
      "confident_error_rate": 0.03375
    },
    "choice": {
      "count": 1020,
      "accuracy": 0.7029411764705882,
      "brier": 0.39841633259114867,
      "brier_soft": 0.39841633259114867,
      "nll": 0.7394783646578699,
      "nll_soft": 0.7394783646578699,
      "ece": 0.03834348727757606,
      "mean_confidence": 0.7412846637481643,
      "confident_error_rate": 0.011764705882352941
    },
    "score": {
      "count": 200,
      "accuracy": 0.45,
      "brier": 0.634155420139351,
      "brier_soft": 0.634155420139351,
      "nll": 1.2649362408912481,
      "nll_soft": 1.2649362408912481,
      "ece": 0.16487794715140222,
      "mean_confidence": 0.46722106675582253,
      "confident_error_rate": 0.0,
      "score_mae": 0.9795367383039922
    }
  },
  "by_source": {
    "app_reviews": {
      "count": 200,
      "accuracy": 0.45,
      "brier": 0.634155420139351,
      "brier_soft": 0.634155420139351,
      "nll": 1.2649362408912481,
      "nll_soft": 1.2649362408912481,
      "ece": 0.16487794715140222,
      "mean_confidence": 0.46722106675582253,
      "confident_error_rate": 0.0,
      "score_mae": 0.9795367383039922
    },
    "bbc_news": {
      "count": 200,
      "accuracy": 0.97,
      "brier": 0.045028843889621875,
      "brier_soft": 0.045028843889621875,
      "nll": 0.0879174116722789,
      "nll_soft": 0.0879174116722789,
      "ece": 0.033372766687084256,
      "mean_confidence": 0.9562606915180726,
      "confident_error_rate": 0.0
    },
    "copa": {
      "count": 100,
      "accuracy": 0.8,
      "brier": 0.2793984049465368,
      "brier_soft": 0.2793984049465368,
      "nll": 0.43341436730567473,
      "nll_soft": 0.43341436730567473,
      "ece": 0.030292570037666354,
      "mean_confidence": 0.8138699061611925,
      "confident_error_rate": 0.02
    },
    "dream": {
      "count": 200,
      "accuracy": 0.785,
      "brier": 0.3093746739486172,
      "brier_soft": 0.3093746739486172,
      "nll": 0.5593723566722485,
      "nll_soft": 0.5593723566722485,
      "ece": 0.06185597228636879,
      "mean_confidence": 0.8002473621550397,
      "confident_error_rate": 0.015
    },
    "ethos": {
      "count": 200,
      "accuracy": 0.71,
      "brier": 0.4365525991655687,
      "brier_soft": 0.4365525991655687,
      "nll": 0.6733336410747117,
      "nll_soft": 0.6733336410747117,
      "ece": 0.16395509338114295,
      "mean_confidence": 0.8491440397199952,
      "confident_error_rate": 0.1
    },
    "hh_rlhf": {
      "count": 200,
      "accuracy": 0.59,
      "brier": 0.48434266015239147,
      "brier_soft": 0.48434266015239147,
      "nll": 0.675798913004295,
      "nll_soft": 0.675798913004295,
      "ece": 0.0768779816285164,
      "mean_confidence": 0.6020139364942775,
      "confident_error_rate": 0.0
    },
    "medical_questions_pairs": {
      "count": 200,
      "accuracy": 0.835,
      "brier": 0.25880676350180465,
      "brier_soft": 0.25880676350180465,
      "nll": 0.4138157473346344,
      "nll_soft": 0.4138157473346344,
      "ece": 0.10759776921906812,
      "mean_confidence": 0.7410472628912325,
      "confident_error_rate": 0.0
    },
    "scitail": {
      "count": 200,
      "accuracy": 0.77,
      "brier": 0.3273024512031729,
      "brier_soft": 0.3273024512031729,
      "nll": 0.4943908096267626,
      "nll_soft": 0.4943908096267626,
      "ece": 0.06576650128519565,
      "mean_confidence": 0.8152285772676825,
      "confident_error_rate": 0.035
    },
    "stance_climate": {
      "count": 120,
      "accuracy": 0.7333333333333333,
      "brier": 0.35703978074815196,
      "brier_soft": 0.35703978074815196,
      "nll": 0.6215931784341862,
      "nll_soft": 0.6215931784341862,
      "ece": 0.1146295143201135,
      "mean_confidence": 0.6832853748940669,
      "confident_error_rate": 0.0
    },
    "truthful_qa": {
      "count": 200,
      "accuracy": 0.4,
      "brier": 0.8392540473020681,
      "brier_soft": 0.8392540473020681,
      "nll": 1.8585878876929653,
      "nll_soft": 1.8585878876929653,
      "ece": 0.20512361693121203,
      "mean_confidence": 0.605123616931212,
      "confident_error_rate": 0.035
    },
    "tweet_irony": {
      "count": 200,
      "accuracy": 0.76,
      "brier": 0.3249383222601002,
      "brier_soft": 0.3249383222601002,
      "nll": 0.498616805357057,
      "nll_soft": 0.498616805357057,
      "ece": 0.11105966110452868,
      "mean_confidence": 0.6733664342554257,
      "confident_error_rate": 0.0
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 9,
      "confidence": 0.2849088531987708,
      "accuracy": 0.2222222222222222
    },
    {
      "bin": "0.3-0.4",
      "count": 103,
      "confidence": 0.3643654585403831,
      "accuracy": 0.22330097087378642
    },
    {
      "bin": "0.4-0.5",
      "count": 143,
      "confidence": 0.4626790672801925,
      "accuracy": 0.3776223776223776
    },
    {
      "bin": "0.5-0.6",
      "count": 352,
      "confidence": 0.54898583955173,
      "accuracy": 0.5767045454545454
    },
    {
      "bin": "0.6-0.7",
      "count": 338,
      "confidence": 0.6496209727562262,
      "accuracy": 0.6390532544378699
    },
    {
      "bin": "0.7-0.8",
      "count": 257,
      "confidence": 0.7524589863167557,
      "accuracy": 0.754863813229572
    },
    {
      "bin": "0.8-0.9",
      "count": 286,
      "confidence": 0.8522751744259672,
      "accuracy": 0.8286713286713286
    },
    {
      "bin": "0.9-1.0",
      "count": 532,
      "confidence": 0.9569686860687917,
      "accuracy": 0.9266917293233082
    }
  ],
  "seconds": 4.772568941116333,
  "model": "runs/assay-0.6b-v4",
  "data": "data/v4/holdout.jsonl"
}