{
  "overall": {
    "count": 764,
    "accuracy": 0.6361256544502618,
    "brier": 0.5172346498279451,
    "brier_soft": 0.5172346498279451,
    "nll": 0.9118470266186511,
    "nll_soft": 0.9118470266186511,
    "ece": 0.1574055154685096,
    "mean_confidence": 0.7935311699187713,
    "confident_error_rate": 0.07722513089005235,
    "score_mae": 0.26194014217696393
  },
  "by_type": {
    "bool": {
      "count": 280,
      "accuracy": 0.6642857142857143,
      "brier": 0.5148589570798352,
      "brier_soft": 0.5148589570798352,
      "nll": 0.8016488353036343,
      "nll_soft": 0.8016488353036343,
      "ece": 0.19575883647034126,
      "mean_confidence": 0.848601454717336,
      "confident_error_rate": 0.13214285714285715
    },
    "choice": {
      "count": 444,
      "accuracy": 0.5923423423423423,
      "brier": 0.5487743828697039,
      "brier_soft": 0.5487743828697039,
      "nll": 1.0307850284398343,
      "nll_soft": 1.0307850284398343,
      "ece": 0.17313409277617448,
      "mean_confidence": 0.7654764351185169,
      "confident_error_rate": 0.04954954954954955
    },
    "score": {
      "count": 40,
      "accuracy": 0.925,
      "brier": 0.1837734623011918,
      "brier_soft": 0.1837734623011918,
      "nll": 0.36302254560863306,
      "nll_soft": 0.36302254560863306,
      "ece": 0.22877284823058613,
      "mean_confidence": 0.7194467326116454,
      "confident_error_rate": 0.0,
      "score_mae": 0.26194014217696393
    }
  },
  "by_source": {
    "composition_held_and_or": {
      "count": 32,
      "accuracy": 0.5,
      "brier": 0.7092949409522205,
      "brier_soft": 0.7092949409522205,
      "nll": 0.9877625696623309,
      "nll_soft": 0.9877625696623309,
      "ece": 0.4540132369328322,
      "mean_confidence": 0.7837868646408732,
      "confident_error_rate": 0.03125
    },
    "composition_held_conditional": {
      "count": 32,
      "accuracy": 0.625,
      "brier": 0.6493002049653968,
      "brier_soft": 0.6493002049653968,
      "nll": 0.9276462868599313,
      "nll_soft": 0.9276462868599313,
      "ece": 0.37479555167599893,
      "mean_confidence": 0.7723797661346661,
      "confident_error_rate": 0.0625
    },
    "composition_held_or_not": {
      "count": 32,
      "accuracy": 0.5625,
      "brier": 0.4861723394546122,
      "brier_soft": 0.4861723394546122,
      "nll": 0.6773762063372166,
      "nll_soft": 0.6773762063372166,
      "ece": 0.2708231571121984,
      "mean_confidence": 0.7411207695713653,
      "confident_error_rate": 0.0
    },
    "contrastive_authorization": {
      "count": 40,
      "accuracy": 0.5,
      "brier": 0.8895817722240779,
      "brier_soft": 0.8895817722240779,
      "nll": 1.4986839071795353,
      "nll_soft": 1.4986839071795353,
      "ece": 0.46812489874267255,
      "mean_confidence": 0.9681248987426724,
      "confident_error_rate": 0.475
    },
    "contrastive_deadline": {
      "count": 40,
      "accuracy": 0.925,
      "brier": 0.1837734623011918,
      "brier_soft": 0.1837734623011918,
      "nll": 0.36302254560863306,
      "nll_soft": 0.36302254560863306,
      "ece": 0.22877284823058613,
      "mean_confidence": 0.7194467326116454,
      "confident_error_rate": 0.0,
      "score_mae": 0.26194014217696393
    },
    "emotion": {
      "count": 116,
      "accuracy": 0.45689655172413796,
      "brier": 0.7389898180850448,
      "brier_soft": 0.7389898180850448,
      "nll": 1.4824586721590065,
      "nll_soft": 1.4824586721590065,
      "ece": 0.25607238639448815,
      "mean_confidence": 0.7129689381186259,
      "confident_error_rate": 0.09482758620689655
    },
    "mmlu": {
      "count": 116,
      "accuracy": 0.45689655172413796,
      "brier": 0.6579950172042954,
      "brier_soft": 0.6579950172042954,
      "nll": 1.265234928976915,
      "nll_soft": 1.265234928976915,
      "ece": 0.19014657092264517,
      "mean_confidence": 0.6470431226467831,
      "confident_error_rate": 0.017241379310344827
    },
    "paws": {
      "count": 80,
      "accuracy": 0.5875,
      "brier": 0.5756890068833255,
      "brier_soft": 0.5756890068833255,
      "nll": 0.8440926754097665,
      "nll_soft": 0.8440926754097665,
      "ece": 0.2933412652419587,
      "mean_confidence": 0.8109503950137414,
      "confident_error_rate": 0.0875
    },
    "qnli": {
      "count": 80,
      "accuracy": 0.775,
      "brier": 0.31223393190475546,
      "brier_soft": 0.31223393190475546,
      "nll": 0.47942241716946404,
      "nll_soft": 0.47942241716946404,
      "ece": 0.0778396244346931,
      "mean_confidence": 0.7880813990695568,
      "confident_error_rate": 0.0375
    },
    "sciq": {
      "count": 116,
      "accuracy": 0.8879310344827587,
      "brier": 0.19459504800580804,
      "brier_soft": 0.19459504800580804,
      "nll": 0.4824739041725539,
      "nll_soft": 0.4824739041725539,
      "ece": 0.06633123740955421,
      "mean_confidence": 0.9361805287304577,
      "confident_error_rate": 0.05172413793103448
    },
    "tweet_offensive": {
      "count": 80,
      "accuracy": 0.7125,
      "brier": 0.4692925248793035,
      "brier_soft": 0.4692925248793035,
      "nll": 0.7329138773937217,
      "nll_soft": 0.7329138773937217,
      "ece": 0.21328910565862647,
      "mean_confidence": 0.8870108480560412,
      "confident_error_rate": 0.1
    }
  },
  "reliability": [
    {
      "bin": "0.0-0.1",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.1-0.2",
      "count": 0,
      "confidence": NaN,
      "accuracy": NaN
    },
    {
      "bin": "0.2-0.3",
      "count": 3,
      "confidence": 0.2958333662963604,
      "accuracy": 0.0
    },
    {
      "bin": "0.3-0.4",
      "count": 22,
      "confidence": 0.3612045461726474,
      "accuracy": 0.22727272727272727
    },
    {
      "bin": "0.4-0.5",
      "count": 42,
      "confidence": 0.46611278257339644,
      "accuracy": 0.38095238095238093
    },
    {
      "bin": "0.5-0.6",
      "count": 84,
      "confidence": 0.5535734667004667,
      "accuracy": 0.4880952380952381
    },
    {
      "bin": "0.6-0.7",
      "count": 69,
      "confidence": 0.6512806744816579,
      "accuracy": 0.5362318840579711
    },
    {
      "bin": "0.7-0.8",
      "count": 95,
      "confidence": 0.7553533705903317,
      "accuracy": 0.6421052631578947
    },
    {
      "bin": "0.8-0.9",
      "count": 164,
      "confidence": 0.8537295165882544,
      "accuracy": 0.6097560975609756
    },
    {
      "bin": "0.9-1.0",
      "count": 285,
      "confidence": 0.9636432567247109,
      "accuracy": 0.7929824561403509
    }
  ],
  "seconds": 1.6747174263000488,
  "model": "runs/assay-0.6b-v4",
  "data": "data/suites/kev-transfer-v4-dev.jsonl"
}