{
 "results": {
  "clinc_oos": {
   "n": 1500,
   "acc": 0.9773333333333334,
   "nll": 0.09173629432916641,
   "brier": 0.03913933411240578,
   "ece": 0.013027977565924286,
   "aurc": 0.002160483016578994,
   "acc_at_80": 0.9975,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.1,
   "mean_conf": 0.9865841269493103,
   "sec": 2.4,
   "heldout": false
  },
  "banking77": {
   "n": 1500,
   "acc": 0.9666666666666667,
   "nll": 0.10046543180942535,
   "brier": 0.04899560660123825,
   "ece": 0.01317060301701228,
   "aurc": 0.0015889115300367546,
   "acc_at_80": 0.9991666666666666,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9798372387886047,
   "sec": 1.3,
   "heldout": false
  },
  "massive_intent": {
   "n": 1500,
   "acc": 0.9573333333333334,
   "nll": 0.14932970702648163,
   "brier": 0.06778250634670258,
   "ece": 0.016796776016553244,
   "aurc": 0.004776378371526423,
   "acc_at_80": 0.9941666666666666,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.1,
   "mean_conf": 0.9656777381896973,
   "sec": 1.3,
   "heldout": false
  },
  "massive_scenario": {
   "n": 1500,
   "acc": 0.756,
   "nll": 0.7479000687599182,
   "brier": 0.35114040970802307,
   "ece": 0.10180614854892094,
   "aurc": 0.08514369896051291,
   "acc_at_80": 0.8433333333333334,
   "acc_at_50": 0.9333333333333333,
   "chance": 0.1,
   "mean_conf": 0.8578061461448669,
   "sec": 1.3,
   "heldout": true
  },
  "bitext_support": {
   "n": 3000,
   "acc": 0.997,
   "nll": 0.011109774000942707,
   "brier": 0.004990722052752972,
   "ece": 0.0024439626336097706,
   "aurc": 1.969131620660601e-05,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.09999999999999998,
   "mean_conf": 0.9992172718048096,
   "sec": 1.8,
   "heldout": false,
   "per_q": [
    0.9993333333333333,
    0.9946666666666667
   ]
  },
  "support_tickets": {
   "n": 4500,
   "acc": 0.5708888888888889,
   "nll": 0.9939086437225342,
   "brier": 0.5205920934677124,
   "ece": 0.025038347750902176,
   "aurc": 0.21663716050861523,
   "acc_at_80": 0.6386111111111111,
   "acc_at_50": 0.76,
   "chance": 0.22777777777777777,
   "mean_conf": 0.5925066471099854,
   "sec": 2.4,
   "heldout": false,
   "per_q": [
    0.82,
    0.4106666666666667,
    0.482
   ]
  },
  "ag_news": {
   "n": 1500,
   "acc": 0.9106666666666666,
   "nll": 0.24641166627407074,
   "brier": 0.13118577003479004,
   "ece": 0.015849134624004363,
   "aurc": 0.01962706163947694,
   "acc_at_80": 0.9658333333333333,
   "acc_at_50": 0.9853333333333333,
   "chance": 0.25,
   "mean_conf": 0.9186857342720032,
   "sec": 1.3,
   "heldout": false
  },
  "dbpedia": {
   "n": 1500,
   "acc": 0.9913333333333333,
   "nll": 0.031197283416986465,
   "brier": 0.01437467709183693,
   "ece": 0.004729895810286192,
   "aurc": 0.00012132173934791126,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.994809091091156,
   "sec": 1.6,
   "heldout": false
  },
  "yahoo_topics": {
   "n": 1500,
   "acc": 0.748,
   "nll": 0.7660647630691528,
   "brier": 0.35162588953971863,
   "ece": 0.03356954632202783,
   "aurc": 0.08826358611475389,
   "acc_at_80": 0.8416666666666667,
   "acc_at_50": 0.9293333333333333,
   "chance": 0.1,
   "mean_conf": 0.7593891024589539,
   "sec": 1.5,
   "heldout": false
  },
  "newsgroups": {
   "n": 1462,
   "acc": 0.8077975376196991,
   "nll": 0.5773629546165466,
   "brier": 0.26579606533050537,
   "ece": 0.06540349922900977,
   "aurc": 0.04088176394142939,
   "acc_at_80": 0.911965811965812,
   "acc_at_50": 0.9849521203830369,
   "chance": 0.09999999999999999,
   "mean_conf": 0.8732010126113892,
   "sec": 2.7,
   "heldout": false
  },
  "bbc_news": {
   "n": 1000,
   "acc": 0.927,
   "nll": 0.21117442846298218,
   "brier": 0.11654213815927505,
   "ece": 0.026313644617795985,
   "aurc": 0.012799123011482493,
   "acc_at_80": 0.9725,
   "acc_at_50": 0.998,
   "chance": 0.20000000000000004,
   "mean_conf": 0.9459534883499146,
   "sec": 2.6,
   "heldout": true
  },
  "trec": {
   "n": 500,
   "acc": 0.784,
   "nll": 0.6409680247306824,
   "brier": 0.32721808552742004,
   "ece": 0.07267213892936707,
   "aurc": 0.10896437878729759,
   "acc_at_80": 0.825,
   "acc_at_50": 0.92,
   "chance": 0.16666666666666663,
   "mean_conf": 0.8298088312149048,
   "sec": 0.4,
   "heldout": true
  },
  "student_questions": {
   "n": 1500,
   "acc": 0.9246666666666666,
   "nll": 0.21436315774917603,
   "brier": 0.11058846116065979,
   "ece": 0.017640525956948608,
   "aurc": 0.01580899073511152,
   "acc_at_80": 0.975,
   "acc_at_50": 0.992,
   "chance": 0.25,
   "mean_conf": 0.928837239742279,
   "sec": 1.5,
   "heldout": true
  },
  "dolly_category": {
   "n": 1500,
   "acc": 0.2986666666666667,
   "nll": 2.0224030017852783,
   "brier": 0.8629860281944275,
   "ece": 0.27236494100093844,
   "aurc": 0.5039295041873185,
   "acc_at_80": 0.32666666666666666,
   "acc_at_50": 0.416,
   "chance": 0.125,
   "mean_conf": 0.5710316300392151,
   "sec": 1.8,
   "heldout": true
  },
  "imdb": {
   "n": 1500,
   "acc": 0.9613333333333334,
   "nll": 0.1142592579126358,
   "brier": 0.06406151503324509,
   "ece": 0.024656122922897324,
   "aurc": 0.002928303347136234,
   "acc_at_80": 0.9975,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9826299548149109,
   "sec": 2.8,
   "heldout": false
  },
  "sst2": {
   "n": 872,
   "acc": 0.9380733944954128,
   "nll": 0.17604376375675201,
   "brier": 0.09775912761688232,
   "ece": 0.030137871725296785,
   "aurc": 0.01111562757046914,
   "acc_at_80": 0.9842406876790831,
   "acc_at_50": 0.9954128440366973,
   "chance": 0.5,
   "mean_conf": 0.9652942419052124,
   "sec": 0.6,
   "heldout": false
  },
  "sst5": {
   "n": 1500,
   "acc": 0.604,
   "nll": 0.9462839961051941,
   "brier": 0.5338310599327087,
   "ece": 0.039909611364205684,
   "aurc": 0.30754727422711775,
   "acc_at_80": 0.635,
   "acc_at_50": 0.6866666666666666,
   "chance": 0.2,
   "mean_conf": 0.642160177230835,
   "sec": 1.1,
   "heldout": false
  },
  "yelp": {
   "n": 1500,
   "acc": 0.6826666666666666,
   "nll": 0.7469708919525146,
   "brier": 0.42934244871139526,
   "ece": 0.08389960529406866,
   "aurc": 0.1661291012599122,
   "acc_at_80": 0.7408333333333333,
   "acc_at_50": 0.816,
   "chance": 0.2,
   "mean_conf": 0.765559732913971,
   "sec": 2.3,
   "heldout": false
  },
  "amazon_stars": {
   "n": 1500,
   "acc": 0.6033333333333334,
   "nll": 0.9380196332931519,
   "brier": 0.5161817669868469,
   "ece": 0.07665159477790198,
   "aurc": 0.23412008065702158,
   "acc_at_80": 0.6541666666666667,
   "acc_at_50": 0.748,
   "chance": 0.2,
   "mean_conf": 0.6799849271774292,
   "sec": 1.3,
   "heldout": false
  },
  "emotion": {
   "n": 1500,
   "acc": 0.8693333333333333,
   "nll": 0.375094473361969,
   "brier": 0.1965477466583252,
   "ece": 0.047137337585290234,
   "aurc": 0.031846251103281674,
   "acc_at_80": 0.9383333333333334,
   "acc_at_50": 0.98,
   "chance": 0.16666666666666666,
   "mean_conf": 0.9126279950141907,
   "sec": 1.0,
   "heldout": false
  },
  "go_emotions": {
   "n": 1500,
   "acc": 0.7793333333333333,
   "nll": 0.6534133553504944,
   "brier": 0.3117011487483978,
   "ece": 0.04330594710508986,
   "aurc": 0.07619826478481186,
   "acc_at_80": 0.8633333333333333,
   "acc_at_50": 0.936,
   "chance": 0.1,
   "mean_conf": 0.8226393461227417,
   "sec": 1.2,
   "heldout": false
  },
  "tweet_sentiment": {
   "n": 1500,
   "acc": 0.7206666666666667,
   "nll": 0.6113293170928955,
   "brier": 0.37921401858329773,
   "ece": 0.04055946866671242,
   "aurc": 0.1466257866538856,
   "acc_at_80": 0.765,
   "acc_at_50": 0.8346666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.7430600523948669,
   "sec": 0.9,
   "heldout": false
  },
  "tweet_emotion": {
   "n": 1421,
   "acc": 0.8367346938775511,
   "nll": 0.45104628801345825,
   "brier": 0.2379906326532364,
   "ece": 0.025843723171785783,
   "aurc": 0.04399644830028519,
   "acc_at_80": 0.9076517150395779,
   "acc_at_50": 0.971830985915493,
   "chance": 0.25,
   "mean_conf": 0.8583319187164307,
   "sec": 1.0,
   "heldout": false
  },
  "tweet_irony": {
   "n": 784,
   "acc": 0.7946428571428571,
   "nll": 0.4676361083984375,
   "brier": 0.300514280796051,
   "ece": 0.035634773954445,
   "aurc": 0.1096303514372389,
   "acc_at_80": 0.8437001594896332,
   "acc_at_50": 0.8647959183673469,
   "chance": 0.5,
   "mean_conf": 0.8008275032043457,
   "sec": 0.5,
   "heldout": true
  },
  "fin_sentiment": {
   "n": 1500,
   "acc": 0.7933333333333333,
   "nll": 0.4863547086715698,
   "brier": 0.28904733061790466,
   "ece": 0.019916596571604408,
   "aurc": 0.08617910313882864,
   "acc_at_80": 0.8575,
   "acc_at_50": 0.9146666666666666,
   "chance": 0.3333333333333333,
   "mean_conf": 0.7970937490463257,
   "sec": 1.0,
   "heldout": true
  },
  "cr_reviews": {
   "n": 753,
   "acc": 0.9030544488711819,
   "nll": 0.25531020760536194,
   "brier": 0.14807458221912384,
   "ece": 0.05294444131977842,
   "aurc": 0.018211001734510564,
   "acc_at_80": 0.9651162790697675,
   "acc_at_50": 0.9946808510638298,
   "chance": 0.5,
   "mean_conf": 0.9507516026496887,
   "sec": 0.5,
   "heldout": true
  },
  "counterfactual": {
   "n": 1500,
   "acc": 0.954,
   "nll": 0.14455512166023254,
   "brier": 0.0772322341799736,
   "ece": 0.017266486525535557,
   "aurc": 0.007466418318042988,
   "acc_at_80": 0.9866666666666667,
   "acc_at_50": 0.9973333333333333,
   "chance": 0.5,
   "mean_conf": 0.9662280082702637,
   "sec": 1.0,
   "heldout": false
  },
  "subjectivity": {
   "n": 1500,
   "acc": 0.9593333333333334,
   "nll": 0.11742740124464035,
   "brier": 0.0635886937379837,
   "ece": 0.018429159760475204,
   "aurc": 0.004805976587222085,
   "acc_at_80": 0.9916666666666667,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9714123010635376,
   "sec": 1.0,
   "heldout": false
  },
  "tweet_offensive": {
   "n": 860,
   "acc": 0.8569767441860465,
   "nll": 0.321306049823761,
   "brier": 0.19854530692100525,
   "ece": 0.03273247341777005,
   "aurc": 0.04094359780914508,
   "acc_at_80": 0.9215116279069767,
   "acc_at_50": 0.9674418604651163,
   "chance": 0.5,
   "mean_conf": 0.8376211524009705,
   "sec": 0.6,
   "heldout": false
  },
  "tweet_hate": {
   "n": 1500,
   "acc": 0.5613333333333334,
   "nll": 0.984972357749939,
   "brier": 0.643694281578064,
   "ece": 0.2259776959021886,
   "aurc": 0.43112132729844205,
   "acc_at_80": 0.5641666666666667,
   "acc_at_50": 0.5213333333333333,
   "chance": 0.5,
   "mean_conf": 0.7813499569892883,
   "sec": 1.0,
   "heldout": false
  },
  "hate_offensive": {
   "n": 1500,
   "acc": 0.912,
   "nll": 0.24573880434036255,
   "brier": 0.13170845806598663,
   "ece": 0.0208393304944038,
   "aurc": 0.018957146656246305,
   "acc_at_80": 0.9683333333333334,
   "acc_at_50": 0.9866666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9264606237411499,
   "sec": 1.0,
   "heldout": false
  },
  "civil_comments": {
   "n": 7500,
   "acc": 0.9330666666666667,
   "nll": 0.16645771265029907,
   "brier": 0.09957150369882584,
   "ece": 0.022440280524889583,
   "aurc": 0.008493115215878448,
   "acc_at_80": 0.9856666666666667,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.955498218536377,
   "sec": 2.1,
   "heldout": false,
   "per_q": [
    0.8733333333333333,
    0.9766666666666667,
    0.9906666666666667,
    0.85,
    0.9746666666666667
   ]
  },
  "toxic_chat": {
   "n": 3000,
   "acc": 0.9753333333333334,
   "nll": 0.0650152713060379,
   "brier": 0.035060759633779526,
   "ece": 0.011077121953169521,
   "aurc": 0.0014917009140776134,
   "acc_at_80": 0.99875,
   "acc_at_50": 0.9993333333333333,
   "chance": 0.5,
   "mean_conf": 0.985869288444519,
   "sec": 1.4,
   "heldout": false,
   "per_q": [
    0.958,
    0.9926666666666667
   ]
  },
  "sms_spam": {
   "n": 1000,
   "acc": 0.985,
   "nll": 0.06413730978965759,
   "brier": 0.02651730179786682,
   "ece": 0.013071102797985047,
   "aurc": 0.0017184124196473523,
   "acc_at_80": 0.9975,
   "acc_at_50": 0.998,
   "chance": 0.5,
   "mean_conf": 0.9963533878326416,
   "sec": 0.7,
   "heldout": false
  },
  "enron_spam": {
   "n": 1500,
   "acc": 0.9853333333333333,
   "nll": 0.05048864707350731,
   "brier": 0.023362739011645317,
   "ece": 0.009452791094779985,
   "aurc": 0.0010555480417750408,
   "acc_at_80": 0.9991666666666666,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9947861433029175,
   "sec": 2.9,
   "heldout": false
  },
  "insincere_questions": {
   "n": 1500,
   "acc": 0.952,
   "nll": 0.11656896770000458,
   "brier": 0.06938926875591278,
   "ece": 0.02098088498910269,
   "aurc": 0.004221551290811616,
   "acc_at_80": 0.9983333333333333,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9717850089073181,
   "sec": 0.9,
   "heldout": false
  },
  "ade": {
   "n": 1500,
   "acc": 0.8173333333333334,
   "nll": 0.4560953378677368,
   "brier": 0.2800695300102234,
   "ece": 0.0687519727150599,
   "aurc": 0.07776987989583131,
   "acc_at_80": 0.8625,
   "acc_at_50": 0.9173333333333333,
   "chance": 0.5,
   "mean_conf": 0.8821967840194702,
   "sec": 1.0,
   "heldout": true
  },
  "snli": {
   "n": 1476,
   "acc": 0.9214092140921409,
   "nll": 0.2496514618396759,
   "brier": 0.12413201481103897,
   "ece": 0.02373483920404258,
   "aurc": 0.02304695057683426,
   "acc_at_80": 0.9669771380186283,
   "acc_at_50": 0.981029810298103,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9317938089370728,
   "sec": 1.3,
   "heldout": false
  },
  "mnli": {
   "n": 1500,
   "acc": 0.8846666666666667,
   "nll": 0.3279857933521271,
   "brier": 0.17651496827602386,
   "ece": 0.02796298484007516,
   "aurc": 0.02839381927398111,
   "acc_at_80": 0.9441666666666667,
   "acc_at_50": 0.9853333333333333,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9101898074150085,
   "sec": 1.3,
   "heldout": false
  },
  "rte": {
   "n": 277,
   "acc": 0.8664259927797834,
   "nll": 0.3891109526157379,
   "brier": 0.2202744483947754,
   "ece": 0.06889465591107032,
   "aurc": 0.05497760440143546,
   "acc_at_80": 0.9144144144144144,
   "acc_at_50": 0.9565217391304348,
   "chance": 0.5,
   "mean_conf": 0.9213083386421204,
   "sec": 0.3,
   "heldout": false
  },
  "qnli": {
   "n": 1500,
   "acc": 0.938,
   "nll": 0.1725793182849884,
   "brier": 0.09701354801654816,
   "ece": 0.018119730949401894,
   "aurc": 0.01105178177734522,
   "acc_at_80": 0.9825,
   "acc_at_50": 0.9946666666666667,
   "chance": 0.5,
   "mean_conf": 0.9547027945518494,
   "sec": 1.2,
   "heldout": false
  },
  "qqp": {
   "n": 1500,
   "acc": 0.8486666666666667,
   "nll": 0.33667847514152527,
   "brier": 0.21694913506507874,
   "ece": 0.027132281541824343,
   "aurc": 0.04492686265704937,
   "acc_at_80": 0.9008333333333334,
   "acc_at_50": 0.9706666666666667,
   "chance": 0.5,
   "mean_conf": 0.8645197153091431,
   "sec": 1.1,
   "heldout": false
  },
  "mrpc": {
   "n": 408,
   "acc": 0.8480392156862745,
   "nll": 0.3339563310146332,
   "brier": 0.2105294018983841,
   "ece": 0.04274974734175441,
   "aurc": 0.04511716527430815,
   "acc_at_80": 0.9079754601226994,
   "acc_at_50": 0.9754901960784313,
   "chance": 0.5,
   "mean_conf": 0.8559635877609253,
   "sec": 0.4,
   "heldout": false
  },
  "paws": {
   "n": 1500,
   "acc": 0.724,
   "nll": 0.694682776927948,
   "brier": 0.4271811544895172,
   "ece": 0.17940687636534372,
   "aurc": 0.12480813768846086,
   "acc_at_80": 0.78,
   "acc_at_50": 0.8853333333333333,
   "chance": 0.5,
   "mean_conf": 0.9034069180488586,
   "sec": 1.3,
   "heldout": true
  },
  "cola": {
   "n": 1043,
   "acc": 0.8015340364333653,
   "nll": 0.4175913631916046,
   "brier": 0.2705059051513672,
   "ece": 0.027572413876095537,
   "aurc": 0.07668882275831405,
   "acc_at_80": 0.8657074340527577,
   "acc_at_50": 0.9329501915708812,
   "chance": 0.5,
   "mean_conf": 0.8236285448074341,
   "sec": 0.6,
   "heldout": false
  },
  "boolq": {
   "n": 1500,
   "acc": 0.876,
   "nll": 0.357710599899292,
   "brier": 0.19683362543582916,
   "ece": 0.059815298040707926,
   "aurc": 0.03911365860036364,
   "acc_at_80": 0.9241666666666667,
   "acc_at_50": 0.9693333333333334,
   "chance": 0.5,
   "mean_conf": 0.935267984867096,
   "sec": 1.9,
   "heldout": false
  },
  "strategyqa": {
   "n": 687,
   "acc": 0.5516739446870451,
   "nll": 0.7452422380447388,
   "brier": 0.5299192070960999,
   "ece": 0.17586313932049571,
   "aurc": 0.32907532533270495,
   "acc_at_80": 0.5745454545454546,
   "acc_at_50": 0.6366279069767442,
   "chance": 0.5,
   "mean_conf": 0.7275370359420776,
   "sec": 0.4,
   "heldout": true
  },
  "pubmedqa": {
   "n": 500,
   "acc": 0.756,
   "nll": 0.7593856453895569,
   "brier": 0.3763442933559418,
   "ece": 0.13125784403085708,
   "aurc": 0.11776619019690367,
   "acc_at_80": 0.8375,
   "acc_at_50": 0.908,
   "chance": 0.33333333333333326,
   "mean_conf": 0.8847917318344116,
   "sec": 1.2,
   "heldout": true
  },
  "arc": {
   "n": 1172,
   "acc": 0.8609215017064846,
   "nll": 0.3946582078933716,
   "brier": 0.20159626007080078,
   "ece": 0.03699784692847281,
   "aurc": 0.029178906832842126,
   "acc_at_80": 0.9402985074626866,
   "acc_at_50": 0.9863481228668942,
   "chance": 0.25015642775881686,
   "mean_conf": 0.8952547311782837,
   "sec": 1.0,
   "heldout": false
  },
  "commonsense_qa": {
   "n": 1221,
   "acc": 0.7461097461097461,
   "nll": 0.657385528087616,
   "brier": 0.3438973128795624,
   "ece": 0.06063837679461509,
   "aurc": 0.08669029470544302,
   "acc_at_80": 0.8311156601842374,
   "acc_at_50": 0.9311475409836065,
   "chance": 0.19999999999999998,
   "mean_conf": 0.8057844042778015,
   "sec": 0.8,
   "heldout": false
  },
  "qasc": {
   "n": 926,
   "acc": 0.8012958963282938,
   "nll": 0.612847626209259,
   "brier": 0.2819903790950775,
   "ece": 0.02709557122468436,
   "aurc": 0.05141996432246003,
   "acc_at_80": 0.8866396761133604,
   "acc_at_50": 0.9827213822894169,
   "chance": 0.125,
   "mean_conf": 0.7910774946212769,
   "sec": 0.8,
   "heldout": false
  },
  "openbookqa": {
   "n": 500,
   "acc": 0.816,
   "nll": 0.5107881426811218,
   "brier": 0.2631987929344177,
   "ece": 0.033833789587020886,
   "aurc": 0.05619724629150928,
   "acc_at_80": 0.8925,
   "acc_at_50": 0.976,
   "chance": 0.25,
   "mean_conf": 0.8462465405464172,
   "sec": 0.3,
   "heldout": false
  },
  "sciq": {
   "n": 1000,
   "acc": 0.982,
   "nll": 0.04754561558365822,
   "brier": 0.023483367636799812,
   "ece": 0.014726210862398123,
   "aurc": 0.000321545900436324,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.25,
   "mean_conf": 0.9794425368309021,
   "sec": 1.1,
   "heldout": true
  },
  "hellaswag": {
   "n": 1500,
   "acc": 0.866,
   "nll": 0.3764917254447937,
   "brier": 0.19079357385635376,
   "ece": 0.06253247086207067,
   "aurc": 0.02407728816052661,
   "acc_at_80": 0.9525,
   "acc_at_50": 0.9946666666666667,
   "chance": 0.25,
   "mean_conf": 0.9285325407981873,
   "sec": 2.2,
   "heldout": false
  },
  "piqa": {
   "n": 1500,
   "acc": 0.83,
   "nll": 0.3894118666648865,
   "brier": 0.24019156396389008,
   "ece": 0.04349609518051145,
   "aurc": 0.0600857158889656,
   "acc_at_80": 0.8941666666666667,
   "acc_at_50": 0.944,
   "chance": 0.5,
   "mean_conf": 0.8716477751731873,
   "sec": 1.2,
   "heldout": false
  },
  "social_iqa": {
   "n": 1500,
   "acc": 0.708,
   "nll": 0.7983387112617493,
   "brier": 0.4352244734764099,
   "ece": 0.11886229503154755,
   "aurc": 0.15761526845761895,
   "acc_at_80": 0.7558333333333334,
   "acc_at_50": 0.844,
   "chance": 0.3333333333333333,
   "mean_conf": 0.8267069458961487,
   "sec": 1.1,
   "heldout": true
  },
  "winogrande": {
   "n": 1267,
   "acc": 0.6921862667719021,
   "nll": 0.8599734902381897,
   "brier": 0.48017874360084534,
   "ece": 0.18726083422234047,
   "aurc": 0.20496797159884195,
   "acc_at_80": 0.7268244575936884,
   "acc_at_50": 0.7823343848580442,
   "chance": 0.5,
   "mean_conf": 0.879352331161499,
   "sec": 0.8,
   "heldout": false
  },
  "race": {
   "n": 1500,
   "acc": 0.8286666666666667,
   "nll": 0.5294487476348877,
   "brier": 0.2535547614097595,
   "ece": 0.08820948191483813,
   "aurc": 0.0441451468912063,
   "acc_at_80": 0.9191666666666667,
   "acc_at_50": 0.976,
   "chance": 0.25,
   "mean_conf": 0.9168761372566223,
   "sec": 3.8,
   "heldout": false
  },
  "mmlu": {
   "n": 1500,
   "acc": 0.6293333333333333,
   "nll": 0.9297420382499695,
   "brier": 0.47942283749580383,
   "ece": 0.1056801940202713,
   "aurc": 0.15805034675139773,
   "acc_at_80": 0.7116666666666667,
   "acc_at_50": 0.8626666666666667,
   "chance": 0.25,
   "mean_conf": 0.735013484954834,
   "sec": 1.5,
   "heldout": false
  },
  "medqa": {
   "n": 1273,
   "acc": 0.5443833464257659,
   "nll": 1.2023534774780273,
   "brier": 0.6267179250717163,
   "ece": 0.18061440218553143,
   "aurc": 0.2882548640060218,
   "acc_at_80": 0.5943025540275049,
   "acc_at_50": 0.6808176100628931,
   "chance": 0.25,
   "mean_conf": 0.7214598655700684,
   "sec": 2.0,
   "heldout": false
  },
  "truthfulqa": {
   "n": 817,
   "acc": 0.5373317013463892,
   "nll": 1.5506761074066162,
   "brier": 0.6525235176086426,
   "ece": 0.16036296565646493,
   "aurc": 0.2660050026273411,
   "acc_at_80": 0.599388379204893,
   "acc_at_50": 0.7279411764705882,
   "chance": 0.22622350449767833,
   "mean_conf": 0.6951266527175903,
   "sec": 0.7,
   "heldout": true
  },
  "fin_phrasebank": {
   "n": 970,
   "acc": 0.6938144329896907,
   "nll": 0.710655927658081,
   "brier": 0.432155579328537,
   "ece": 0.09275854675425697,
   "aurc": 0.1946467193061562,
   "acc_at_80": 0.7332474226804123,
   "acc_at_50": 0.8061855670103093,
   "chance": 0.33333333333333326,
   "mean_conf": 0.7770669460296631,
   "sec": 2.0,
   "heldout": true
  },
  "bias_in_bios": {
   "n": 3000,
   "acc": 0.9513333333333334,
   "nll": 0.1453825682401657,
   "brier": 0.0709514245390892,
   "ece": 0.013315296888351444,
   "aurc": 0.0035279360929807786,
   "acc_at_80": 0.99625,
   "acc_at_50": 1.0,
   "chance": 0.30000000000000004,
   "mean_conf": 0.9636430740356445,
   "sec": 2.0,
   "heldout": false,
   "per_q": [
    0.9053333333333333,
    0.9973333333333333
   ]
  },
  "helpsteer2": {
   "n": 5190,
   "acc": 0.5895953757225434,
   "nll": 0.9746355414390564,
   "brier": 0.5325403809547424,
   "ece": 0.057795672655564984,
   "aurc": 0.2721544202844174,
   "acc_at_80": 0.6536608863198459,
   "acc_at_50": 0.735645472061657,
   "chance": 0.2,
   "mean_conf": 0.6461822986602783,
   "sec": 4.8,
   "heldout": false,
   "per_q": [
    0.4325626204238921,
    0.48554913294797686,
    0.7186897880539499,
    0.6204238921001927,
    0.6907514450867052
   ]
  },
  "helpsteer3_pref": {
   "n": 1176,
   "acc": 0.407312925170068,
   "nll": 1.518996000289917,
   "brier": 0.7332671284675598,
   "ece": 0.1154663114726138,
   "aurc": 0.5171907165068561,
   "acc_at_80": 0.4303931987247609,
   "acc_at_50": 0.46258503401360546,
   "chance": 0.14285714285714282,
   "mean_conf": 0.5227791666984558,
   "sec": 6.4,
   "heldout": false
  },
  "hate_speech_scales": {
   "n": 7500,
   "acc": 0.554,
   "nll": 1.0020463466644287,
   "brier": 0.5518749952316284,
   "ece": 0.05263813356558482,
   "aurc": 0.2725105506447112,
   "acc_at_80": 0.6043333333333333,
   "acc_at_50": 0.7026666666666667,
   "chance": 0.22666666666666674,
   "mean_conf": 0.6066381335258484,
   "sec": 3.0,
   "heldout": false,
   "per_q": [
    0.7033333333333334,
    0.472,
    0.43866666666666665,
    0.5686666666666667,
    0.5873333333333334
   ]
  },
  "liar2": {
   "n": 1500,
   "acc": 0.37066666666666664,
   "nll": 1.487428069114685,
   "brier": 0.7230392098426819,
   "ece": 0.042378827313582085,
   "aurc": 0.48226130915305326,
   "acc_at_80": 0.4008333333333333,
   "acc_at_50": 0.472,
   "chance": 0.16666666666666666,
   "mean_conf": 0.4129585921764374,
   "sec": 1.9,
   "heldout": false
  },
  "prosocial_safety": {
   "n": 1500,
   "acc": 0.5126666666666667,
   "nll": 1.1911370754241943,
   "brier": 0.601366400718689,
   "ece": 0.04166399759054183,
   "aurc": 0.28211418515491415,
   "acc_at_80": 0.5758333333333333,
   "acc_at_50": 0.6933333333333334,
   "chance": 0.2,
   "mean_conf": 0.541668713092804,
   "sec": 1.3,
   "heldout": false
  },
  "ultrafeedback_pref": {
   "n": 1500,
   "acc": 0.7506666666666667,
   "nll": 0.5978830456733704,
   "brier": 0.3642973303794861,
   "ece": 0.12037857890129089,
   "aurc": 0.12885069203247962,
   "acc_at_80": 0.8033333333333333,
   "acc_at_50": 0.864,
   "chance": 0.5,
   "mean_conf": 0.8620939254760742,
   "sec": 5.2,
   "heldout": false
  },
  "shp": {
   "n": 1500,
   "acc": 0.7433333333333333,
   "nll": 0.5542265176773071,
   "brier": 0.3617444932460785,
   "ece": 0.08052628513177235,
   "aurc": 0.1415611995055057,
   "acc_at_80": 0.7875,
   "acc_at_50": 0.8613333333333333,
   "chance": 0.5,
   "mean_conf": 0.8218792080879211,
   "sec": 3.8,
   "heldout": false
  },
  "hh_rlhf": {
   "n": 1488,
   "acc": 0.6283602150537635,
   "nll": 0.7063506841659546,
   "brier": 0.48729172348976135,
   "ece": 0.1238586102682416,
   "aurc": 0.2775212031705402,
   "acc_at_80": 0.6470588235294118,
   "acc_at_50": 0.6975806451612904,
   "chance": 0.5,
   "mean_conf": 0.745833694934845,
   "sec": 3.1,
   "heldout": false
  },
  "arena_pref": {
   "n": 1500,
   "acc": 0.4826666666666667,
   "nll": 1.3242307901382446,
   "brier": 0.7265019416809082,
   "ece": 0.24603747620185218,
   "aurc": 0.42174124750905745,
   "acc_at_80": 0.5166666666666667,
   "acc_at_50": 0.564,
   "chance": 0.3333333333333333,
   "mean_conf": 0.7287040948867798,
   "sec": 4.9,
   "heldout": true
  },
  "reward_bench": {
   "n": 1500,
   "acc": 0.8186666666666667,
   "nll": 0.4543558359146118,
   "brier": 0.274769127368927,
   "ece": 0.07598515292008719,
   "aurc": 0.07768967728506293,
   "acc_at_80": 0.8666666666666667,
   "acc_at_50": 0.9333333333333333,
   "chance": 0.5,
   "mean_conf": 0.8845229744911194,
   "sec": 3.9,
   "heldout": true
  },
  "glaive_tools": {
   "n": 1348,
   "acc": 0.9413946587537092,
   "nll": 0.19328154623508453,
   "brier": 0.10143861919641495,
   "ece": 0.022093097250963853,
   "aurc": 0.01690886528489672,
   "acc_at_80": 0.9693877551020408,
   "acc_at_50": 0.983679525222552,
   "chance": 0.16666666666666666,
   "mean_conf": 0.955422580242157,
   "sec": 1.6,
   "heldout": false
  },
  "toolace": {
   "n": 1000,
   "acc": 0.931,
   "nll": 0.2512577474117279,
   "brier": 0.11597812175750732,
   "ece": 0.031337061107158684,
   "aurc": 0.02448370972130845,
   "acc_at_80": 0.97,
   "acc_at_50": 0.982,
   "chance": 0.12470555555555554,
   "mean_conf": 0.9608047604560852,
   "sec": 2.0,
   "heldout": false
  },
  "hermes_tools": {
   "n": 1500,
   "acc": 0.7226666666666667,
   "nll": 0.8366944789886475,
   "brier": 0.464679479598999,
   "ece": 0.23630806771914162,
   "aurc": 0.06836947758114767,
   "acc_at_80": 0.8566666666666667,
   "acc_at_50": 0.984,
   "chance": 0.125,
   "mean_conf": 0.9589747786521912,
   "sec": 4.7,
   "heldout": true
  },
  "copa": {
   "n": 100,
   "acc": 0.93,
   "nll": 0.20605678856372833,
   "brier": 0.11243906617164612,
   "ece": 0.028781158924102764,
   "aurc": 0.019294681675211874,
   "acc_at_80": 0.975,
   "acc_at_50": 0.98,
   "chance": 0.5,
   "mean_conf": 0.9292109608650208,
   "sec": 0.1,
   "heldout": false
  },
  "wic": {
   "n": 638,
   "acc": 0.6943573667711599,
   "nll": 0.6192224621772766,
   "brier": 0.41049328446388245,
   "ece": 0.10207162110783091,
   "aurc": 0.2013770101215354,
   "acc_at_80": 0.7490196078431373,
   "acc_at_50": 0.8056426332288401,
   "chance": 0.5,
   "mean_conf": 0.7964289784431458,
   "sec": 0.4,
   "heldout": false
  },
  "multirc": {
   "n": 1500,
   "acc": 0.8873333333333333,
   "nll": 0.353663831949234,
   "brier": 0.18488313257694244,
   "ece": 0.057831449985504116,
   "aurc": 0.04080578362832766,
   "acc_at_80": 0.9341666666666667,
   "acc_at_50": 0.972,
   "chance": 0.5,
   "mean_conf": 0.9418337345123291,
   "sec": 3.5,
   "heldout": false
  },
  "cb": {
   "n": 56,
   "acc": 0.8571428571428571,
   "nll": 0.39587274193763733,
   "brier": 0.1826748102903366,
   "ece": 0.061279319226741805,
   "aurc": 0.024518910173493138,
   "acc_at_80": 0.9555555555555556,
   "acc_at_50": 1.0,
   "chance": 0.3333333333333333,
   "mean_conf": 0.8873938322067261,
   "sec": 0.1,
   "heldout": true
  },
  "fever": {
   "n": 1500,
   "acc": 0.888,
   "nll": 0.3543616235256195,
   "brier": 0.18396615982055664,
   "ece": 0.05364456375439957,
   "aurc": 0.035409527613622344,
   "acc_at_80": 0.9416666666666667,
   "acc_at_50": 0.9706666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9349583387374878,
   "sec": 1.4,
   "heldout": false
  },
  "wiki_qa": {
   "n": 879,
   "acc": 0.8953356086461889,
   "nll": 0.2812374532222748,
   "brier": 0.15986107289791107,
   "ece": 0.03158905461379691,
   "aurc": 0.029564777940039345,
   "acc_at_80": 0.9473684210526315,
   "acc_at_50": 0.9772727272727273,
   "chance": 0.5,
   "mean_conf": 0.9265106320381165,
   "sec": 0.7,
   "heldout": false
  },
  "msmarco_rel": {
   "n": 1446,
   "acc": 0.6535269709543569,
   "nll": 0.6131556034088135,
   "brier": 0.42479565739631653,
   "ece": 0.03628980136511236,
   "aurc": 0.23044128645847725,
   "acc_at_80": 0.6871218668971478,
   "acc_at_50": 0.7565698478561549,
   "chance": 0.5,
   "mean_conf": 0.6898168325424194,
   "sec": 1.5,
   "heldout": false
  },
  "medmcqa": {
   "n": 1500,
   "acc": 0.5,
   "nll": 1.1675626039505005,
   "brier": 0.6214771270751953,
   "ece": 0.1292713791926702,
   "aurc": 0.3062450473649221,
   "acc_at_80": 0.5616666666666666,
   "acc_at_50": 0.6666666666666666,
   "chance": 0.25,
   "mean_conf": 0.629271388053894,
   "sec": 1.1,
   "heldout": false
  },
  "quality": {
   "n": 1500,
   "acc": 0.494,
   "nll": 1.5713512897491455,
   "brier": 0.7469174861907959,
   "ece": 0.29271218756834666,
   "aurc": 0.34188336200889935,
   "acc_at_80": 0.545,
   "acc_at_50": 0.6346666666666667,
   "chance": 0.25,
   "mean_conf": 0.7867121696472168,
   "sec": 10.7,
   "heldout": true
  },
  "xstory_cloze": {
   "n": 1500,
   "acc": 0.962,
   "nll": 0.10309821367263794,
   "brier": 0.05630293861031532,
   "ece": 0.00773787120978037,
   "aurc": 0.004358402846713855,
   "acc_at_80": 0.9941666666666666,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9653077125549316,
   "sec": 1.3,
   "heldout": true
  },
  "abstain_probe": {
   "n": 1620,
   "acc": 0.6055555555555555,
   "nll": 1.1703203916549683,
   "brier": 0.5663847923278809,
   "ece": 0.19228262847956318,
   "aurc": 0.20466473892179798,
   "acc_at_80": 0.6929012345679012,
   "acc_at_50": 0.7851851851851852,
   "chance": 0.17897217323143252,
   "mean_conf": 0.7978381514549255,
   "sec": 1.9,
   "heldout": true
  },
  "agenttraj": {
   "n": 1500,
   "acc": 0.916,
   "nll": 0.22318525612354279,
   "brier": 0.11291919648647308,
   "ece": 0.03268852589527768,
   "aurc": 0.009361436014432775,
   "acc_at_80": 0.9858333333333333,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.2,
   "mean_conf": 0.9431377053260803,
   "sec": 3.4,
   "heldout": false
  },
  "mind2web": {
   "n": 1500,
   "acc": 0.8266666666666667,
   "nll": 0.49047866463661194,
   "brier": 0.25011762976646423,
   "ece": 0.056447255154450725,
   "aurc": 0.04288666994842876,
   "acc_at_80": 0.9033333333333333,
   "acc_at_50": 0.9813333333333333,
   "chance": 0.16666666666666666,
   "mean_conf": 0.8782970309257507,
   "sec": 2.0,
   "heldout": false
  },
  "synth": {
   "n": 300,
   "acc": 0.7933333333333333,
   "nll": 0.6004238128662109,
   "brier": 0.3256992995738983,
   "ece": 0.07842869222164153,
   "aurc": 0.08963059071585025,
   "acc_at_80": 0.8458333333333333,
   "acc_at_50": 0.8933333333333333,
   "chance": 0.37149999999999994,
   "mean_conf": 0.8574635982513428,
   "sec": 0.2,
   "heldout": false,
   "per_q": [
    0.7666666666666667,
    0.82
   ]
  },
  "offtopic_probe": {
   "n": 1620,
   "acc": 0.841358024691358,
   "nll": 0.5042567253112793,
   "brier": 0.22333061695098877,
   "ece": 0.04802086000089292,
   "aurc": 0.030512282206217035,
   "acc_at_80": 0.9405864197530864,
   "acc_at_50": 0.9888888888888889,
   "chance": 0.17979423868312755,
   "mean_conf": 0.8893788456916809,
   "sec": 1.8,
   "heldout": true
  },
  "hwu64": {
   "n": 1076,
   "acc": 0.9609665427509294,
   "nll": 0.12583288550376892,
   "brier": 0.06144118681550026,
   "ece": 0.009324527861460477,
   "aurc": 0.003260867906700039,
   "acc_at_80": 0.9953542392566783,
   "acc_at_50": 1.0,
   "chance": 0.09999999999999999,
   "mean_conf": 0.9562390446662903,
   "sec": 0.9,
   "heldout": true
  },
  "dbpedia_l2": {
   "n": 1500,
   "acc": 0.95,
   "nll": 0.16574443876743317,
   "brier": 0.08210423588752747,
   "ece": 0.029746596733729078,
   "aurc": 0.005207273669368498,
   "acc_at_80": 0.9941666666666666,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9745652675628662,
   "sec": 2.2,
   "heldout": true
  },
  "dbpedia_l3": {
   "n": 1500,
   "acc": 0.9866666666666667,
   "nll": 0.054405681788921356,
   "brier": 0.022596782073378563,
   "ece": 0.007581249455610876,
   "aurc": 0.0009766034191328878,
   "acc_at_80": 0.9991666666666666,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.1,
   "mean_conf": 0.9932629466056824,
   "sec": 2.2,
   "heldout": true
  },
  "quality_full": {
   "n": 600,
   "acc": 0.5083333333333333,
   "nll": 1.5723801851272583,
   "brier": 0.7437332272529602,
   "ece": 0.261873353322347,
   "aurc": 0.371980887569386,
   "acc_at_80": 0.5333333333333333,
   "acc_at_50": 0.6166666666666667,
   "chance": 0.25,
   "mean_conf": 0.7702066898345947,
   "sec": 5.4,
   "heldout": true
  }
 },
 "agg": {
  "in_task": {
   "acc": 0.8053939243254699,
   "nll": 0.4901052765242422,
   "brier": 0.26693312430392896,
   "ece": 0.051122906597442695,
   "aurc": 0.09948275075275431,
   "acc_at_80": 0.8563021912710674,
   "chance": 0.32540293886491484
  },
  "heldout": {
   "acc": 0.7550550130420666,
   "nll": 0.6816884187449302,
   "brier": 0.3505160380154848,
   "ece": 0.10750615544056343,
   "aurc": 0.134422784017669,
   "acc_at_80": 0.8086450646710732,
   "chance": 0.2893448779671037
  },
  "n_in": 67,
  "n_heldout": 28
 },
 "model": "/mnt/data/mark/projects/decider/runs/calibrated_rl_2b_v2/control_9419/checkpoint-384",
 "engine": ""
}