{
 "model": "decider-35b-a3b v1",
 "temperature": 1.075,
 "protocol": "public regression set rebuilt on this machine: eval half of data/mixture_full.pkl, 95 tasks (67 in-task / 28 held-out), state-first layout, max_ctx 1536, large label sets sub-sampled to 10 options; metrics at the stored temperature",
 "aggregate": {
  "in_task": {
   "acc": 0.8551758035695551,
   "nll": 0.3571210244530074,
   "ece": 0.026005523425337858
  },
  "heldout": {
   "acc": 0.8104967349952829,
   "nll": 0.4970387108931333,
   "ece": 0.06886512604372556
  }
 },
 "results": {
  "clinc_oos": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9826666666666667,
   "nll": 0.06116773560643196,
   "brier": 0.026913650333881378,
   "ece": 0.004699912309646655,
   "aurc": 0.001413522036819675,
   "acc_at_80": 0.9975,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9831148982048035
  },
  "banking77": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9866666666666667,
   "nll": 0.051965903490781784,
   "brier": 0.022627051919698715,
   "ece": 0.016074508627255804,
   "aurc": 0.0006460488654478452,
   "acc_at_80": 0.9991666666666666,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9768534898757935
  },
  "massive_intent": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9633333333333334,
   "nll": 0.11785730719566345,
   "brier": 0.0557381696999073,
   "ece": 0.008312105695406588,
   "aurc": 0.0031118976074550235,
   "acc_at_80": 0.9966666666666667,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.1,
   "mean_conf": 0.9660524129867554
  },
  "massive_scenario": {
   "n": 1500,
   "heldout": true,
   "acc": 0.7993333333333333,
   "nll": 0.5332828760147095,
   "brier": 0.271218866109848,
   "ece": 0.02829863039652506,
   "aurc": 0.055478553769578394,
   "acc_at_80": 0.8808333333333334,
   "acc_at_50": 0.9733333333333334,
   "chance": 0.1,
   "mean_conf": 0.7986482977867126
  },
  "bitext_support": {
   "n": 3000,
   "heldout": false,
   "acc": 0.9993333333333333,
   "nll": 0.0021056807599961758,
   "brier": 0.0011287310626357794,
   "ece": 0.0003376175860563954,
   "aurc": 1.1120011870401867e-06,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.09999999999999998,
   "mean_conf": 0.9992170333862305
  },
  "support_tickets": {
   "n": 4500,
   "heldout": false,
   "acc": 0.5933333333333334,
   "nll": 0.9494668245315552,
   "brier": 0.5020924210548401,
   "ece": 0.017903674913777236,
   "aurc": 0.20277115689216718,
   "acc_at_80": 0.6583333333333333,
   "acc_at_50": 0.7777777777777778,
   "chance": 0.22777777777777777,
   "mean_conf": 0.5856716632843018
  },
  "ag_news": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9346666666666666,
   "nll": 0.20106011629104614,
   "brier": 0.10288789123296738,
   "ece": 0.010400939603646572,
   "aurc": 0.013618430266727854,
   "acc_at_80": 0.9783333333333334,
   "acc_at_50": 0.9893333333333333,
   "chance": 0.25,
   "mean_conf": 0.9317083358764648
  },
  "dbpedia": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9926666666666667,
   "nll": 0.019056648015975952,
   "brier": 0.010244570672512054,
   "ece": 0.00390036821365361,
   "aurc": 6.964876547762933e-05,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9953445792198181
  },
  "yahoo_topics": {
   "n": 1500,
   "heldout": false,
   "acc": 0.7726666666666666,
   "nll": 0.6962489485740662,
   "brier": 0.3215087354183197,
   "ece": 0.04092799503604572,
   "aurc": 0.07648607099774656,
   "acc_at_80": 0.86,
   "acc_at_50": 0.944,
   "chance": 0.1,
   "mean_conf": 0.7425376176834106
  },
  "newsgroups": {
   "n": 1462,
   "heldout": false,
   "acc": 0.8502051983584131,
   "nll": 0.449008584022522,
   "brier": 0.21066278219223022,
   "ece": 0.03997525807199984,
   "aurc": 0.026273657246117587,
   "acc_at_80": 0.9435897435897436,
   "acc_at_50": 0.9931600547195623,
   "chance": 0.09999999999999999,
   "mean_conf": 0.88435298204422
  },
  "bbc_news": {
   "n": 1000,
   "heldout": true,
   "acc": 0.944,
   "nll": 0.13256792724132538,
   "brier": 0.07240951806306839,
   "ece": 0.027219559907913193,
   "aurc": 0.004171691057266529,
   "acc_at_80": 0.995,
   "acc_at_50": 1.0,
   "chance": 0.20000000000000004,
   "mean_conf": 0.9385547637939453
  },
  "trec": {
   "n": 500,
   "heldout": true,
   "acc": 0.832,
   "nll": 0.587404727935791,
   "brier": 0.28857845067977905,
   "ece": 0.1598434355854988,
   "aurc": 0.06672534257121955,
   "acc_at_80": 0.8925,
   "acc_at_50": 0.952,
   "chance": 0.16666666666666663,
   "mean_conf": 0.6734781265258789
  },
  "student_questions": {
   "n": 1500,
   "heldout": true,
   "acc": 0.954,
   "nll": 0.2031065672636032,
   "brier": 0.08712532371282578,
   "ece": 0.08987796316544214,
   "aurc": 0.006595183463977914,
   "acc_at_80": 0.9916666666666667,
   "acc_at_50": 0.9933333333333333,
   "chance": 0.25,
   "mean_conf": 0.865302562713623
  },
  "dolly_category": {
   "n": 1500,
   "heldout": true,
   "acc": 0.354,
   "nll": 1.631442666053772,
   "brier": 0.7210286259651184,
   "ece": 0.09806547661622367,
   "aurc": 0.4519247410276184,
   "acc_at_80": 0.38666666666666666,
   "acc_at_50": 0.47333333333333333,
   "chance": 0.125,
   "mean_conf": 0.4256460666656494
  },
  "imdb": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9706666666666667,
   "nll": 0.0796746090054512,
   "brier": 0.04415109381079674,
   "ece": 0.009556351860364288,
   "aurc": 0.0021247781519913296,
   "acc_at_80": 0.9966666666666667,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9738101959228516
  },
  "sst2": {
   "n": 872,
   "heldout": false,
   "acc": 0.963302752293578,
   "nll": 0.1075686514377594,
   "brier": 0.058069903403520584,
   "ece": 0.007435061174248355,
   "aurc": 0.004894615707237302,
   "acc_at_80": 0.9914040114613181,
   "acc_at_50": 0.9977064220183486,
   "chance": 0.5,
   "mean_conf": 0.9640254974365234
  },
  "sst5": {
   "n": 1500,
   "heldout": false,
   "acc": 0.6326666666666667,
   "nll": 0.8465693593025208,
   "brier": 0.5005199313163757,
   "ece": 0.02862738460302352,
   "aurc": 0.30424582658146127,
   "acc_at_80": 0.6608333333333334,
   "acc_at_50": 0.6973333333333334,
   "chance": 0.2,
   "mean_conf": 0.6309818625450134
  },
  "yelp": {
   "n": 1500,
   "heldout": false,
   "acc": 0.7266666666666667,
   "nll": 0.6343568563461304,
   "brier": 0.37920209765434265,
   "ece": 0.04076567333936691,
   "aurc": 0.13746089140338316,
   "acc_at_80": 0.7716666666666666,
   "acc_at_50": 0.8533333333333334,
   "chance": 0.2,
   "mean_conf": 0.739836573600769
  },
  "amazon_stars": {
   "n": 1500,
   "heldout": false,
   "acc": 0.6213333333333333,
   "nll": 0.8520724177360535,
   "brier": 0.4818972945213318,
   "ece": 0.03010182730356851,
   "aurc": 0.21266833331300722,
   "acc_at_80": 0.6741666666666667,
   "acc_at_50": 0.7693333333333333,
   "chance": 0.2,
   "mean_conf": 0.6432384252548218
  },
  "emotion": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9086666666666666,
   "nll": 0.22119809687137604,
   "brier": 0.12285951524972916,
   "ece": 0.015565859278043073,
   "aurc": 0.012805207153182863,
   "acc_at_80": 0.9808333333333333,
   "acc_at_50": 0.9973333333333333,
   "chance": 0.16666666666666666,
   "mean_conf": 0.9207584857940674
  },
  "go_emotions": {
   "n": 1500,
   "heldout": false,
   "acc": 0.794,
   "nll": 0.5889915227890015,
   "brier": 0.28999069333076477,
   "ece": 0.029260090112686165,
   "aurc": 0.0672639072365525,
   "acc_at_80": 0.8775,
   "acc_at_50": 0.9533333333333334,
   "chance": 0.1,
   "mean_conf": 0.7891288995742798
  },
  "tweet_sentiment": {
   "n": 1500,
   "heldout": false,
   "acc": 0.7393333333333333,
   "nll": 0.5619461536407471,
   "brier": 0.3484695255756378,
   "ece": 0.053492017388343804,
   "aurc": 0.12272191307348365,
   "acc_at_80": 0.8075,
   "acc_at_50": 0.8733333333333333,
   "chance": 0.3333333333333333,
   "mean_conf": 0.706974446773529
  },
  "tweet_emotion": {
   "n": 1421,
   "heldout": false,
   "acc": 0.8648838845883181,
   "nll": 0.3760232925415039,
   "brier": 0.19416743516921997,
   "ece": 0.048255335898738254,
   "aurc": 0.027839586844617643,
   "acc_at_80": 0.9481090589270009,
   "acc_at_50": 0.9887323943661972,
   "chance": 0.25,
   "mean_conf": 0.8190691471099854
  },
  "tweet_irony": {
   "n": 784,
   "heldout": true,
   "acc": 0.860969387755102,
   "nll": 0.3990391194820404,
   "brier": 0.24129782617092133,
   "ece": 0.12924721418898932,
   "aurc": 0.04448957010495625,
   "acc_at_80": 0.9202551834130781,
   "acc_at_50": 0.9642857142857143,
   "chance": 0.5,
   "mean_conf": 0.7329258322715759
  },
  "fin_sentiment": {
   "n": 1500,
   "heldout": true,
   "acc": 0.8386666666666667,
   "nll": 0.46592089533805847,
   "brier": 0.27144962549209595,
   "ece": 0.13573047741254168,
   "aurc": 0.06709094165414199,
   "acc_at_80": 0.8941666666666667,
   "acc_at_50": 0.9466666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.7034342288970947
  },
  "cr_reviews": {
   "n": 753,
   "heldout": true,
   "acc": 0.9136786188579017,
   "nll": 0.1997925043106079,
   "brier": 0.12444894015789032,
   "ece": 0.032485922019320164,
   "aurc": 0.013022305207380291,
   "acc_at_80": 0.978405315614618,
   "acc_at_50": 0.9973404255319149,
   "chance": 0.5,
   "mean_conf": 0.940420389175415
  },
  "counterfactual": {
   "n": 1500,
   "heldout": false,
   "acc": 0.948,
   "nll": 0.12409600615501404,
   "brier": 0.07306990027427673,
   "ece": 0.015566750446955315,
   "aurc": 0.005129710883211171,
   "acc_at_80": 0.995,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9607836008071899
  },
  "subjectivity": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9773333333333334,
   "nll": 0.06512857228517532,
   "brier": 0.03518040478229523,
   "ece": 0.006610139727592426,
   "aurc": 0.001447138121234917,
   "acc_at_80": 0.9983333333333333,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9769946336746216
  },
  "tweet_offensive": {
   "n": 860,
   "heldout": false,
   "acc": 0.8511627906976744,
   "nll": 0.3344406485557556,
   "brier": 0.20971718430519104,
   "ece": 0.039765768203624464,
   "aurc": 0.041883052711596785,
   "acc_at_80": 0.9171511627906976,
   "acc_at_50": 0.9767441860465116,
   "chance": 0.5,
   "mean_conf": 0.8272612690925598
  },
  "tweet_hate": {
   "n": 1500,
   "heldout": false,
   "acc": 0.564,
   "nll": 1.0489171743392944,
   "brier": 0.6417430639266968,
   "ece": 0.2539481555223465,
   "aurc": 0.3877638717389759,
   "acc_at_80": 0.5816666666666667,
   "acc_at_50": 0.5893333333333334,
   "chance": 0.5,
   "mean_conf": 0.8179481625556946
  },
  "hate_offensive": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9266666666666666,
   "nll": 0.2252429574728012,
   "brier": 0.11920014768838882,
   "ece": 0.018018441379070264,
   "aurc": 0.016734557157267283,
   "acc_at_80": 0.9741666666666666,
   "acc_at_50": 0.9906666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9164168238639832
  },
  "civil_comments": {
   "n": 7500,
   "heldout": false,
   "acc": 0.9409333333333333,
   "nll": 0.14164941012859344,
   "brier": 0.08587079495191574,
   "ece": 0.0033977080663044872,
   "aurc": 0.0068072070084911175,
   "acc_at_80": 0.9895,
   "acc_at_50": 0.9994666666666666,
   "chance": 0.5,
   "mean_conf": 0.9395306706428528
  },
  "toxic_chat": {
   "n": 3000,
   "heldout": false,
   "acc": 0.9843333333333333,
   "nll": 0.04290607571601868,
   "brier": 0.023861009627580643,
   "ece": 0.005654305239518452,
   "aurc": 0.0006420751251156756,
   "acc_at_80": 0.9995833333333334,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9809242486953735
  },
  "sms_spam": {
   "n": 1000,
   "heldout": false,
   "acc": 0.994,
   "nll": 0.02838769368827343,
   "brier": 0.01269171480089426,
   "ece": 0.0056210076212883115,
   "aurc": 0.00042434280645942466,
   "acc_at_80": 0.99875,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9934377670288086
  },
  "enron_spam": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9953333333333333,
   "nll": 0.014212104491889477,
   "brier": 0.008566728793084621,
   "ece": 0.004383029421170538,
   "aurc": 6.272064980121389e-05,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.994971513748169
  },
  "insincere_questions": {
   "n": 1500,
   "heldout": false,
   "acc": 0.958,
   "nll": 0.09950008988380432,
   "brier": 0.058815646916627884,
   "ece": 0.009067946950594569,
   "aurc": 0.0033202849523762314,
   "acc_at_80": 0.9966666666666667,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9590474963188171
  },
  "ade": {
   "n": 1500,
   "heldout": true,
   "acc": 0.8373333333333334,
   "nll": 0.3693406283855438,
   "brier": 0.23190657794475555,
   "ece": 0.035113215565681456,
   "aurc": 0.054497040404324716,
   "acc_at_80": 0.8941666666666667,
   "acc_at_50": 0.96,
   "chance": 0.5,
   "mean_conf": 0.807280421257019
  },
  "snli": {
   "n": 1476,
   "heldout": false,
   "acc": 0.9376693766937669,
   "nll": 0.18385177850723267,
   "brier": 0.0963604748249054,
   "ece": 0.02269113342451853,
   "aurc": 0.011467612685230727,
   "acc_at_80": 0.9856054191363252,
   "acc_at_50": 0.9932249322493225,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9178707599639893
  },
  "mnli": {
   "n": 1500,
   "heldout": false,
   "acc": 0.92,
   "nll": 0.22414781153202057,
   "brier": 0.12338652461767197,
   "ece": 0.017153067986170414,
   "aurc": 0.01603407033360699,
   "acc_at_80": 0.9716666666666667,
   "acc_at_50": 0.992,
   "chance": 0.3333333333333333,
   "mean_conf": 0.9121871590614319
  },
  "rte": {
   "n": 277,
   "heldout": false,
   "acc": 0.9350180505415162,
   "nll": 0.18287278711795807,
   "brier": 0.10617374628782272,
   "ece": 0.017879047118369462,
   "aurc": 0.014183728145915892,
   "acc_at_80": 0.9774774774774775,
   "acc_at_50": 0.9927536231884058,
   "chance": 0.5,
   "mean_conf": 0.9323904514312744
  },
  "qnli": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9693333333333334,
   "nll": 0.10291172564029694,
   "brier": 0.05302723869681358,
   "ece": 0.025760468522707608,
   "aurc": 0.0035364855733933526,
   "acc_at_80": 0.9975,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9482638239860535
  },
  "qqp": {
   "n": 1500,
   "heldout": false,
   "acc": 0.8553333333333333,
   "nll": 0.29866960644721985,
   "brier": 0.19174186885356903,
   "ece": 0.02773328121503195,
   "aurc": 0.033885104827278134,
   "acc_at_80": 0.9275,
   "acc_at_50": 0.9826666666666667,
   "chance": 0.5,
   "mean_conf": 0.8796607851982117
  },
  "mrpc": {
   "n": 408,
   "heldout": false,
   "acc": 0.8872549019607843,
   "nll": 0.26749035716056824,
   "brier": 0.16417920589447021,
   "ece": 0.022229443724248905,
   "aurc": 0.027313885803315112,
   "acc_at_80": 0.941717791411043,
   "acc_at_50": 0.9852941176470589,
   "chance": 0.5,
   "mean_conf": 0.8898838758468628
  },
  "paws": {
   "n": 1500,
   "heldout": true,
   "acc": 0.7293333333333333,
   "nll": 0.7380470633506775,
   "brier": 0.4195193648338318,
   "ece": 0.16848981821537015,
   "aurc": 0.15006823722693508,
   "acc_at_80": 0.7791666666666667,
   "acc_at_50": 0.8613333333333333,
   "chance": 0.5,
   "mean_conf": 0.8978231549263
  },
  "cola": {
   "n": 1043,
   "heldout": false,
   "acc": 0.8609779482262704,
   "nll": 0.30430471897125244,
   "brier": 0.1895570456981659,
   "ece": 0.03202069451459189,
   "aurc": 0.035469170008069956,
   "acc_at_80": 0.9268585131894485,
   "acc_at_50": 0.975095785440613,
   "chance": 0.5,
   "mean_conf": 0.8630536794662476
  },
  "boolq": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9206666666666666,
   "nll": 0.21292415261268616,
   "brier": 0.12108734250068665,
   "ece": 0.015688150127728763,
   "aurc": 0.01887893970259577,
   "acc_at_80": 0.965,
   "acc_at_50": 0.988,
   "chance": 0.5,
   "mean_conf": 0.9252172112464905
  },
  "strategyqa": {
   "n": 687,
   "heldout": true,
   "acc": 0.7394468704512372,
   "nll": 0.518520712852478,
   "brier": 0.34887009859085083,
   "ece": 0.03629209233510271,
   "aurc": 0.13807672950510763,
   "acc_at_80": 0.78,
   "acc_at_50": 0.8604651162790697,
   "chance": 0.5,
   "mean_conf": 0.7302948236465454
  },
  "pubmedqa": {
   "n": 500,
   "heldout": true,
   "acc": 0.82,
   "nll": 0.5464581251144409,
   "brier": 0.29492902755737305,
   "ece": 0.07814770770072936,
   "aurc": 0.07736585822989323,
   "acc_at_80": 0.8725,
   "acc_at_50": 0.916,
   "chance": 0.33333333333333326,
   "mean_conf": 0.7438270449638367
  },
  "arc": {
   "n": 1172,
   "heldout": false,
   "acc": 0.9641638225255973,
   "nll": 0.11465426534414291,
   "brier": 0.052921112626791,
   "ece": 0.028441500261985424,
   "aurc": 0.0038676297457262325,
   "acc_at_80": 0.9936034115138592,
   "acc_at_50": 0.9982935153583617,
   "chance": 0.25015642775881686,
   "mean_conf": 0.9453142881393433
  },
  "commonsense_qa": {
   "n": 1221,
   "heldout": false,
   "acc": 0.8705978705978706,
   "nll": 0.33728107810020447,
   "brier": 0.18371696770191193,
   "ece": 0.022902015874270598,
   "aurc": 0.027533759986903508,
   "acc_at_80": 0.9437052200614124,
   "acc_at_50": 0.9836065573770492,
   "chance": 0.19999999999999998,
   "mean_conf": 0.8542661666870117
  },
  "qasc": {
   "n": 926,
   "heldout": false,
   "acc": 0.9146868250539957,
   "nll": 0.28406572341918945,
   "brier": 0.13141772150993347,
   "ece": 0.05652497216383276,
   "aurc": 0.011926056294752855,
   "acc_at_80": 0.9811066126855601,
   "acc_at_50": 0.9978401727861771,
   "chance": 0.125,
   "mean_conf": 0.8592483401298523
  },
  "openbookqa": {
   "n": 500,
   "heldout": false,
   "acc": 0.962,
   "nll": 0.15028874576091766,
   "brier": 0.062103498727083206,
   "ece": 0.029689819455146767,
   "aurc": 0.008756976491078847,
   "acc_at_80": 0.9825,
   "acc_at_50": 0.996,
   "chance": 0.25,
   "mean_conf": 0.9473608136177063
  },
  "sciq": {
   "n": 1000,
   "heldout": true,
   "acc": 0.993,
   "nll": 0.026743656024336815,
   "brier": 0.011477932333946228,
   "ece": 0.010730246603488931,
   "aurc": 0.00012297224385520662,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.25,
   "mean_conf": 0.9845147132873535
  },
  "hellaswag": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9693333333333334,
   "nll": 0.07776311039924622,
   "brier": 0.042344722896814346,
   "ece": 0.006844995915889716,
   "aurc": 0.0013723910741853661,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.25,
   "mean_conf": 0.9741500616073608
  },
  "piqa": {
   "n": 1500,
   "heldout": false,
   "acc": 0.954,
   "nll": 0.12870272994041443,
   "brier": 0.07193071395158768,
   "ece": 0.014045794884363829,
   "aurc": 0.006066841189670484,
   "acc_at_80": 0.9891666666666666,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.5,
   "mean_conf": 0.9444655179977417
  },
  "social_iqa": {
   "n": 1500,
   "heldout": true,
   "acc": 0.8233333333333334,
   "nll": 0.4479636251926422,
   "brier": 0.2568584680557251,
   "ece": 0.02492147314548492,
   "aurc": 0.06027574211898331,
   "acc_at_80": 0.8875,
   "acc_at_50": 0.9493333333333334,
   "chance": 0.3333333333333333,
   "mean_conf": 0.817057192325592
  },
  "winogrande": {
   "n": 1267,
   "heldout": false,
   "acc": 0.8918705603788477,
   "nll": 0.2955634295940399,
   "brier": 0.16731061041355133,
   "ece": 0.0350758349603805,
   "aurc": 0.03663538162242224,
   "acc_at_80": 0.9447731755424064,
   "acc_at_50": 0.973186119873817,
   "chance": 0.5,
   "mean_conf": 0.9254278540611267
  },
  "race": {
   "n": 1500,
   "heldout": false,
   "acc": 0.932,
   "nll": 0.229837566614151,
   "brier": 0.10741978138685226,
   "ece": 0.012891520102818818,
   "aurc": 0.015083615590388144,
   "acc_at_80": 0.9783333333333334,
   "acc_at_50": 0.9906666666666667,
   "chance": 0.25,
   "mean_conf": 0.935739278793335
  },
  "mmlu": {
   "n": 1500,
   "heldout": false,
   "acc": 0.82,
   "nll": 0.49843254685401917,
   "brier": 0.24768584966659546,
   "ece": 0.02976374606291451,
   "aurc": 0.05049327310892201,
   "acc_at_80": 0.91,
   "acc_at_50": 0.9693333333333334,
   "chance": 0.25,
   "mean_conf": 0.8007211685180664
  },
  "medqa": {
   "n": 1273,
   "heldout": false,
   "acc": 0.8546739984289081,
   "nll": 0.39887696504592896,
   "brier": 0.20918008685112,
   "ece": 0.019769610706877737,
   "aurc": 0.034844860161640345,
   "acc_at_80": 0.9322200392927309,
   "acc_at_50": 0.9874213836477987,
   "chance": 0.25,
   "mean_conf": 0.8520532846450806
  },
  "truthfulqa": {
   "n": 817,
   "heldout": true,
   "acc": 0.7539779681762546,
   "nll": 0.8089373111724854,
   "brier": 0.364557683467865,
   "ece": 0.06832761250919638,
   "aurc": 0.08947353158858118,
   "acc_at_80": 0.8425076452599388,
   "acc_at_50": 0.9387254901960784,
   "chance": 0.22622350449767833,
   "mean_conf": 0.696648895740509
  },
  "fin_phrasebank": {
   "n": 970,
   "heldout": true,
   "acc": 0.7587628865979381,
   "nll": 0.6080690026283264,
   "brier": 0.3605556786060333,
   "ece": 0.10978183436025048,
   "aurc": 0.11749813303878813,
   "acc_at_80": 0.8195876288659794,
   "acc_at_50": 0.8886597938144329,
   "chance": 0.33333333333333326,
   "mean_conf": 0.6539795398712158
  },
  "bias_in_bios": {
   "n": 3000,
   "heldout": false,
   "acc": 0.9666666666666667,
   "nll": 0.09920930117368698,
   "brier": 0.04930334910750389,
   "ece": 0.008911347289880108,
   "aurc": 0.00193519783058256,
   "acc_at_80": 0.9991666666666666,
   "acc_at_50": 1.0,
   "chance": 0.30000000000000004,
   "mean_conf": 0.9638564586639404
  },
  "helpsteer2": {
   "n": 5190,
   "heldout": false,
   "acc": 0.6163776493256262,
   "nll": 0.8915571570396423,
   "brier": 0.49822765588760376,
   "ece": 0.01934157278037023,
   "aurc": 0.2413207825095569,
   "acc_at_80": 0.6763005780346821,
   "acc_at_50": 0.7533718689788054,
   "chance": 0.2,
   "mean_conf": 0.620047390460968
  },
  "helpsteer3_pref": {
   "n": 1176,
   "heldout": false,
   "acc": 0.5518707482993197,
   "nll": 1.2033027410507202,
   "brier": 0.5932223796844482,
   "ece": 0.025830757785208366,
   "aurc": 0.30366389562151924,
   "acc_at_80": 0.6068012752391073,
   "acc_at_50": 0.6768707482993197,
   "chance": 0.14285714285714282,
   "mean_conf": 0.5695160031318665
  },
  "hate_speech_scales": {
   "n": 7500,
   "heldout": false,
   "acc": 0.5805333333333333,
   "nll": 0.9507390856742859,
   "brier": 0.528562068939209,
   "ece": 0.013830039529005684,
   "aurc": 0.2502365025510708,
   "acc_at_80": 0.6266666666666667,
   "acc_at_50": 0.7202666666666667,
   "chance": 0.22666666666666674,
   "mean_conf": 0.5702805519104004
  },
  "liar2": {
   "n": 1500,
   "heldout": false,
   "acc": 0.422,
   "nll": 1.3505219221115112,
   "brier": 0.6785846948623657,
   "ece": 0.028518199036518737,
   "aurc": 0.4162147525556497,
   "acc_at_80": 0.4708333333333333,
   "acc_at_50": 0.5653333333333334,
   "chance": 0.16666666666666666,
   "mean_conf": 0.41971758008003235
  },
  "prosocial_safety": {
   "n": 1500,
   "heldout": false,
   "acc": 0.528,
   "nll": 1.1013150215148926,
   "brier": 0.5662647485733032,
   "ece": 0.030879737019538868,
   "aurc": 0.2518870991484317,
   "acc_at_80": 0.5991666666666666,
   "acc_at_50": 0.7213333333333334,
   "chance": 0.2,
   "mean_conf": 0.518919050693512
  },
  "ultrafeedback_pref": {
   "n": 1500,
   "heldout": false,
   "acc": 0.784,
   "nll": 0.43619513511657715,
   "brier": 0.28585085272789,
   "ece": 0.03089568722248076,
   "aurc": 0.08515379420314682,
   "acc_at_80": 0.8475,
   "acc_at_50": 0.9266666666666666,
   "chance": 0.5,
   "mean_conf": 0.8054375648498535
  },
  "shp": {
   "n": 1500,
   "heldout": false,
   "acc": 0.79,
   "nll": 0.44603705406188965,
   "brier": 0.2889350950717926,
   "ece": 0.02012235975265503,
   "aurc": 0.09319282571857365,
   "acc_at_80": 0.8491666666666666,
   "acc_at_50": 0.9146666666666666,
   "chance": 0.5,
   "mean_conf": 0.7964972257614136
  },
  "hh_rlhf": {
   "n": 1488,
   "heldout": false,
   "acc": 0.6901881720430108,
   "nll": 0.5672697424888611,
   "brier": 0.38868722319602966,
   "ece": 0.018854505592776898,
   "aurc": 0.18054136093942436,
   "acc_at_80": 0.7361344537815127,
   "acc_at_50": 0.8077956989247311,
   "chance": 0.5,
   "mean_conf": 0.7015084028244019
  },
  "arena_pref": {
   "n": 1500,
   "heldout": true,
   "acc": 0.5206666666666667,
   "nll": 1.0640077590942383,
   "brier": 0.6215369701385498,
   "ece": 0.1206609118382136,
   "aurc": 0.36246136582814387,
   "acc_at_80": 0.55,
   "acc_at_50": 0.616,
   "chance": 0.3333333333333333,
   "mean_conf": 0.6390127539634705
  },
  "reward_bench": {
   "n": 1500,
   "heldout": true,
   "acc": 0.9186666666666666,
   "nll": 0.2031487673521042,
   "brier": 0.12078241258859634,
   "ece": 0.023855606953302997,
   "aurc": 0.014145065893525021,
   "acc_at_80": 0.9758333333333333,
   "acc_at_50": 0.9946666666666667,
   "chance": 0.5,
   "mean_conf": 0.8953665494918823
  },
  "glaive_tools": {
   "n": 1348,
   "heldout": false,
   "acc": 0.9532640949554896,
   "nll": 0.14300261437892914,
   "brier": 0.07691621780395508,
   "ece": 0.02094891802228167,
   "aurc": 0.007086222936904486,
   "acc_at_80": 0.9860853432282004,
   "acc_at_50": 0.9985163204747775,
   "chance": 0.16666666666666666,
   "mean_conf": 0.9427018165588379
  },
  "toolace": {
   "n": 1000,
   "heldout": false,
   "acc": 0.94,
   "nll": 0.1773465871810913,
   "brier": 0.08758451044559479,
   "ece": 0.019288902848959,
   "aurc": 0.01419978829962143,
   "acc_at_80": 0.98125,
   "acc_at_50": 0.992,
   "chance": 0.12470555555555554,
   "mean_conf": 0.9342852830886841
  },
  "hermes_tools": {
   "n": 1500,
   "heldout": true,
   "acc": 0.7986666666666666,
   "nll": 0.36825039982795715,
   "brier": 0.23031076788902283,
   "ece": 0.08474178238709767,
   "aurc": 0.04541348669345317,
   "acc_at_80": 0.8966666666666666,
   "acc_at_50": 0.988,
   "chance": 0.125,
   "mean_conf": 0.8443251848220825
  },
  "copa": {
   "n": 100,
   "heldout": false,
   "acc": 1.0,
   "nll": 0.018447600305080414,
   "brier": 0.004432467743754387,
   "ece": 0.017155210971832275,
   "aurc": 0.0,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9828447699546814
  },
  "wic": {
   "n": 638,
   "heldout": false,
   "acc": 0.7727272727272727,
   "nll": 0.47851547598838806,
   "brier": 0.3126711845397949,
   "ece": 0.06492637840557994,
   "aurc": 0.10600177937132382,
   "acc_at_80": 0.8352941176470589,
   "acc_at_50": 0.8996865203761756,
   "chance": 0.5,
   "mean_conf": 0.828840970993042
  },
  "multirc": {
   "n": 1500,
   "heldout": false,
   "acc": 0.918,
   "nll": 0.23172627389431,
   "brier": 0.1296291947364807,
   "ece": 0.02276274232069647,
   "aurc": 0.02589494647653243,
   "acc_at_80": 0.965,
   "acc_at_50": 0.9773333333333334,
   "chance": 0.5,
   "mean_conf": 0.9249215722084045
  },
  "cb": {
   "n": 56,
   "heldout": true,
   "acc": 0.8928571428571429,
   "nll": 0.2884705662727356,
   "brier": 0.15287914872169495,
   "ece": 0.08375836323414529,
   "aurc": 0.01340165852858295,
   "acc_at_80": 0.9777777777777777,
   "acc_at_50": 1.0,
   "chance": 0.3333333333333333,
   "mean_conf": 0.8508859276771545
  },
  "fever": {
   "n": 1500,
   "heldout": false,
   "acc": 0.91,
   "nll": 0.2639821767807007,
   "brier": 0.1403927505016327,
   "ece": 0.01887864359219866,
   "aurc": 0.021236299620246823,
   "acc_at_80": 0.9666666666666667,
   "acc_at_50": 0.9866666666666667,
   "chance": 0.3333333333333333,
   "mean_conf": 0.911332368850708
  },
  "wiki_qa": {
   "n": 879,
   "heldout": false,
   "acc": 0.9089874857792947,
   "nll": 0.22839467227458954,
   "brier": 0.13368310034275055,
   "ece": 0.023650864081984878,
   "aurc": 0.020796820711007552,
   "acc_at_80": 0.9644381223328592,
   "acc_at_50": 0.9886363636363636,
   "chance": 0.5,
   "mean_conf": 0.8948045372962952
  },
  "msmarco_rel": {
   "n": 1446,
   "heldout": false,
   "acc": 0.6970954356846473,
   "nll": 0.5717816352844238,
   "brier": 0.39096301794052124,
   "ece": 0.03819479437784535,
   "aurc": 0.18340211744146934,
   "acc_at_80": 0.7303370786516854,
   "acc_at_50": 0.8091286307053942,
   "chance": 0.5,
   "mean_conf": 0.7234166860580444
  },
  "medmcqa": {
   "n": 1500,
   "heldout": false,
   "acc": 0.742,
   "nll": 0.6534839272499084,
   "brier": 0.3494548797607422,
   "ece": 0.04476275533437729,
   "aurc": 0.09236420121984344,
   "acc_at_80": 0.8175,
   "acc_at_50": 0.932,
   "chance": 0.25,
   "mean_conf": 0.7632542848587036
  },
  "quality": {
   "n": 1500,
   "heldout": true,
   "acc": 0.632,
   "nll": 0.9736621379852295,
   "brier": 0.5085250735282898,
   "ece": 0.09598391598463062,
   "aurc": 0.20000924342798884,
   "acc_at_80": 0.69,
   "acc_at_50": 0.7893333333333333,
   "chance": 0.25,
   "mean_conf": 0.726018488407135
  },
  "xstory_cloze": {
   "n": 1500,
   "heldout": true,
   "acc": 0.9946666666666667,
   "nll": 0.026280736550688744,
   "brier": 0.011265838518738747,
   "ece": 0.015819201707839965,
   "aurc": 8.187136167601649e-05,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.5,
   "mean_conf": 0.9801904559135437
  },
  "abstain_probe": {
   "n": 1620,
   "heldout": true,
   "acc": 0.6222222222222222,
   "nll": 1.1009931564331055,
   "brier": 0.5079091191291809,
   "ece": 0.08518773355969675,
   "aurc": 0.17190129595658887,
   "acc_at_80": 0.7106481481481481,
   "acc_at_50": 0.8222222222222222,
   "chance": 0.17897217323143252,
   "mean_conf": 0.7074099779129028
  },
  "agenttraj": {
   "n": 1500,
   "heldout": false,
   "acc": 0.9366666666666666,
   "nll": 0.15657173097133636,
   "brier": 0.08615637570619583,
   "ece": 0.018042015492916074,
   "aurc": 0.004964706283705984,
   "acc_at_80": 0.9958333333333333,
   "acc_at_50": 0.9986666666666667,
   "chance": 0.2,
   "mean_conf": 0.9253047108650208
  },
  "mind2web": {
   "n": 1500,
   "heldout": false,
   "acc": 0.8853333333333333,
   "nll": 0.33396658301353455,
   "brier": 0.1709548383951187,
   "ece": 0.015519337852795898,
   "aurc": 0.02253901936947198,
   "acc_at_80": 0.9558333333333333,
   "acc_at_50": 0.9933333333333333,
   "chance": 0.16666666666666666,
   "mean_conf": 0.885744035243988
  },
  "synth": {
   "n": 300,
   "heldout": false,
   "acc": 0.8466666666666667,
   "nll": 0.39028388261795044,
   "brier": 0.21442179381847382,
   "ece": 0.03632553676764172,
   "aurc": 0.041584943277947255,
   "acc_at_80": 0.9166666666666666,
   "acc_at_50": 0.9733333333333334,
   "chance": 0.37149999999999994,
   "mean_conf": 0.8786032199859619
  },
  "offtopic_probe": {
   "n": 1620,
   "heldout": true,
   "acc": 0.8697530864197531,
   "nll": 0.3856513202190399,
   "brier": 0.17581190168857574,
   "ece": 0.037777107972421745,
   "aurc": 0.018382871739502517,
   "acc_at_80": 0.9683641975308642,
   "acc_at_50": 0.9950617283950617,
   "chance": 0.17979423868312755,
   "mean_conf": 0.8545409440994263
  },
  "hwu64": {
   "n": 1076,
   "heldout": true,
   "acc": 0.974907063197026,
   "nll": 0.08065392076969147,
   "brier": 0.04049574211239815,
   "ece": 0.021740955629534923,
   "aurc": 0.0012811690911756525,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.09999999999999999,
   "mean_conf": 0.9563130140304565
  },
  "dbpedia_l2": {
   "n": 1500,
   "heldout": true,
   "acc": 0.9606666666666667,
   "nll": 0.11009299755096436,
   "brier": 0.060155414044857025,
   "ece": 0.01014547139406203,
   "aurc": 0.003270819231754314,
   "acc_at_80": 0.9958333333333333,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9609081149101257
  },
  "dbpedia_l3": {
   "n": 1500,
   "heldout": true,
   "acc": 0.992,
   "nll": 0.02752424031496048,
   "brier": 0.013588163070380688,
   "ece": 0.0035614574154217704,
   "aurc": 0.000210942320122308,
   "acc_at_80": 1.0,
   "acc_at_50": 1.0,
   "chance": 0.1,
   "mean_conf": 0.9927933216094971
  },
  "quality_full": {
   "n": 600,
   "heldout": true,
   "acc": 0.565,
   "nll": 1.071710467338562,
   "brier": 0.5663425922393799,
   "ece": 0.11241838062802949,
   "aurc": 0.2527098290509578,
   "acc_at_80": 0.6270833333333333,
   "acc_at_50": 0.7366666666666667,
   "chance": 0.25,
   "mean_conf": 0.6741220951080322
  }
 },
 "other_sets": {
  "general_validation_847": {
   "acc": 0.8996,
   "nll": 0.3285,
   "ece": 0.0442
  },
  "typesafe_102": {
   "acc": 0.8627,
   "nll": 0.3415,
   "ece": 0.065,
   "tv_to_reference": 0.1648
  },
  "openjev_5252": {
   "acc": 0.6828,
   "macro_f1": 0.8567,
   "nll": 0.7523,
   "ece": 0.0491
  },
  "mind2web_1770": {
   "acc": 0.8955,
   "nll": 0.3159,
   "ece": 0.0571
  },
  "jevbench_public_items": {
   "easy": 1.0,
   "standard": 0.9722,
   "hard": 0.6757
  },
  "bespoke_public_suite": {
   "macro": 0.7737,
   "micro": 0.7866,
   "macro_untrained": 0.7435
  },
  "miniwob_live_22x8": {
   "sampled": 0.8636,
   "sampled_held_out": 0.7917,
   "greedy": 0.9716,
   "greedy_held_out": 0.9792
  },
  "zero_shot_games_win_rate": {
   "sampled_all": 0.2415,
   "greedy_all": 0.3718
  }
 }
}