| { |
| "results": { |
| "clinc_oos": { |
| "n": 1500, |
| "acc": 0.9773333333333334, |
| "nll": 0.09173629432916641, |
| "brier": 0.03913933411240578, |
| "ece": 0.013027977565924286, |
| "aurc": 0.002160483016578994, |
| "acc_at_80": 0.9975, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.1, |
| "mean_conf": 0.9865841269493103, |
| "sec": 2.4, |
| "heldout": false |
| }, |
| "banking77": { |
| "n": 1500, |
| "acc": 0.9666666666666667, |
| "nll": 0.10046543180942535, |
| "brier": 0.04899560660123825, |
| "ece": 0.01317060301701228, |
| "aurc": 0.0015889115300367546, |
| "acc_at_80": 0.9991666666666666, |
| "acc_at_50": 1.0, |
| "chance": 0.1, |
| "mean_conf": 0.9798372387886047, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "massive_intent": { |
| "n": 1500, |
| "acc": 0.9573333333333334, |
| "nll": 0.14932970702648163, |
| "brier": 0.06778250634670258, |
| "ece": 0.016796776016553244, |
| "aurc": 0.004776378371526423, |
| "acc_at_80": 0.9941666666666666, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.1, |
| "mean_conf": 0.9656777381896973, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "massive_scenario": { |
| "n": 1500, |
| "acc": 0.756, |
| "nll": 0.7479000687599182, |
| "brier": 0.35114040970802307, |
| "ece": 0.10180614854892094, |
| "aurc": 0.08514369896051291, |
| "acc_at_80": 0.8433333333333334, |
| "acc_at_50": 0.9333333333333333, |
| "chance": 0.1, |
| "mean_conf": 0.8578061461448669, |
| "sec": 1.3, |
| "heldout": true |
| }, |
| "bitext_support": { |
| "n": 3000, |
| "acc": 0.997, |
| "nll": 0.011109774000942707, |
| "brier": 0.004990722052752972, |
| "ece": 0.0024439626336097706, |
| "aurc": 1.969131620660601e-05, |
| "acc_at_80": 1.0, |
| "acc_at_50": 1.0, |
| "chance": 0.09999999999999998, |
| "mean_conf": 0.9992172718048096, |
| "sec": 1.8, |
| "heldout": false, |
| "per_q": [ |
| 0.9993333333333333, |
| 0.9946666666666667 |
| ] |
| }, |
| "support_tickets": { |
| "n": 4500, |
| "acc": 0.5708888888888889, |
| "nll": 0.9939086437225342, |
| "brier": 0.5205920934677124, |
| "ece": 0.025038347750902176, |
| "aurc": 0.21663716050861523, |
| "acc_at_80": 0.6386111111111111, |
| "acc_at_50": 0.76, |
| "chance": 0.22777777777777777, |
| "mean_conf": 0.5925066471099854, |
| "sec": 2.4, |
| "heldout": false, |
| "per_q": [ |
| 0.82, |
| 0.4106666666666667, |
| 0.482 |
| ] |
| }, |
| "ag_news": { |
| "n": 1500, |
| "acc": 0.9106666666666666, |
| "nll": 0.24641166627407074, |
| "brier": 0.13118577003479004, |
| "ece": 0.015849134624004363, |
| "aurc": 0.01962706163947694, |
| "acc_at_80": 0.9658333333333333, |
| "acc_at_50": 0.9853333333333333, |
| "chance": 0.25, |
| "mean_conf": 0.9186857342720032, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "dbpedia": { |
| "n": 1500, |
| "acc": 0.9913333333333333, |
| "nll": 0.031197283416986465, |
| "brier": 0.01437467709183693, |
| "ece": 0.004729895810286192, |
| "aurc": 0.00012132173934791126, |
| "acc_at_80": 1.0, |
| "acc_at_50": 1.0, |
| "chance": 0.1, |
| "mean_conf": 0.994809091091156, |
| "sec": 1.6, |
| "heldout": false |
| }, |
| "yahoo_topics": { |
| "n": 1500, |
| "acc": 0.748, |
| "nll": 0.7660647630691528, |
| "brier": 0.35162588953971863, |
| "ece": 0.03356954632202783, |
| "aurc": 0.08826358611475389, |
| "acc_at_80": 0.8416666666666667, |
| "acc_at_50": 0.9293333333333333, |
| "chance": 0.1, |
| "mean_conf": 0.7593891024589539, |
| "sec": 1.5, |
| "heldout": false |
| }, |
| "newsgroups": { |
| "n": 1462, |
| "acc": 0.8077975376196991, |
| "nll": 0.5773629546165466, |
| "brier": 0.26579606533050537, |
| "ece": 0.06540349922900977, |
| "aurc": 0.04088176394142939, |
| "acc_at_80": 0.911965811965812, |
| "acc_at_50": 0.9849521203830369, |
| "chance": 0.09999999999999999, |
| "mean_conf": 0.8732010126113892, |
| "sec": 2.7, |
| "heldout": false |
| }, |
| "bbc_news": { |
| "n": 1000, |
| "acc": 0.927, |
| "nll": 0.21117442846298218, |
| "brier": 0.11654213815927505, |
| "ece": 0.026313644617795985, |
| "aurc": 0.012799123011482493, |
| "acc_at_80": 0.9725, |
| "acc_at_50": 0.998, |
| "chance": 0.20000000000000004, |
| "mean_conf": 0.9459534883499146, |
| "sec": 2.6, |
| "heldout": true |
| }, |
| "trec": { |
| "n": 500, |
| "acc": 0.784, |
| "nll": 0.6409680247306824, |
| "brier": 0.32721808552742004, |
| "ece": 0.07267213892936707, |
| "aurc": 0.10896437878729759, |
| "acc_at_80": 0.825, |
| "acc_at_50": 0.92, |
| "chance": 0.16666666666666663, |
| "mean_conf": 0.8298088312149048, |
| "sec": 0.4, |
| "heldout": true |
| }, |
| "student_questions": { |
| "n": 1500, |
| "acc": 0.9246666666666666, |
| "nll": 0.21436315774917603, |
| "brier": 0.11058846116065979, |
| "ece": 0.017640525956948608, |
| "aurc": 0.01580899073511152, |
| "acc_at_80": 0.975, |
| "acc_at_50": 0.992, |
| "chance": 0.25, |
| "mean_conf": 0.928837239742279, |
| "sec": 1.5, |
| "heldout": true |
| }, |
| "dolly_category": { |
| "n": 1500, |
| "acc": 0.2986666666666667, |
| "nll": 2.0224030017852783, |
| "brier": 0.8629860281944275, |
| "ece": 0.27236494100093844, |
| "aurc": 0.5039295041873185, |
| "acc_at_80": 0.32666666666666666, |
| "acc_at_50": 0.416, |
| "chance": 0.125, |
| "mean_conf": 0.5710316300392151, |
| "sec": 1.8, |
| "heldout": true |
| }, |
| "imdb": { |
| "n": 1500, |
| "acc": 0.9613333333333334, |
| "nll": 0.1142592579126358, |
| "brier": 0.06406151503324509, |
| "ece": 0.024656122922897324, |
| "aurc": 0.002928303347136234, |
| "acc_at_80": 0.9975, |
| "acc_at_50": 1.0, |
| "chance": 0.5, |
| "mean_conf": 0.9826299548149109, |
| "sec": 2.8, |
| "heldout": false |
| }, |
| "sst2": { |
| "n": 872, |
| "acc": 0.9380733944954128, |
| "nll": 0.17604376375675201, |
| "brier": 0.09775912761688232, |
| "ece": 0.030137871725296785, |
| "aurc": 0.01111562757046914, |
| "acc_at_80": 0.9842406876790831, |
| "acc_at_50": 0.9954128440366973, |
| "chance": 0.5, |
| "mean_conf": 0.9652942419052124, |
| "sec": 0.6, |
| "heldout": false |
| }, |
| "sst5": { |
| "n": 1500, |
| "acc": 0.604, |
| "nll": 0.9462839961051941, |
| "brier": 0.5338310599327087, |
| "ece": 0.039909611364205684, |
| "aurc": 0.30754727422711775, |
| "acc_at_80": 0.635, |
| "acc_at_50": 0.6866666666666666, |
| "chance": 0.2, |
| "mean_conf": 0.642160177230835, |
| "sec": 1.1, |
| "heldout": false |
| }, |
| "yelp": { |
| "n": 1500, |
| "acc": 0.6826666666666666, |
| "nll": 0.7469708919525146, |
| "brier": 0.42934244871139526, |
| "ece": 0.08389960529406866, |
| "aurc": 0.1661291012599122, |
| "acc_at_80": 0.7408333333333333, |
| "acc_at_50": 0.816, |
| "chance": 0.2, |
| "mean_conf": 0.765559732913971, |
| "sec": 2.3, |
| "heldout": false |
| }, |
| "amazon_stars": { |
| "n": 1500, |
| "acc": 0.6033333333333334, |
| "nll": 0.9380196332931519, |
| "brier": 0.5161817669868469, |
| "ece": 0.07665159477790198, |
| "aurc": 0.23412008065702158, |
| "acc_at_80": 0.6541666666666667, |
| "acc_at_50": 0.748, |
| "chance": 0.2, |
| "mean_conf": 0.6799849271774292, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "emotion": { |
| "n": 1500, |
| "acc": 0.8693333333333333, |
| "nll": 0.375094473361969, |
| "brier": 0.1965477466583252, |
| "ece": 0.047137337585290234, |
| "aurc": 0.031846251103281674, |
| "acc_at_80": 0.9383333333333334, |
| "acc_at_50": 0.98, |
| "chance": 0.16666666666666666, |
| "mean_conf": 0.9126279950141907, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "go_emotions": { |
| "n": 1500, |
| "acc": 0.7793333333333333, |
| "nll": 0.6534133553504944, |
| "brier": 0.3117011487483978, |
| "ece": 0.04330594710508986, |
| "aurc": 0.07619826478481186, |
| "acc_at_80": 0.8633333333333333, |
| "acc_at_50": 0.936, |
| "chance": 0.1, |
| "mean_conf": 0.8226393461227417, |
| "sec": 1.2, |
| "heldout": false |
| }, |
| "tweet_sentiment": { |
| "n": 1500, |
| "acc": 0.7206666666666667, |
| "nll": 0.6113293170928955, |
| "brier": 0.37921401858329773, |
| "ece": 0.04055946866671242, |
| "aurc": 0.1466257866538856, |
| "acc_at_80": 0.765, |
| "acc_at_50": 0.8346666666666667, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.7430600523948669, |
| "sec": 0.9, |
| "heldout": false |
| }, |
| "tweet_emotion": { |
| "n": 1421, |
| "acc": 0.8367346938775511, |
| "nll": 0.45104628801345825, |
| "brier": 0.2379906326532364, |
| "ece": 0.025843723171785783, |
| "aurc": 0.04399644830028519, |
| "acc_at_80": 0.9076517150395779, |
| "acc_at_50": 0.971830985915493, |
| "chance": 0.25, |
| "mean_conf": 0.8583319187164307, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "tweet_irony": { |
| "n": 784, |
| "acc": 0.7946428571428571, |
| "nll": 0.4676361083984375, |
| "brier": 0.300514280796051, |
| "ece": 0.035634773954445, |
| "aurc": 0.1096303514372389, |
| "acc_at_80": 0.8437001594896332, |
| "acc_at_50": 0.8647959183673469, |
| "chance": 0.5, |
| "mean_conf": 0.8008275032043457, |
| "sec": 0.5, |
| "heldout": true |
| }, |
| "fin_sentiment": { |
| "n": 1500, |
| "acc": 0.7933333333333333, |
| "nll": 0.4863547086715698, |
| "brier": 0.28904733061790466, |
| "ece": 0.019916596571604408, |
| "aurc": 0.08617910313882864, |
| "acc_at_80": 0.8575, |
| "acc_at_50": 0.9146666666666666, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.7970937490463257, |
| "sec": 1.0, |
| "heldout": true |
| }, |
| "cr_reviews": { |
| "n": 753, |
| "acc": 0.9030544488711819, |
| "nll": 0.25531020760536194, |
| "brier": 0.14807458221912384, |
| "ece": 0.05294444131977842, |
| "aurc": 0.018211001734510564, |
| "acc_at_80": 0.9651162790697675, |
| "acc_at_50": 0.9946808510638298, |
| "chance": 0.5, |
| "mean_conf": 0.9507516026496887, |
| "sec": 0.5, |
| "heldout": true |
| }, |
| "counterfactual": { |
| "n": 1500, |
| "acc": 0.954, |
| "nll": 0.14455512166023254, |
| "brier": 0.0772322341799736, |
| "ece": 0.017266486525535557, |
| "aurc": 0.007466418318042988, |
| "acc_at_80": 0.9866666666666667, |
| "acc_at_50": 0.9973333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.9662280082702637, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "subjectivity": { |
| "n": 1500, |
| "acc": 0.9593333333333334, |
| "nll": 0.11742740124464035, |
| "brier": 0.0635886937379837, |
| "ece": 0.018429159760475204, |
| "aurc": 0.004805976587222085, |
| "acc_at_80": 0.9916666666666667, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.9714123010635376, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "tweet_offensive": { |
| "n": 860, |
| "acc": 0.8569767441860465, |
| "nll": 0.321306049823761, |
| "brier": 0.19854530692100525, |
| "ece": 0.03273247341777005, |
| "aurc": 0.04094359780914508, |
| "acc_at_80": 0.9215116279069767, |
| "acc_at_50": 0.9674418604651163, |
| "chance": 0.5, |
| "mean_conf": 0.8376211524009705, |
| "sec": 0.6, |
| "heldout": false |
| }, |
| "tweet_hate": { |
| "n": 1500, |
| "acc": 0.5613333333333334, |
| "nll": 0.984972357749939, |
| "brier": 0.643694281578064, |
| "ece": 0.2259776959021886, |
| "aurc": 0.43112132729844205, |
| "acc_at_80": 0.5641666666666667, |
| "acc_at_50": 0.5213333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.7813499569892883, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "hate_offensive": { |
| "n": 1500, |
| "acc": 0.912, |
| "nll": 0.24573880434036255, |
| "brier": 0.13170845806598663, |
| "ece": 0.0208393304944038, |
| "aurc": 0.018957146656246305, |
| "acc_at_80": 0.9683333333333334, |
| "acc_at_50": 0.9866666666666667, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.9264606237411499, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "civil_comments": { |
| "n": 7500, |
| "acc": 0.9330666666666667, |
| "nll": 0.16645771265029907, |
| "brier": 0.09957150369882584, |
| "ece": 0.022440280524889583, |
| "aurc": 0.008493115215878448, |
| "acc_at_80": 0.9856666666666667, |
| "acc_at_50": 1.0, |
| "chance": 0.5, |
| "mean_conf": 0.955498218536377, |
| "sec": 2.1, |
| "heldout": false, |
| "per_q": [ |
| 0.8733333333333333, |
| 0.9766666666666667, |
| 0.9906666666666667, |
| 0.85, |
| 0.9746666666666667 |
| ] |
| }, |
| "toxic_chat": { |
| "n": 3000, |
| "acc": 0.9753333333333334, |
| "nll": 0.0650152713060379, |
| "brier": 0.035060759633779526, |
| "ece": 0.011077121953169521, |
| "aurc": 0.0014917009140776134, |
| "acc_at_80": 0.99875, |
| "acc_at_50": 0.9993333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.985869288444519, |
| "sec": 1.4, |
| "heldout": false, |
| "per_q": [ |
| 0.958, |
| 0.9926666666666667 |
| ] |
| }, |
| "sms_spam": { |
| "n": 1000, |
| "acc": 0.985, |
| "nll": 0.06413730978965759, |
| "brier": 0.02651730179786682, |
| "ece": 0.013071102797985047, |
| "aurc": 0.0017184124196473523, |
| "acc_at_80": 0.9975, |
| "acc_at_50": 0.998, |
| "chance": 0.5, |
| "mean_conf": 0.9963533878326416, |
| "sec": 0.7, |
| "heldout": false |
| }, |
| "enron_spam": { |
| "n": 1500, |
| "acc": 0.9853333333333333, |
| "nll": 0.05048864707350731, |
| "brier": 0.023362739011645317, |
| "ece": 0.009452791094779985, |
| "aurc": 0.0010555480417750408, |
| "acc_at_80": 0.9991666666666666, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.9947861433029175, |
| "sec": 2.9, |
| "heldout": false |
| }, |
| "insincere_questions": { |
| "n": 1500, |
| "acc": 0.952, |
| "nll": 0.11656896770000458, |
| "brier": 0.06938926875591278, |
| "ece": 0.02098088498910269, |
| "aurc": 0.004221551290811616, |
| "acc_at_80": 0.9983333333333333, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.9717850089073181, |
| "sec": 0.9, |
| "heldout": false |
| }, |
| "ade": { |
| "n": 1500, |
| "acc": 0.8173333333333334, |
| "nll": 0.4560953378677368, |
| "brier": 0.2800695300102234, |
| "ece": 0.0687519727150599, |
| "aurc": 0.07776987989583131, |
| "acc_at_80": 0.8625, |
| "acc_at_50": 0.9173333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.8821967840194702, |
| "sec": 1.0, |
| "heldout": true |
| }, |
| "snli": { |
| "n": 1476, |
| "acc": 0.9214092140921409, |
| "nll": 0.2496514618396759, |
| "brier": 0.12413201481103897, |
| "ece": 0.02373483920404258, |
| "aurc": 0.02304695057683426, |
| "acc_at_80": 0.9669771380186283, |
| "acc_at_50": 0.981029810298103, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.9317938089370728, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "mnli": { |
| "n": 1500, |
| "acc": 0.8846666666666667, |
| "nll": 0.3279857933521271, |
| "brier": 0.17651496827602386, |
| "ece": 0.02796298484007516, |
| "aurc": 0.02839381927398111, |
| "acc_at_80": 0.9441666666666667, |
| "acc_at_50": 0.9853333333333333, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.9101898074150085, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "rte": { |
| "n": 277, |
| "acc": 0.8664259927797834, |
| "nll": 0.3891109526157379, |
| "brier": 0.2202744483947754, |
| "ece": 0.06889465591107032, |
| "aurc": 0.05497760440143546, |
| "acc_at_80": 0.9144144144144144, |
| "acc_at_50": 0.9565217391304348, |
| "chance": 0.5, |
| "mean_conf": 0.9213083386421204, |
| "sec": 0.3, |
| "heldout": false |
| }, |
| "qnli": { |
| "n": 1500, |
| "acc": 0.938, |
| "nll": 0.1725793182849884, |
| "brier": 0.09701354801654816, |
| "ece": 0.018119730949401894, |
| "aurc": 0.01105178177734522, |
| "acc_at_80": 0.9825, |
| "acc_at_50": 0.9946666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.9547027945518494, |
| "sec": 1.2, |
| "heldout": false |
| }, |
| "qqp": { |
| "n": 1500, |
| "acc": 0.8486666666666667, |
| "nll": 0.33667847514152527, |
| "brier": 0.21694913506507874, |
| "ece": 0.027132281541824343, |
| "aurc": 0.04492686265704937, |
| "acc_at_80": 0.9008333333333334, |
| "acc_at_50": 0.9706666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.8645197153091431, |
| "sec": 1.1, |
| "heldout": false |
| }, |
| "mrpc": { |
| "n": 408, |
| "acc": 0.8480392156862745, |
| "nll": 0.3339563310146332, |
| "brier": 0.2105294018983841, |
| "ece": 0.04274974734175441, |
| "aurc": 0.04511716527430815, |
| "acc_at_80": 0.9079754601226994, |
| "acc_at_50": 0.9754901960784313, |
| "chance": 0.5, |
| "mean_conf": 0.8559635877609253, |
| "sec": 0.4, |
| "heldout": false |
| }, |
| "paws": { |
| "n": 1500, |
| "acc": 0.724, |
| "nll": 0.694682776927948, |
| "brier": 0.4271811544895172, |
| "ece": 0.17940687636534372, |
| "aurc": 0.12480813768846086, |
| "acc_at_80": 0.78, |
| "acc_at_50": 0.8853333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.9034069180488586, |
| "sec": 1.3, |
| "heldout": true |
| }, |
| "cola": { |
| "n": 1043, |
| "acc": 0.8015340364333653, |
| "nll": 0.4175913631916046, |
| "brier": 0.2705059051513672, |
| "ece": 0.027572413876095537, |
| "aurc": 0.07668882275831405, |
| "acc_at_80": 0.8657074340527577, |
| "acc_at_50": 0.9329501915708812, |
| "chance": 0.5, |
| "mean_conf": 0.8236285448074341, |
| "sec": 0.6, |
| "heldout": false |
| }, |
| "boolq": { |
| "n": 1500, |
| "acc": 0.876, |
| "nll": 0.357710599899292, |
| "brier": 0.19683362543582916, |
| "ece": 0.059815298040707926, |
| "aurc": 0.03911365860036364, |
| "acc_at_80": 0.9241666666666667, |
| "acc_at_50": 0.9693333333333334, |
| "chance": 0.5, |
| "mean_conf": 0.935267984867096, |
| "sec": 1.9, |
| "heldout": false |
| }, |
| "strategyqa": { |
| "n": 687, |
| "acc": 0.5516739446870451, |
| "nll": 0.7452422380447388, |
| "brier": 0.5299192070960999, |
| "ece": 0.17586313932049571, |
| "aurc": 0.32907532533270495, |
| "acc_at_80": 0.5745454545454546, |
| "acc_at_50": 0.6366279069767442, |
| "chance": 0.5, |
| "mean_conf": 0.7275370359420776, |
| "sec": 0.4, |
| "heldout": true |
| }, |
| "pubmedqa": { |
| "n": 500, |
| "acc": 0.756, |
| "nll": 0.7593856453895569, |
| "brier": 0.3763442933559418, |
| "ece": 0.13125784403085708, |
| "aurc": 0.11776619019690367, |
| "acc_at_80": 0.8375, |
| "acc_at_50": 0.908, |
| "chance": 0.33333333333333326, |
| "mean_conf": 0.8847917318344116, |
| "sec": 1.2, |
| "heldout": true |
| }, |
| "arc": { |
| "n": 1172, |
| "acc": 0.8609215017064846, |
| "nll": 0.3946582078933716, |
| "brier": 0.20159626007080078, |
| "ece": 0.03699784692847281, |
| "aurc": 0.029178906832842126, |
| "acc_at_80": 0.9402985074626866, |
| "acc_at_50": 0.9863481228668942, |
| "chance": 0.25015642775881686, |
| "mean_conf": 0.8952547311782837, |
| "sec": 1.0, |
| "heldout": false |
| }, |
| "commonsense_qa": { |
| "n": 1221, |
| "acc": 0.7461097461097461, |
| "nll": 0.657385528087616, |
| "brier": 0.3438973128795624, |
| "ece": 0.06063837679461509, |
| "aurc": 0.08669029470544302, |
| "acc_at_80": 0.8311156601842374, |
| "acc_at_50": 0.9311475409836065, |
| "chance": 0.19999999999999998, |
| "mean_conf": 0.8057844042778015, |
| "sec": 0.8, |
| "heldout": false |
| }, |
| "qasc": { |
| "n": 926, |
| "acc": 0.8012958963282938, |
| "nll": 0.612847626209259, |
| "brier": 0.2819903790950775, |
| "ece": 0.02709557122468436, |
| "aurc": 0.05141996432246003, |
| "acc_at_80": 0.8866396761133604, |
| "acc_at_50": 0.9827213822894169, |
| "chance": 0.125, |
| "mean_conf": 0.7910774946212769, |
| "sec": 0.8, |
| "heldout": false |
| }, |
| "openbookqa": { |
| "n": 500, |
| "acc": 0.816, |
| "nll": 0.5107881426811218, |
| "brier": 0.2631987929344177, |
| "ece": 0.033833789587020886, |
| "aurc": 0.05619724629150928, |
| "acc_at_80": 0.8925, |
| "acc_at_50": 0.976, |
| "chance": 0.25, |
| "mean_conf": 0.8462465405464172, |
| "sec": 0.3, |
| "heldout": false |
| }, |
| "sciq": { |
| "n": 1000, |
| "acc": 0.982, |
| "nll": 0.04754561558365822, |
| "brier": 0.023483367636799812, |
| "ece": 0.014726210862398123, |
| "aurc": 0.000321545900436324, |
| "acc_at_80": 1.0, |
| "acc_at_50": 1.0, |
| "chance": 0.25, |
| "mean_conf": 0.9794425368309021, |
| "sec": 1.1, |
| "heldout": true |
| }, |
| "hellaswag": { |
| "n": 1500, |
| "acc": 0.866, |
| "nll": 0.3764917254447937, |
| "brier": 0.19079357385635376, |
| "ece": 0.06253247086207067, |
| "aurc": 0.02407728816052661, |
| "acc_at_80": 0.9525, |
| "acc_at_50": 0.9946666666666667, |
| "chance": 0.25, |
| "mean_conf": 0.9285325407981873, |
| "sec": 2.2, |
| "heldout": false |
| }, |
| "piqa": { |
| "n": 1500, |
| "acc": 0.83, |
| "nll": 0.3894118666648865, |
| "brier": 0.24019156396389008, |
| "ece": 0.04349609518051145, |
| "aurc": 0.0600857158889656, |
| "acc_at_80": 0.8941666666666667, |
| "acc_at_50": 0.944, |
| "chance": 0.5, |
| "mean_conf": 0.8716477751731873, |
| "sec": 1.2, |
| "heldout": false |
| }, |
| "social_iqa": { |
| "n": 1500, |
| "acc": 0.708, |
| "nll": 0.7983387112617493, |
| "brier": 0.4352244734764099, |
| "ece": 0.11886229503154755, |
| "aurc": 0.15761526845761895, |
| "acc_at_80": 0.7558333333333334, |
| "acc_at_50": 0.844, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.8267069458961487, |
| "sec": 1.1, |
| "heldout": true |
| }, |
| "winogrande": { |
| "n": 1267, |
| "acc": 0.6921862667719021, |
| "nll": 0.8599734902381897, |
| "brier": 0.48017874360084534, |
| "ece": 0.18726083422234047, |
| "aurc": 0.20496797159884195, |
| "acc_at_80": 0.7268244575936884, |
| "acc_at_50": 0.7823343848580442, |
| "chance": 0.5, |
| "mean_conf": 0.879352331161499, |
| "sec": 0.8, |
| "heldout": false |
| }, |
| "race": { |
| "n": 1500, |
| "acc": 0.8286666666666667, |
| "nll": 0.5294487476348877, |
| "brier": 0.2535547614097595, |
| "ece": 0.08820948191483813, |
| "aurc": 0.0441451468912063, |
| "acc_at_80": 0.9191666666666667, |
| "acc_at_50": 0.976, |
| "chance": 0.25, |
| "mean_conf": 0.9168761372566223, |
| "sec": 3.8, |
| "heldout": false |
| }, |
| "mmlu": { |
| "n": 1500, |
| "acc": 0.6293333333333333, |
| "nll": 0.9297420382499695, |
| "brier": 0.47942283749580383, |
| "ece": 0.1056801940202713, |
| "aurc": 0.15805034675139773, |
| "acc_at_80": 0.7116666666666667, |
| "acc_at_50": 0.8626666666666667, |
| "chance": 0.25, |
| "mean_conf": 0.735013484954834, |
| "sec": 1.5, |
| "heldout": false |
| }, |
| "medqa": { |
| "n": 1273, |
| "acc": 0.5443833464257659, |
| "nll": 1.2023534774780273, |
| "brier": 0.6267179250717163, |
| "ece": 0.18061440218553143, |
| "aurc": 0.2882548640060218, |
| "acc_at_80": 0.5943025540275049, |
| "acc_at_50": 0.6808176100628931, |
| "chance": 0.25, |
| "mean_conf": 0.7214598655700684, |
| "sec": 2.0, |
| "heldout": false |
| }, |
| "truthfulqa": { |
| "n": 817, |
| "acc": 0.5373317013463892, |
| "nll": 1.5506761074066162, |
| "brier": 0.6525235176086426, |
| "ece": 0.16036296565646493, |
| "aurc": 0.2660050026273411, |
| "acc_at_80": 0.599388379204893, |
| "acc_at_50": 0.7279411764705882, |
| "chance": 0.22622350449767833, |
| "mean_conf": 0.6951266527175903, |
| "sec": 0.7, |
| "heldout": true |
| }, |
| "fin_phrasebank": { |
| "n": 970, |
| "acc": 0.6938144329896907, |
| "nll": 0.710655927658081, |
| "brier": 0.432155579328537, |
| "ece": 0.09275854675425697, |
| "aurc": 0.1946467193061562, |
| "acc_at_80": 0.7332474226804123, |
| "acc_at_50": 0.8061855670103093, |
| "chance": 0.33333333333333326, |
| "mean_conf": 0.7770669460296631, |
| "sec": 2.0, |
| "heldout": true |
| }, |
| "bias_in_bios": { |
| "n": 3000, |
| "acc": 0.9513333333333334, |
| "nll": 0.1453825682401657, |
| "brier": 0.0709514245390892, |
| "ece": 0.013315296888351444, |
| "aurc": 0.0035279360929807786, |
| "acc_at_80": 0.99625, |
| "acc_at_50": 1.0, |
| "chance": 0.30000000000000004, |
| "mean_conf": 0.9636430740356445, |
| "sec": 2.0, |
| "heldout": false, |
| "per_q": [ |
| 0.9053333333333333, |
| 0.9973333333333333 |
| ] |
| }, |
| "helpsteer2": { |
| "n": 5190, |
| "acc": 0.5895953757225434, |
| "nll": 0.9746355414390564, |
| "brier": 0.5325403809547424, |
| "ece": 0.057795672655564984, |
| "aurc": 0.2721544202844174, |
| "acc_at_80": 0.6536608863198459, |
| "acc_at_50": 0.735645472061657, |
| "chance": 0.2, |
| "mean_conf": 0.6461822986602783, |
| "sec": 4.8, |
| "heldout": false, |
| "per_q": [ |
| 0.4325626204238921, |
| 0.48554913294797686, |
| 0.7186897880539499, |
| 0.6204238921001927, |
| 0.6907514450867052 |
| ] |
| }, |
| "helpsteer3_pref": { |
| "n": 1176, |
| "acc": 0.407312925170068, |
| "nll": 1.518996000289917, |
| "brier": 0.7332671284675598, |
| "ece": 0.1154663114726138, |
| "aurc": 0.5171907165068561, |
| "acc_at_80": 0.4303931987247609, |
| "acc_at_50": 0.46258503401360546, |
| "chance": 0.14285714285714282, |
| "mean_conf": 0.5227791666984558, |
| "sec": 6.4, |
| "heldout": false |
| }, |
| "hate_speech_scales": { |
| "n": 7500, |
| "acc": 0.554, |
| "nll": 1.0020463466644287, |
| "brier": 0.5518749952316284, |
| "ece": 0.05263813356558482, |
| "aurc": 0.2725105506447112, |
| "acc_at_80": 0.6043333333333333, |
| "acc_at_50": 0.7026666666666667, |
| "chance": 0.22666666666666674, |
| "mean_conf": 0.6066381335258484, |
| "sec": 3.0, |
| "heldout": false, |
| "per_q": [ |
| 0.7033333333333334, |
| 0.472, |
| 0.43866666666666665, |
| 0.5686666666666667, |
| 0.5873333333333334 |
| ] |
| }, |
| "liar2": { |
| "n": 1500, |
| "acc": 0.37066666666666664, |
| "nll": 1.487428069114685, |
| "brier": 0.7230392098426819, |
| "ece": 0.042378827313582085, |
| "aurc": 0.48226130915305326, |
| "acc_at_80": 0.4008333333333333, |
| "acc_at_50": 0.472, |
| "chance": 0.16666666666666666, |
| "mean_conf": 0.4129585921764374, |
| "sec": 1.9, |
| "heldout": false |
| }, |
| "prosocial_safety": { |
| "n": 1500, |
| "acc": 0.5126666666666667, |
| "nll": 1.1911370754241943, |
| "brier": 0.601366400718689, |
| "ece": 0.04166399759054183, |
| "aurc": 0.28211418515491415, |
| "acc_at_80": 0.5758333333333333, |
| "acc_at_50": 0.6933333333333334, |
| "chance": 0.2, |
| "mean_conf": 0.541668713092804, |
| "sec": 1.3, |
| "heldout": false |
| }, |
| "ultrafeedback_pref": { |
| "n": 1500, |
| "acc": 0.7506666666666667, |
| "nll": 0.5978830456733704, |
| "brier": 0.3642973303794861, |
| "ece": 0.12037857890129089, |
| "aurc": 0.12885069203247962, |
| "acc_at_80": 0.8033333333333333, |
| "acc_at_50": 0.864, |
| "chance": 0.5, |
| "mean_conf": 0.8620939254760742, |
| "sec": 5.2, |
| "heldout": false |
| }, |
| "shp": { |
| "n": 1500, |
| "acc": 0.7433333333333333, |
| "nll": 0.5542265176773071, |
| "brier": 0.3617444932460785, |
| "ece": 0.08052628513177235, |
| "aurc": 0.1415611995055057, |
| "acc_at_80": 0.7875, |
| "acc_at_50": 0.8613333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.8218792080879211, |
| "sec": 3.8, |
| "heldout": false |
| }, |
| "hh_rlhf": { |
| "n": 1488, |
| "acc": 0.6283602150537635, |
| "nll": 0.7063506841659546, |
| "brier": 0.48729172348976135, |
| "ece": 0.1238586102682416, |
| "aurc": 0.2775212031705402, |
| "acc_at_80": 0.6470588235294118, |
| "acc_at_50": 0.6975806451612904, |
| "chance": 0.5, |
| "mean_conf": 0.745833694934845, |
| "sec": 3.1, |
| "heldout": false |
| }, |
| "arena_pref": { |
| "n": 1500, |
| "acc": 0.4826666666666667, |
| "nll": 1.3242307901382446, |
| "brier": 0.7265019416809082, |
| "ece": 0.24603747620185218, |
| "aurc": 0.42174124750905745, |
| "acc_at_80": 0.5166666666666667, |
| "acc_at_50": 0.564, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.7287040948867798, |
| "sec": 4.9, |
| "heldout": true |
| }, |
| "reward_bench": { |
| "n": 1500, |
| "acc": 0.8186666666666667, |
| "nll": 0.4543558359146118, |
| "brier": 0.274769127368927, |
| "ece": 0.07598515292008719, |
| "aurc": 0.07768967728506293, |
| "acc_at_80": 0.8666666666666667, |
| "acc_at_50": 0.9333333333333333, |
| "chance": 0.5, |
| "mean_conf": 0.8845229744911194, |
| "sec": 3.9, |
| "heldout": true |
| }, |
| "glaive_tools": { |
| "n": 1348, |
| "acc": 0.9413946587537092, |
| "nll": 0.19328154623508453, |
| "brier": 0.10143861919641495, |
| "ece": 0.022093097250963853, |
| "aurc": 0.01690886528489672, |
| "acc_at_80": 0.9693877551020408, |
| "acc_at_50": 0.983679525222552, |
| "chance": 0.16666666666666666, |
| "mean_conf": 0.955422580242157, |
| "sec": 1.6, |
| "heldout": false |
| }, |
| "toolace": { |
| "n": 1000, |
| "acc": 0.931, |
| "nll": 0.2512577474117279, |
| "brier": 0.11597812175750732, |
| "ece": 0.031337061107158684, |
| "aurc": 0.02448370972130845, |
| "acc_at_80": 0.97, |
| "acc_at_50": 0.982, |
| "chance": 0.12470555555555554, |
| "mean_conf": 0.9608047604560852, |
| "sec": 2.0, |
| "heldout": false |
| }, |
| "hermes_tools": { |
| "n": 1500, |
| "acc": 0.7226666666666667, |
| "nll": 0.8366944789886475, |
| "brier": 0.464679479598999, |
| "ece": 0.23630806771914162, |
| "aurc": 0.06836947758114767, |
| "acc_at_80": 0.8566666666666667, |
| "acc_at_50": 0.984, |
| "chance": 0.125, |
| "mean_conf": 0.9589747786521912, |
| "sec": 4.7, |
| "heldout": true |
| }, |
| "copa": { |
| "n": 100, |
| "acc": 0.93, |
| "nll": 0.20605678856372833, |
| "brier": 0.11243906617164612, |
| "ece": 0.028781158924102764, |
| "aurc": 0.019294681675211874, |
| "acc_at_80": 0.975, |
| "acc_at_50": 0.98, |
| "chance": 0.5, |
| "mean_conf": 0.9292109608650208, |
| "sec": 0.1, |
| "heldout": false |
| }, |
| "wic": { |
| "n": 638, |
| "acc": 0.6943573667711599, |
| "nll": 0.6192224621772766, |
| "brier": 0.41049328446388245, |
| "ece": 0.10207162110783091, |
| "aurc": 0.2013770101215354, |
| "acc_at_80": 0.7490196078431373, |
| "acc_at_50": 0.8056426332288401, |
| "chance": 0.5, |
| "mean_conf": 0.7964289784431458, |
| "sec": 0.4, |
| "heldout": false |
| }, |
| "multirc": { |
| "n": 1500, |
| "acc": 0.8873333333333333, |
| "nll": 0.353663831949234, |
| "brier": 0.18488313257694244, |
| "ece": 0.057831449985504116, |
| "aurc": 0.04080578362832766, |
| "acc_at_80": 0.9341666666666667, |
| "acc_at_50": 0.972, |
| "chance": 0.5, |
| "mean_conf": 0.9418337345123291, |
| "sec": 3.5, |
| "heldout": false |
| }, |
| "cb": { |
| "n": 56, |
| "acc": 0.8571428571428571, |
| "nll": 0.39587274193763733, |
| "brier": 0.1826748102903366, |
| "ece": 0.061279319226741805, |
| "aurc": 0.024518910173493138, |
| "acc_at_80": 0.9555555555555556, |
| "acc_at_50": 1.0, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.8873938322067261, |
| "sec": 0.1, |
| "heldout": true |
| }, |
| "fever": { |
| "n": 1500, |
| "acc": 0.888, |
| "nll": 0.3543616235256195, |
| "brier": 0.18396615982055664, |
| "ece": 0.05364456375439957, |
| "aurc": 0.035409527613622344, |
| "acc_at_80": 0.9416666666666667, |
| "acc_at_50": 0.9706666666666667, |
| "chance": 0.3333333333333333, |
| "mean_conf": 0.9349583387374878, |
| "sec": 1.4, |
| "heldout": false |
| }, |
| "wiki_qa": { |
| "n": 879, |
| "acc": 0.8953356086461889, |
| "nll": 0.2812374532222748, |
| "brier": 0.15986107289791107, |
| "ece": 0.03158905461379691, |
| "aurc": 0.029564777940039345, |
| "acc_at_80": 0.9473684210526315, |
| "acc_at_50": 0.9772727272727273, |
| "chance": 0.5, |
| "mean_conf": 0.9265106320381165, |
| "sec": 0.7, |
| "heldout": false |
| }, |
| "msmarco_rel": { |
| "n": 1446, |
| "acc": 0.6535269709543569, |
| "nll": 0.6131556034088135, |
| "brier": 0.42479565739631653, |
| "ece": 0.03628980136511236, |
| "aurc": 0.23044128645847725, |
| "acc_at_80": 0.6871218668971478, |
| "acc_at_50": 0.7565698478561549, |
| "chance": 0.5, |
| "mean_conf": 0.6898168325424194, |
| "sec": 1.5, |
| "heldout": false |
| }, |
| "medmcqa": { |
| "n": 1500, |
| "acc": 0.5, |
| "nll": 1.1675626039505005, |
| "brier": 0.6214771270751953, |
| "ece": 0.1292713791926702, |
| "aurc": 0.3062450473649221, |
| "acc_at_80": 0.5616666666666666, |
| "acc_at_50": 0.6666666666666666, |
| "chance": 0.25, |
| "mean_conf": 0.629271388053894, |
| "sec": 1.1, |
| "heldout": false |
| }, |
| "quality": { |
| "n": 1500, |
| "acc": 0.494, |
| "nll": 1.5713512897491455, |
| "brier": 0.7469174861907959, |
| "ece": 0.29271218756834666, |
| "aurc": 0.34188336200889935, |
| "acc_at_80": 0.545, |
| "acc_at_50": 0.6346666666666667, |
| "chance": 0.25, |
| "mean_conf": 0.7867121696472168, |
| "sec": 10.7, |
| "heldout": true |
| }, |
| "xstory_cloze": { |
| "n": 1500, |
| "acc": 0.962, |
| "nll": 0.10309821367263794, |
| "brier": 0.05630293861031532, |
| "ece": 0.00773787120978037, |
| "aurc": 0.004358402846713855, |
| "acc_at_80": 0.9941666666666666, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.5, |
| "mean_conf": 0.9653077125549316, |
| "sec": 1.3, |
| "heldout": true |
| }, |
| "abstain_probe": { |
| "n": 1620, |
| "acc": 0.6055555555555555, |
| "nll": 1.1703203916549683, |
| "brier": 0.5663847923278809, |
| "ece": 0.19228262847956318, |
| "aurc": 0.20466473892179798, |
| "acc_at_80": 0.6929012345679012, |
| "acc_at_50": 0.7851851851851852, |
| "chance": 0.17897217323143252, |
| "mean_conf": 0.7978381514549255, |
| "sec": 1.9, |
| "heldout": true |
| }, |
| "agenttraj": { |
| "n": 1500, |
| "acc": 0.916, |
| "nll": 0.22318525612354279, |
| "brier": 0.11291919648647308, |
| "ece": 0.03268852589527768, |
| "aurc": 0.009361436014432775, |
| "acc_at_80": 0.9858333333333333, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.2, |
| "mean_conf": 0.9431377053260803, |
| "sec": 3.4, |
| "heldout": false |
| }, |
| "mind2web": { |
| "n": 1500, |
| "acc": 0.8266666666666667, |
| "nll": 0.49047866463661194, |
| "brier": 0.25011762976646423, |
| "ece": 0.056447255154450725, |
| "aurc": 0.04288666994842876, |
| "acc_at_80": 0.9033333333333333, |
| "acc_at_50": 0.9813333333333333, |
| "chance": 0.16666666666666666, |
| "mean_conf": 0.8782970309257507, |
| "sec": 2.0, |
| "heldout": false |
| }, |
| "synth": { |
| "n": 300, |
| "acc": 0.7933333333333333, |
| "nll": 0.6004238128662109, |
| "brier": 0.3256992995738983, |
| "ece": 0.07842869222164153, |
| "aurc": 0.08963059071585025, |
| "acc_at_80": 0.8458333333333333, |
| "acc_at_50": 0.8933333333333333, |
| "chance": 0.37149999999999994, |
| "mean_conf": 0.8574635982513428, |
| "sec": 0.2, |
| "heldout": false, |
| "per_q": [ |
| 0.7666666666666667, |
| 0.82 |
| ] |
| }, |
| "offtopic_probe": { |
| "n": 1620, |
| "acc": 0.841358024691358, |
| "nll": 0.5042567253112793, |
| "brier": 0.22333061695098877, |
| "ece": 0.04802086000089292, |
| "aurc": 0.030512282206217035, |
| "acc_at_80": 0.9405864197530864, |
| "acc_at_50": 0.9888888888888889, |
| "chance": 0.17979423868312755, |
| "mean_conf": 0.8893788456916809, |
| "sec": 1.8, |
| "heldout": true |
| }, |
| "hwu64": { |
| "n": 1076, |
| "acc": 0.9609665427509294, |
| "nll": 0.12583288550376892, |
| "brier": 0.06144118681550026, |
| "ece": 0.009324527861460477, |
| "aurc": 0.003260867906700039, |
| "acc_at_80": 0.9953542392566783, |
| "acc_at_50": 1.0, |
| "chance": 0.09999999999999999, |
| "mean_conf": 0.9562390446662903, |
| "sec": 0.9, |
| "heldout": true |
| }, |
| "dbpedia_l2": { |
| "n": 1500, |
| "acc": 0.95, |
| "nll": 0.16574443876743317, |
| "brier": 0.08210423588752747, |
| "ece": 0.029746596733729078, |
| "aurc": 0.005207273669368498, |
| "acc_at_80": 0.9941666666666666, |
| "acc_at_50": 1.0, |
| "chance": 0.1, |
| "mean_conf": 0.9745652675628662, |
| "sec": 2.2, |
| "heldout": true |
| }, |
| "dbpedia_l3": { |
| "n": 1500, |
| "acc": 0.9866666666666667, |
| "nll": 0.054405681788921356, |
| "brier": 0.022596782073378563, |
| "ece": 0.007581249455610876, |
| "aurc": 0.0009766034191328878, |
| "acc_at_80": 0.9991666666666666, |
| "acc_at_50": 0.9986666666666667, |
| "chance": 0.1, |
| "mean_conf": 0.9932629466056824, |
| "sec": 2.2, |
| "heldout": true |
| }, |
| "quality_full": { |
| "n": 600, |
| "acc": 0.5083333333333333, |
| "nll": 1.5723801851272583, |
| "brier": 0.7437332272529602, |
| "ece": 0.261873353322347, |
| "aurc": 0.371980887569386, |
| "acc_at_80": 0.5333333333333333, |
| "acc_at_50": 0.6166666666666667, |
| "chance": 0.25, |
| "mean_conf": 0.7702066898345947, |
| "sec": 5.4, |
| "heldout": true |
| } |
| }, |
| "agg": { |
| "in_task": { |
| "acc": 0.8053939243254699, |
| "nll": 0.4901052765242422, |
| "brier": 0.26693312430392896, |
| "ece": 0.051122906597442695, |
| "aurc": 0.09948275075275431, |
| "acc_at_80": 0.8563021912710674, |
| "chance": 0.32540293886491484 |
| }, |
| "heldout": { |
| "acc": 0.7550550130420666, |
| "nll": 0.6816884187449302, |
| "brier": 0.3505160380154848, |
| "ece": 0.10750615544056343, |
| "aurc": 0.134422784017669, |
| "acc_at_80": 0.8086450646710732, |
| "chance": 0.2893448779671037 |
| }, |
| "n_in": 67, |
| "n_heldout": 28 |
| }, |
| "model": "/mnt/data/mark/projects/decider/runs/calibrated_rl_2b_v2/control_9419/checkpoint-384", |
| "engine": "" |
| } |
