Invalid JSON:Unexpected token 'N', ..."fidence": NaN,
"... is not valid JSON
| { | |
| "overall": { | |
| "count": 5513, | |
| "accuracy": 0.8342100489751496, | |
| "brier": 0.24499939603673204, | |
| "brier_soft": 0.22122350305548508, | |
| "nll": 0.4655553015844398, | |
| "nll_soft": 0.48562642001788464, | |
| "ece": 0.04843755672557183, | |
| "mean_confidence": 0.8631851509248467, | |
| "confident_error_rate": 0.0359151097406131, | |
| "score_mae": 0.4705968118889738, | |
| "evidence_auc": 0.9999878244320566 | |
| }, | |
| "by_type": { | |
| "bool": { | |
| "count": 1620, | |
| "accuracy": 0.8753086419753087, | |
| "brier": 0.19217866542576822, | |
| "brier_soft": 0.18932657502523315, | |
| "nll": 0.3103588304174498, | |
| "nll_soft": 0.3175118193167458, | |
| "ece": 0.06666441711019516, | |
| "mean_confidence": 0.9396361920258015, | |
| "confident_error_rate": 0.043209876543209874, | |
| "evidence_auc": 0.9999940298151033 | |
| }, | |
| "choice": { | |
| "count": 2993, | |
| "accuracy": 0.8529903107250251, | |
| "brier": 0.2219172131885553, | |
| "brier_soft": 0.19782538508715383, | |
| "nll": 0.4634021404898978, | |
| "nll_soft": 0.48052663665615447, | |
| "ece": 0.05453548012022721, | |
| "mean_confidence": 0.8982264049258726, | |
| "confident_error_rate": 0.04276645506181089, | |
| "evidence_auc": 0.9999655528763348 | |
| }, | |
| "score": { | |
| "count": 900, | |
| "accuracy": 0.6977777777777778, | |
| "brier": 0.41683779254157016, | |
| "brier_soft": 0.356449714709067, | |
| "nll": 0.7520694065249816, | |
| "nll_soft": 0.8051923141706663, | |
| "ece": 0.09215386988560845, | |
| "mean_confidence": 0.6090416400263835, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4705968118889738 | |
| } | |
| }, | |
| "by_source": { | |
| "ag_news": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.1465319479149763, | |
| "brier_soft": 0.1465319479149763, | |
| "nll": 0.2904767555416907, | |
| "nll_soft": 0.2904767555416907, | |
| "ece": 0.06696793567024037, | |
| "mean_confidence": 0.9411689492273619, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "amazon_polarity": { | |
| "count": 100, | |
| "accuracy": 0.95, | |
| "brier": 0.06128869305634376, | |
| "brier_soft": 0.06128869305634376, | |
| "nll": 0.10096164621475562, | |
| "nll_soft": 0.10096164621475562, | |
| "ece": 0.023046591891905697, | |
| "mean_confidence": 0.9730465918919056, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "amazon_stars": { | |
| "count": 100, | |
| "accuracy": 0.69, | |
| "brier": 0.4731412386648261, | |
| "brier_soft": 0.4731412386648261, | |
| "nll": 0.8151410149736901, | |
| "nll_soft": 0.8151410149736901, | |
| "ece": 0.1366735053126959, | |
| "mean_confidence": 0.579111282045835, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4509802678028108 | |
| }, | |
| "anli": { | |
| "count": 100, | |
| "accuracy": 0.74, | |
| "brier": 0.3976344462707522, | |
| "brier_soft": 0.3976344462707522, | |
| "nll": 0.7224163582147134, | |
| "nll_soft": 0.7224163582147134, | |
| "ece": 0.14788199940722557, | |
| "mean_confidence": 0.8672915745774106, | |
| "confident_error_rate": 0.08 | |
| }, | |
| "arc_challenge": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.03366315261301923, | |
| "brier_soft": 0.03366315261301923, | |
| "nll": 0.060837047229714225, | |
| "nll_soft": 0.060837047229714225, | |
| "ece": 0.011432103687330048, | |
| "mean_confidence": 0.9886713474416573, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "arc_easy": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.029996236913866735, | |
| "brier_soft": 0.029996236913866735, | |
| "nll": 0.05253505337779216, | |
| "nll_soft": 0.05253505337779216, | |
| "ece": 0.018028940214597153, | |
| "mean_confidence": 0.990060141873229, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "banking77": { | |
| "count": 100, | |
| "accuracy": 0.83, | |
| "brier": 0.28099173269454264, | |
| "brier_soft": 0.28099173269454264, | |
| "nll": 0.7554653567994057, | |
| "nll_soft": 0.7554653567994057, | |
| "ece": 0.11505295474931794, | |
| "mean_confidence": 0.8828400651254894, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "boolq": { | |
| "count": 125, | |
| "accuracy": 0.72, | |
| "brier": 0.43268232797354933, | |
| "brier_soft": 0.43268232797354933, | |
| "nll": 0.6422496879755143, | |
| "nll_soft": 0.6422496879755143, | |
| "ece": 0.22190986014776565, | |
| "mean_confidence": 0.9281403357012442, | |
| "confident_error_rate": 0.072, | |
| "evidence_auc": 1.0 | |
| }, | |
| "cb": { | |
| "count": 28, | |
| "accuracy": 0.9642857142857143, | |
| "brier": 0.07835667610715101, | |
| "brier_soft": 0.07835667610715101, | |
| "nll": 0.13341829300103206, | |
| "nll_soft": 0.13341829300103206, | |
| "ece": 0.05279191576259608, | |
| "mean_confidence": 0.954691373765687, | |
| "confident_error_rate": 0.03571428571428571 | |
| }, | |
| "civil_comments_severity": { | |
| "count": 100, | |
| "accuracy": 0.85, | |
| "brier": 0.21990075062806394, | |
| "brier_soft": 0.15256546144647498, | |
| "nll": 0.42595271409351193, | |
| "nll_soft": 0.5859219055739261, | |
| "ece": 0.1029050524606589, | |
| "mean_confidence": 0.7619406731562297, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3013470975763519 | |
| }, | |
| "civil_comments_toxic": { | |
| "count": 100, | |
| "accuracy": 0.96, | |
| "brier": 0.08184155550658603, | |
| "brier_soft": 0.035637691017917736, | |
| "nll": 0.15796842110857123, | |
| "nll_soft": 0.2738468412771658, | |
| "ece": 0.07203162905008202, | |
| "mean_confidence": 0.8907618087052214, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "clinc150": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.19125913762163368, | |
| "brier_soft": 0.19125913762163368, | |
| "nll": 0.5776643056115763, | |
| "nll_soft": 0.5776643056115763, | |
| "ece": 0.09283844266870002, | |
| "mean_confidence": 0.9496720060766602, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "cola": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.23386052759583184, | |
| "brier_soft": 0.23386052759583184, | |
| "nll": 0.37037031376428564, | |
| "nll_soft": 0.37037031376428564, | |
| "ece": 0.09766561804689824, | |
| "mean_confidence": 0.8833423357882064, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "commonsense_qa": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.15377198586333088, | |
| "brier_soft": 0.15377198586333088, | |
| "nll": 0.3819044445089102, | |
| "nll_soft": 0.3819044445089102, | |
| "ece": 0.05144685490055159, | |
| "mean_confidence": 0.9596385183475031, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "cosmos_qa": { | |
| "count": 120, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.4672354484316241, | |
| "brier_soft": 0.4672354484316241, | |
| "nll": 1.0482225583041105, | |
| "nll_soft": 1.0482225583041105, | |
| "ece": 0.19858953692704095, | |
| "mean_confidence": 0.8962658679831853, | |
| "confident_error_rate": 0.13333333333333333, | |
| "evidence_auc": 0.999375 | |
| }, | |
| "dbpedia": { | |
| "count": 100, | |
| "accuracy": 1.0, | |
| "brier": 0.0033691323280231667, | |
| "brier_soft": 0.0033691323280231667, | |
| "nll": 0.00628770684289355, | |
| "nll_soft": 0.00628770684289355, | |
| "ece": 0.0050215623881869585, | |
| "mean_confidence": 0.994978437611813, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "dynasent": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.37323258779066765, | |
| "brier_soft": 0.2605018455661368, | |
| "nll": 0.6422479980426224, | |
| "nll_soft": 0.7148729964174031, | |
| "ece": 0.15595018764547483, | |
| "mean_confidence": 0.6386131537774967, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.42700063346629696 | |
| }, | |
| "enron_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.020538910131957966, | |
| "brier_soft": 0.020538910131957966, | |
| "nll": 0.037125773454637925, | |
| "nll_soft": 0.037125773454637925, | |
| "ece": 0.018135869575440167, | |
| "mean_confidence": 0.9889960329597918, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "go_emotions": { | |
| "count": 100, | |
| "accuracy": 0.6, | |
| "brier": 0.5706133279962924, | |
| "brier_soft": 0.2146223457949257, | |
| "nll": 1.4243747447457458, | |
| "nll_soft": 1.9114536152236135, | |
| "ece": 0.2153529377728328, | |
| "mean_confidence": 0.4180754880267198, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "hellaswag": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04398586313752212, | |
| "brier_soft": 0.04398586313752212, | |
| "nll": 0.07282392796652562, | |
| "nll_soft": 0.07282392796652562, | |
| "ece": 0.03281997526145711, | |
| "mean_confidence": 0.967702411105094, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "imdb": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04503053337307919, | |
| "brier_soft": 0.04503053337307919, | |
| "nll": 0.10202723595434093, | |
| "nll_soft": 0.10202723595434093, | |
| "ece": 0.02799987491297936, | |
| "mean_confidence": 0.9735376597268748, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "ledgar": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.31022259775593414, | |
| "brier_soft": 0.31022259775593414, | |
| "nll": 0.7916681164712028, | |
| "nll_soft": 0.7916681164712028, | |
| "ece": 0.10685236902591677, | |
| "mean_confidence": 0.8791648234039952, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "liar": { | |
| "count": 100, | |
| "accuracy": 0.39, | |
| "brier": 0.7168465726369471, | |
| "brier_soft": 0.7168465726369471, | |
| "nll": 1.430973024742663, | |
| "nll_soft": 1.430973024742663, | |
| "ece": 0.07922166544125636, | |
| "mean_confidence": 0.33914869030092576, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.9383143759928696 | |
| }, | |
| "measuring_hate_speech": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2523259794626622, | |
| "brier_soft": 0.1350772440937763, | |
| "nll": 0.48511890737634117, | |
| "nll_soft": 0.6516605742694233, | |
| "ece": 0.1276134336558651, | |
| "mean_confidence": 0.7243118123610279, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.49301893575302613 | |
| }, | |
| "medqa": { | |
| "count": 100, | |
| "accuracy": 0.88, | |
| "brier": 0.15509559657024174, | |
| "brier_soft": 0.15509559657024174, | |
| "nll": 0.2630871286801779, | |
| "nll_soft": 0.2630871286801779, | |
| "ece": 0.07094760936712856, | |
| "mean_confidence": 0.9227796946829281, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "mnli": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.05622277315175722, | |
| "brier_soft": 0.05622277315175722, | |
| "nll": 0.11140887517203295, | |
| "nll_soft": 0.11140887517203295, | |
| "ece": 0.06498924366805267, | |
| "mean_confidence": 0.9358105562109016, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "mrpc": { | |
| "count": 100, | |
| "accuracy": 0.9, | |
| "brier": 0.16885290914097817, | |
| "brier_soft": 0.16885290914097817, | |
| "nll": 0.26678509569101083, | |
| "nll_soft": 0.26678509569101083, | |
| "ece": 0.056738491245727135, | |
| "mean_confidence": 0.9094846375493769, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "multirc": { | |
| "count": 125, | |
| "accuracy": 0.768, | |
| "brier": 0.4125322538820213, | |
| "brier_soft": 0.4125322538820213, | |
| "nll": 0.702692751642681, | |
| "nll_soft": 0.702692751642681, | |
| "ece": 0.2040484715924632, | |
| "mean_confidence": 0.9360141123782398, | |
| "confident_error_rate": 0.152, | |
| "evidence_auc": 0.9997333333333334 | |
| }, | |
| "openbookqa": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.036469910012703805, | |
| "brier_soft": 0.036469910012703805, | |
| "nll": 0.07196625626831986, | |
| "nll_soft": 0.07196625626831986, | |
| "ece": 0.02228237729925685, | |
| "mean_confidence": 0.977701842636366, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "piqa": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.12451330680810697, | |
| "brier_soft": 0.12451330680810697, | |
| "nll": 0.23446545298444263, | |
| "nll_soft": 0.23446545298444263, | |
| "ece": 0.06753078707740053, | |
| "mean_confidence": 0.9522836212325586, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "policy_application": { | |
| "count": 150, | |
| "accuracy": 0.9466666666666667, | |
| "brier": 0.0777876053884264, | |
| "brier_soft": 0.0777876053884264, | |
| "nll": 0.13521520720550428, | |
| "nll_soft": 0.13521520720550428, | |
| "ece": 0.04668170693287413, | |
| "mean_confidence": 0.985035205551867, | |
| "confident_error_rate": 0.03333333333333333, | |
| "evidence_auc": 1.0 | |
| }, | |
| "qqp": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.13043954738084676, | |
| "brier_soft": 0.13043954738084676, | |
| "nll": 0.22957888778440388, | |
| "nll_soft": 0.22957888778440388, | |
| "ece": 0.03919742572811998, | |
| "mean_confidence": 0.9198058774243847, | |
| "confident_error_rate": 0.04 | |
| }, | |
| "race": { | |
| "count": 125, | |
| "accuracy": 0.792, | |
| "brier": 0.273025071208486, | |
| "brier_soft": 0.273025071208486, | |
| "nll": 0.5401223012436509, | |
| "nll_soft": 0.5401223012436509, | |
| "ece": 0.09241534466772917, | |
| "mean_confidence": 0.8570207713461646, | |
| "confident_error_rate": 0.04, | |
| "evidence_auc": 1.0 | |
| }, | |
| "rotten_tomatoes": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.035290060552569, | |
| "brier_soft": 0.035290060552569, | |
| "nll": 0.07276457767313435, | |
| "nll_soft": 0.07276457767313435, | |
| "ece": 0.017489686473322937, | |
| "mean_confidence": 0.9711417797012452, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "rte": { | |
| "count": 100, | |
| "accuracy": 0.93, | |
| "brier": 0.11679704186673533, | |
| "brier_soft": 0.11679704186673533, | |
| "nll": 0.20138234100382452, | |
| "nll_soft": 0.20138234100382452, | |
| "ece": 0.04667671248922903, | |
| "mean_confidence": 0.976676712489229, | |
| "confident_error_rate": 0.03 | |
| }, | |
| "scotus": { | |
| "count": 100, | |
| "accuracy": 0.61, | |
| "brier": 0.6341498051249195, | |
| "brier_soft": 0.6341498051249195, | |
| "nll": 1.6277115865464133, | |
| "nll_soft": 1.6277115865464133, | |
| "ece": 0.25533107042012765, | |
| "mean_confidence": 0.8527349247542704, | |
| "confident_error_rate": 0.13 | |
| }, | |
| "shp": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.46443770760873426, | |
| "brier_soft": 0.09936027473515505, | |
| "nll": 0.6572154778957793, | |
| "nll_soft": 0.6826727776739735, | |
| "ece": 0.1103068097721516, | |
| "mean_confidence": 0.5396931902278483, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "sms_spam": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.01279388922890178, | |
| "brier_soft": 0.01279388922890178, | |
| "nll": 0.019404364213720597, | |
| "nll_soft": 0.019404364213720597, | |
| "ece": 0.011369026874576384, | |
| "mean_confidence": 0.994594708705868, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "snli": { | |
| "count": 100, | |
| "accuracy": 0.92, | |
| "brier": 0.1303060757325426, | |
| "brier_soft": 0.1303060757325426, | |
| "nll": 0.240589231743456, | |
| "nll_soft": 0.240589231743456, | |
| "ece": 0.04670265643035693, | |
| "mean_confidence": 0.9518659140843424, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "social_iqa": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.2641777007342555, | |
| "brier_soft": 0.2641777007342555, | |
| "nll": 0.5338294339120516, | |
| "nll_soft": 0.5338294339120516, | |
| "ece": 0.09934345700863399, | |
| "mean_confidence": 0.9154886411744081, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "sst2": { | |
| "count": 100, | |
| "accuracy": 0.98, | |
| "brier": 0.04030628615298806, | |
| "brier_soft": 0.04030628615298806, | |
| "nll": 0.11747284260512826, | |
| "nll_soft": 0.11747284260512826, | |
| "ece": 0.012949483786838481, | |
| "mean_confidence": 0.9895809775228284, | |
| "confident_error_rate": 0.02 | |
| }, | |
| "sst5": { | |
| "count": 100, | |
| "accuracy": 0.66, | |
| "brier": 0.47858939401943057, | |
| "brier_soft": 0.47858939401943057, | |
| "nll": 0.8312380624917151, | |
| "nll_soft": 0.8312380624917151, | |
| "ece": 0.14479568835917578, | |
| "mean_confidence": 0.5538169983846644, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.4382710876978508 | |
| }, | |
| "stance_abortion": { | |
| "count": 60, | |
| "accuracy": 0.65, | |
| "brier": 0.5483838340126799, | |
| "brier_soft": 0.5483838340126799, | |
| "nll": 0.916795129103024, | |
| "nll_soft": 0.916795129103024, | |
| "ece": 0.2470612492514964, | |
| "mean_confidence": 0.8717795686329541, | |
| "confident_error_rate": 0.15 | |
| }, | |
| "stance_atheism": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.31917383626334284, | |
| "brier_soft": 0.31917383626334284, | |
| "nll": 0.49926359046026203, | |
| "nll_soft": 0.49926359046026203, | |
| "ece": 0.15642293053711553, | |
| "mean_confidence": 0.889754045572878, | |
| "confident_error_rate": 0.03333333333333333 | |
| }, | |
| "stance_feminist": { | |
| "count": 60, | |
| "accuracy": 0.7833333333333333, | |
| "brier": 0.3049816554075578, | |
| "brier_soft": 0.3049816554075578, | |
| "nll": 0.5154249440169435, | |
| "nll_soft": 0.5154249440169435, | |
| "ece": 0.10461663391963241, | |
| "mean_confidence": 0.8737570622243047, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "stance_hillary": { | |
| "count": 60, | |
| "accuracy": 0.7166666666666667, | |
| "brier": 0.3812783742364739, | |
| "brier_soft": 0.3812783742364739, | |
| "nll": 0.6242170771095513, | |
| "nll_soft": 0.6242170771095513, | |
| "ece": 0.11280688269490617, | |
| "mean_confidence": 0.8143798914819186, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "stsb": { | |
| "count": 100, | |
| "accuracy": 0.65, | |
| "brier": 0.45170767908012466, | |
| "brier_soft": 0.20552974536260238, | |
| "nll": 0.8029710252702897, | |
| "nll_soft": 0.8819413373331731, | |
| "ece": 0.10983419549350473, | |
| "mean_confidence": 0.6357630237891139, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.44494913340968734 | |
| }, | |
| "subjectivity": { | |
| "count": 100, | |
| "accuracy": 0.99, | |
| "brier": 0.02143104125551823, | |
| "brier_soft": 0.02143104125551823, | |
| "nll": 0.047739931072467476, | |
| "nll_soft": 0.047739931072467476, | |
| "ece": 0.039431940261527555, | |
| "mean_confidence": 0.9635951571408603, | |
| "confident_error_rate": 0.0 | |
| }, | |
| "swag": { | |
| "count": 100, | |
| "accuracy": 0.82, | |
| "brier": 0.2537048639515909, | |
| "brier_soft": 0.2537048639515909, | |
| "nll": 0.4672971887646461, | |
| "nll_soft": 0.4672971887646461, | |
| "ece": 0.08815823139371007, | |
| "mean_confidence": 0.9081582313937099, | |
| "confident_error_rate": 0.05 | |
| }, | |
| "trec": { | |
| "count": 100, | |
| "accuracy": 0.97, | |
| "brier": 0.04378629853100501, | |
| "brier_soft": 0.04378629853100501, | |
| "nll": 0.11549823864485118, | |
| "nll_soft": 0.11549823864485118, | |
| "ece": 0.025471181892042873, | |
| "mean_confidence": 0.9775477901155262, | |
| "confident_error_rate": 0.01 | |
| }, | |
| "tweet_hate": { | |
| "count": 100, | |
| "accuracy": 0.5, | |
| "brier": 0.6883996904854307, | |
| "brier_soft": 0.6883996904854307, | |
| "nll": 0.9833215742341621, | |
| "nll_soft": 0.9833215742341621, | |
| "ece": 0.3459218953511739, | |
| "mean_confidence": 0.8344252768252641, | |
| "confident_error_rate": 0.11 | |
| }, | |
| "tweet_sentiment": { | |
| "count": 100, | |
| "accuracy": 0.74, | |
| "brier": 0.3831706610576699, | |
| "brier_soft": 0.3831706610576699, | |
| "nll": 0.6497370609790863, | |
| "nll_soft": 0.6497370609790863, | |
| "ece": 0.13208543978054657, | |
| "mean_confidence": 0.6252989072735136, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3541000877573742 | |
| }, | |
| "vitaminc": { | |
| "count": 100, | |
| "accuracy": 0.84, | |
| "brier": 0.23269831680007946, | |
| "brier_soft": 0.23269831680007946, | |
| "nll": 0.4394585245066372, | |
| "nll_soft": 0.4394585245066372, | |
| "ece": 0.09072915887713841, | |
| "mean_confidence": 0.9197737336036776, | |
| "confident_error_rate": 0.06 | |
| }, | |
| "wic": { | |
| "count": 100, | |
| "accuracy": 0.78, | |
| "brier": 0.34073352692700504, | |
| "brier_soft": 0.34073352692700504, | |
| "nll": 0.5452054678565941, | |
| "nll_soft": 0.5452054678565941, | |
| "ece": 0.13825743108455368, | |
| "mean_confidence": 0.9071892135345369, | |
| "confident_error_rate": 0.1 | |
| }, | |
| "winogrande": { | |
| "count": 100, | |
| "accuracy": 0.91, | |
| "brier": 0.17067644004760918, | |
| "brier_soft": 0.17067644004760918, | |
| "nll": 0.35855105926065917, | |
| "nll_soft": 0.35855105926065917, | |
| "ece": 0.08624812639081393, | |
| "mean_confidence": 0.9450851377589773, | |
| "confident_error_rate": 0.07 | |
| }, | |
| "yelp": { | |
| "count": 100, | |
| "accuracy": 0.73, | |
| "brier": 0.402625269533739, | |
| "brier_soft": 0.402625269533739, | |
| "nll": 0.6852448507549156, | |
| "nll_soft": 0.6852448507549156, | |
| "ece": 0.16558757141209074, | |
| "mean_confidence": 0.6233702191486455, | |
| "confident_error_rate": 0.0, | |
| "score_mae": 0.3873896875444972 | |
| } | |
| }, | |
| "reliability": [ | |
| { | |
| "bin": "0.0-0.1", | |
| "count": 0, | |
| "confidence": NaN, | |
| "accuracy": NaN | |
| }, | |
| { | |
| "bin": "0.1-0.2", | |
| "count": 5, | |
| "confidence": 0.1795541536525354, | |
| "accuracy": 0.0 | |
| }, | |
| { | |
| "bin": "0.2-0.3", | |
| "count": 67, | |
| "confidence": 0.251950177472451, | |
| "accuracy": 0.29850746268656714 | |
| }, | |
| { | |
| "bin": "0.3-0.4", | |
| "count": 83, | |
| "confidence": 0.35303291214353666, | |
| "accuracy": 0.3493975903614458 | |
| }, | |
| { | |
| "bin": "0.4-0.5", | |
| "count": 271, | |
| "confidence": 0.46348321355010486, | |
| "accuracy": 0.5129151291512916 | |
| }, | |
| { | |
| "bin": "0.5-0.6", | |
| "count": 405, | |
| "confidence": 0.5506984297677288, | |
| "accuracy": 0.6148148148148148 | |
| }, | |
| { | |
| "bin": "0.6-0.7", | |
| "count": 314, | |
| "confidence": 0.6475632613940633, | |
| "accuracy": 0.6719745222929936 | |
| }, | |
| { | |
| "bin": "0.7-0.8", | |
| "count": 328, | |
| "confidence": 0.7527084045456743, | |
| "accuracy": 0.725609756097561 | |
| }, | |
| { | |
| "bin": "0.8-0.9", | |
| "count": 442, | |
| "confidence": 0.8559694673474576, | |
| "accuracy": 0.7081447963800905 | |
| }, | |
| { | |
| "bin": "0.9-1.0", | |
| "count": 3598, | |
| "confidence": 0.9823404759359501, | |
| "accuracy": 0.9449694274596998 | |
| } | |
| ], | |
| "seconds": 387.9096157550812, | |
| "model": "runs/assay-27b", | |
| "data": "data/v2/dev.jsonl" | |
| } |