| { |
| "overall": { |
| "count": 6113, |
| "accuracy": 0.705054801243252, |
| "brier": 0.39068303174557184, |
| "brier_soft": 0.36485200542432994, |
| "nll": 0.7435078762316807, |
| "nll_soft": 0.7539368565298866, |
| "ece": 0.030412426160708377, |
| "mean_confidence": 0.7254209476637835, |
| "confident_error_rate": 0.01930312448879437, |
| "score_mae": 0.6191286255856765, |
| "evidence_auc": 0.9963101774488741 |
| }, |
| "by_type": { |
| "bool": { |
| "count": 1898, |
| "accuracy": 0.8108535300316122, |
| "brier": 0.26805662687023524, |
| "brier_soft": 0.2639773107878006, |
| "nll": 0.4338549421061354, |
| "nll_soft": 0.43754843965694995, |
| "ece": 0.04707414445089357, |
| "mean_confidence": 0.8579276744825058, |
| "confident_error_rate": 0.038461538461538464, |
| "evidence_auc": 0.9915676170846216 |
| }, |
| "choice": { |
| "count": 3248, |
| "accuracy": 0.6736453201970444, |
| "brier": 0.4267776942434168, |
| "brier_soft": 0.4015465270646481, |
| "nll": 0.8669136499375315, |
| "nll_soft": 0.8772741819689498, |
| "ece": 0.045373296227665255, |
| "mean_confidence": 0.7086789911375205, |
| "confident_error_rate": 0.013854679802955665, |
| "evidence_auc": 0.9996868622897755 |
| }, |
| "score": { |
| "count": 967, |
| "accuracy": 0.6028955532574974, |
| "brier": 0.5101343788607614, |
| "brier_soft": 0.43959488456846546, |
| "nll": 0.9367853488001198, |
| "nll_soft": 0.9606644492897173, |
| "ece": 0.0857255052436744, |
| "mean_confidence": 0.5215741092929117, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6191286255856765 |
| } |
| }, |
| "by_source": { |
| "ag_news": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.17167430940678405, |
| "brier_soft": 0.17167430940678405, |
| "nll": 0.336887603212825, |
| "nll_soft": 0.336887603212825, |
| "ece": 0.07163243215848891, |
| "mean_confidence": 0.9219929393584567, |
| "confident_error_rate": 0.06 |
| }, |
| "amazon_polarity": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.08580480427527387, |
| "brier_soft": 0.08580480427527387, |
| "nll": 0.13534072911744766, |
| "nll_soft": 0.13534072911744766, |
| "ece": 0.060871109033524766, |
| "mean_confidence": 0.9416013850702707, |
| "confident_error_rate": 0.01 |
| }, |
| "amazon_stars": { |
| "count": 100, |
| "accuracy": 0.56, |
| "brier": 0.5696269202129539, |
| "brier_soft": 0.5696269202129539, |
| "nll": 1.0261049502969477, |
| "nll_soft": 1.0261049502969477, |
| "ece": 0.12194184749343116, |
| "mean_confidence": 0.47691471251232365, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.614239567042899 |
| }, |
| "anli": { |
| "count": 100, |
| "accuracy": 0.43, |
| "brier": 0.8086471157013881, |
| "brier_soft": 0.8086471157013881, |
| "nll": 1.398779746391494, |
| "nll_soft": 1.398779746391494, |
| "ece": 0.31853672993302656, |
| "mean_confidence": 0.727838441783572, |
| "confident_error_rate": 0.09 |
| }, |
| "arc_challenge": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.46429933197227446, |
| "brier_soft": 0.46429933197227446, |
| "nll": 0.8642889474810811, |
| "nll_soft": 0.8642889474810811, |
| "ece": 0.15669435497659792, |
| "mean_confidence": 0.7235336333706962, |
| "confident_error_rate": 0.02 |
| }, |
| "arc_easy": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.1940937298338349, |
| "brier_soft": 0.1940937298338349, |
| "nll": 0.39064179191107024, |
| "nll_soft": 0.39064179191107024, |
| "ece": 0.07900940316304368, |
| "mean_confidence": 0.8062375951757056, |
| "confident_error_rate": 0.0 |
| }, |
| "banking77": { |
| "count": 100, |
| "accuracy": 0.67, |
| "brier": 0.4784721751139965, |
| "brier_soft": 0.4784721751139965, |
| "nll": 1.5601690451301173, |
| "nll_soft": 1.5601690451301173, |
| "ece": 0.12808546873118945, |
| "mean_confidence": 0.6846104644283264, |
| "confident_error_rate": 0.03 |
| }, |
| "boolq": { |
| "count": 125, |
| "accuracy": 0.664, |
| "brier": 0.5824374662242449, |
| "brier_soft": 0.5824374662242449, |
| "nll": 1.1200996870048703, |
| "nll_soft": 1.1200996870048703, |
| "ece": 0.25503892261697847, |
| "mean_confidence": 0.9009889255438389, |
| "confident_error_rate": 0.24, |
| "evidence_auc": 1.0 |
| }, |
| "cb": { |
| "count": 28, |
| "accuracy": 0.8928571428571429, |
| "brier": 0.11845201944614438, |
| "brier_soft": 0.11845201944614438, |
| "nll": 0.23775217633412454, |
| "nll_soft": 0.23775217633412454, |
| "ece": 0.1078101710685394, |
| "mean_confidence": 0.8733159887773345, |
| "confident_error_rate": 0.0 |
| }, |
| "civil_comments_severity": { |
| "count": 100, |
| "accuracy": 0.83, |
| "brier": 0.30853567636963153, |
| "brier_soft": 0.20466550090702357, |
| "nll": 0.6146090606073197, |
| "nll_soft": 0.7049566039180744, |
| "ece": 0.19732957498121775, |
| "mean_confidence": 0.652800302888791, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.48844389755448875 |
| }, |
| "civil_comments_toxic": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.13107340592724387, |
| "brier_soft": 0.05364798668263418, |
| "nll": 0.234533996986214, |
| "nll_soft": 0.3046365805006745, |
| "ece": 0.06177327425323237, |
| "mean_confidence": 0.8792147122777147, |
| "confident_error_rate": 0.01 |
| }, |
| "clinc150": { |
| "count": 100, |
| "accuracy": 0.69, |
| "brier": 0.4673241163352339, |
| "brier_soft": 0.4673241163352339, |
| "nll": 1.7998103678930273, |
| "nll_soft": 1.7998103678930273, |
| "ece": 0.10405707152125332, |
| "mean_confidence": 0.7034346459910751, |
| "confident_error_rate": 0.01 |
| }, |
| "cola": { |
| "count": 100, |
| "accuracy": 0.64, |
| "brier": 0.45212164375689284, |
| "brier_soft": 0.45212164375689284, |
| "nll": 0.6450283149218293, |
| "nll_soft": 0.6450283149218293, |
| "ece": 0.11530711181948869, |
| "mean_confidence": 0.7540596079177964, |
| "confident_error_rate": 0.02 |
| }, |
| "commonsense_qa": { |
| "count": 100, |
| "accuracy": 0.68, |
| "brier": 0.44878207901084366, |
| "brier_soft": 0.44878207901084366, |
| "nll": 0.8815806784626246, |
| "nll_soft": 0.8815806784626246, |
| "ece": 0.06865873042756807, |
| "mean_confidence": 0.6821657773971803, |
| "confident_error_rate": 0.01 |
| }, |
| "cosmos_qa": { |
| "count": 120, |
| "accuracy": 0.5833333333333334, |
| "brier": 0.5723719210693715, |
| "brier_soft": 0.5723719210693715, |
| "nll": 1.0851163899122087, |
| "nll_soft": 1.0851163899122087, |
| "ece": 0.09618159223747698, |
| "mean_confidence": 0.662741275142061, |
| "confident_error_rate": 0.0, |
| "evidence_auc": 0.9953125 |
| }, |
| "dates": { |
| "count": 300, |
| "accuracy": 0.8466666666666667, |
| "brier": 0.1952099468621781, |
| "brier_soft": 0.1952099468621781, |
| "nll": 0.30503634674823765, |
| "nll_soft": 0.30503634674823765, |
| "ece": 0.043387396134385074, |
| "mean_confidence": 0.84625525258766, |
| "confident_error_rate": 0.0033333333333333335, |
| "score_mae": 0.3230317969202634 |
| }, |
| "dbpedia": { |
| "count": 100, |
| "accuracy": 1.0, |
| "brier": 0.0008420663170771253, |
| "brier_soft": 0.0008420663170771253, |
| "nll": 0.00918498596266742, |
| "nll_soft": 0.00918498596266742, |
| "ece": 0.008904030823745671, |
| "mean_confidence": 0.9910959691762545, |
| "confident_error_rate": 0.0 |
| }, |
| "dynasent": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.5884531231040632, |
| "brier_soft": 0.4423489537858375, |
| "nll": 0.9879611872627133, |
| "nll_soft": 1.0010951533157162, |
| "ece": 0.1554664898909139, |
| "mean_confidence": 0.5607612162644048, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6325307416076793 |
| }, |
| "enron_spam": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.0750898751455318, |
| "brier_soft": 0.0750898751455318, |
| "nll": 0.13481450114911914, |
| "nll_soft": 0.13481450114911914, |
| "ece": 0.058760900159834276, |
| "mean_confidence": 0.936686700660227, |
| "confident_error_rate": 0.01 |
| }, |
| "go_emotions": { |
| "count": 100, |
| "accuracy": 0.47, |
| "brier": 0.718367046561157, |
| "brier_soft": 0.28923457335271163, |
| "nll": 1.926955163826314, |
| "nll_soft": 2.2635551312922435, |
| "ece": 0.17175697839494763, |
| "mean_confidence": 0.33183864668198176, |
| "confident_error_rate": 0.0 |
| }, |
| "hellaswag": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.49074184637761903, |
| "brier_soft": 0.49074184637761903, |
| "nll": 0.9305013577650898, |
| "nll_soft": 0.9305013577650898, |
| "ece": 0.11862829019667494, |
| "mean_confidence": 0.6534043868116024, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.08795359522078826, |
| "brier_soft": 0.08795359522078826, |
| "nll": 0.1703674822113271, |
| "nll_soft": 0.1703674822113271, |
| "ece": 0.040475760842537414, |
| "mean_confidence": 0.940735768754548, |
| "confident_error_rate": 0.03 |
| }, |
| "ledgar": { |
| "count": 100, |
| "accuracy": 0.68, |
| "brier": 0.40614986096299754, |
| "brier_soft": 0.40614986096299754, |
| "nll": 1.4404445803485642, |
| "nll_soft": 1.4404445803485642, |
| "ece": 0.12459564932621359, |
| "mean_confidence": 0.675806081562592, |
| "confident_error_rate": 0.01 |
| }, |
| "liar": { |
| "count": 100, |
| "accuracy": 0.26, |
| "brier": 0.7933185274153484, |
| "brier_soft": 0.7933185274153484, |
| "nll": 1.6679028813048031, |
| "nll_soft": 1.6679028813048031, |
| "ece": 0.06507736504717265, |
| "mean_confidence": 0.24634068079277538, |
| "confident_error_rate": 0.0, |
| "score_mae": 1.2559342591277523 |
| }, |
| "measuring_hate_speech": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.3396697348224549, |
| "brier_soft": 0.19080407243559158, |
| "nll": 0.6206328985442651, |
| "nll_soft": 0.7308991184362474, |
| "ece": 0.17923612715118986, |
| "mean_confidence": 0.6307638728488102, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6545338061589748 |
| }, |
| "medqa": { |
| "count": 100, |
| "accuracy": 0.46, |
| "brier": 0.6830794095906169, |
| "brier_soft": 0.6830794095906169, |
| "nll": 1.260217965022958, |
| "nll_soft": 1.260217965022958, |
| "ece": 0.10410187965371252, |
| "mean_confidence": 0.40546563669920876, |
| "confident_error_rate": 0.0 |
| }, |
| "mnli": { |
| "count": 100, |
| "accuracy": 0.86, |
| "brier": 0.2461392504105252, |
| "brier_soft": 0.2461392504105252, |
| "nll": 0.4500251998903584, |
| "nll_soft": 0.4500251998903584, |
| "ece": 0.07792265817164262, |
| "mean_confidence": 0.809569663745306, |
| "confident_error_rate": 0.01 |
| }, |
| "mrpc": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.23254436799651235, |
| "brier_soft": 0.23254436799651235, |
| "nll": 0.37406467998561715, |
| "nll_soft": 0.37406467998561715, |
| "ece": 0.06513814925679627, |
| "mean_confidence": 0.8063140117596898, |
| "confident_error_rate": 0.0 |
| }, |
| "multirc": { |
| "count": 125, |
| "accuracy": 0.688, |
| "brier": 0.5150958140330812, |
| "brier_soft": 0.5150958140330812, |
| "nll": 0.9321397083694551, |
| "nll_soft": 0.9321397083694551, |
| "ece": 0.22299498214130758, |
| "mean_confidence": 0.8800483904643696, |
| "confident_error_rate": 0.176, |
| "evidence_auc": 0.9813333333333333 |
| }, |
| "openbookqa": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.4791117350116674, |
| "brier_soft": 0.4791117350116674, |
| "nll": 0.8580094565143226, |
| "nll_soft": 0.8580094565143226, |
| "ece": 0.11261566906175771, |
| "mean_confidence": 0.6988737716957278, |
| "confident_error_rate": 0.01 |
| }, |
| "piqa": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.4606743696961655, |
| "brier_soft": 0.4606743696961655, |
| "nll": 0.6937551993329675, |
| "nll_soft": 0.6937551993329675, |
| "ece": 0.11593196583580467, |
| "mean_confidence": 0.7223726310067928, |
| "confident_error_rate": 0.04 |
| }, |
| "policy_application": { |
| "count": 150, |
| "accuracy": 0.8, |
| "brier": 0.25790714562174005, |
| "brier_soft": 0.25790714562174005, |
| "nll": 0.41907410837426357, |
| "nll_soft": 0.41907410837426357, |
| "ece": 0.05254626318981812, |
| "mean_confidence": 0.8000413443765387, |
| "confident_error_rate": 0.013333333333333334, |
| "evidence_auc": 0.9849574885546108 |
| }, |
| "policy_hard": { |
| "count": 300, |
| "accuracy": 0.7166666666666667, |
| "brier": 0.36013972381824627, |
| "brier_soft": 0.36013972381824627, |
| "nll": 0.5704760559630674, |
| "nll_soft": 0.5704760559630674, |
| "ece": 0.06747979684035946, |
| "mean_confidence": 0.7642620234200053, |
| "confident_error_rate": 0.006666666666666667, |
| "evidence_auc": 0.9403935185185185 |
| }, |
| "qqp": { |
| "count": 100, |
| "accuracy": 0.87, |
| "brier": 0.21255833780443303, |
| "brier_soft": 0.21255833780443303, |
| "nll": 0.3375497168485568, |
| "nll_soft": 0.3375497168485568, |
| "ece": 0.08248226236243893, |
| "mean_confidence": 0.8207414940051179, |
| "confident_error_rate": 0.01 |
| }, |
| "race": { |
| "count": 125, |
| "accuracy": 0.608, |
| "brier": 0.491455314176186, |
| "brier_soft": 0.491455314176186, |
| "nll": 0.9626818782840941, |
| "nll_soft": 0.9626818782840941, |
| "ece": 0.14182781184836327, |
| "mean_confidence": 0.7493741312243011, |
| "confident_error_rate": 0.016, |
| "evidence_auc": 0.9938666666666667 |
| }, |
| "rotten_tomatoes": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.14275234295049224, |
| "brier_soft": 0.14275234295049224, |
| "nll": 0.23282286234383792, |
| "nll_soft": 0.23282286234383792, |
| "ece": 0.051450218466265296, |
| "mean_confidence": 0.9054416516058115, |
| "confident_error_rate": 0.01 |
| }, |
| "rte": { |
| "count": 100, |
| "accuracy": 0.82, |
| "brier": 0.2653727111837063, |
| "brier_soft": 0.2653727111837063, |
| "nll": 0.4042521882397484, |
| "nll_soft": 0.4042521882397484, |
| "ece": 0.0661314421326466, |
| "mean_confidence": 0.8619884639516752, |
| "confident_error_rate": 0.03 |
| }, |
| "scotus": { |
| "count": 100, |
| "accuracy": 0.23, |
| "brier": 0.8951972916507671, |
| "brier_soft": 0.8951972916507671, |
| "nll": 2.4117270185815007, |
| "nll_soft": 2.4117270185815007, |
| "ece": 0.07217207065841522, |
| "mean_confidence": 0.1915051363038614, |
| "confident_error_rate": 0.0 |
| }, |
| "shp": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.5093176162653404, |
| "brier_soft": 0.11894177950737726, |
| "nll": 0.7026835810861257, |
| "nll_soft": 0.702593694000663, |
| "ece": 0.1111520075280236, |
| "mean_confidence": 0.5543252050630872, |
| "confident_error_rate": 0.0 |
| }, |
| "sms_spam": { |
| "count": 100, |
| "accuracy": 0.98, |
| "brier": 0.02870347214535624, |
| "brier_soft": 0.02870347214535624, |
| "nll": 0.06499826507213281, |
| "nll_soft": 0.06499826507213281, |
| "ece": 0.03372488679107602, |
| "mean_confidence": 0.9658248068090167, |
| "confident_error_rate": 0.01 |
| }, |
| "snli": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.22331515795857343, |
| "brier_soft": 0.22331515795857343, |
| "nll": 0.39539320777824905, |
| "nll_soft": 0.39539320777824905, |
| "ece": 0.0926695947746054, |
| "mean_confidence": 0.8633797733330062, |
| "confident_error_rate": 0.0 |
| }, |
| "social_iqa": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.474283631912417, |
| "brier_soft": 0.474283631912417, |
| "nll": 0.8302148480877927, |
| "nll_soft": 0.8302148480877927, |
| "ece": 0.11975901245282002, |
| "mean_confidence": 0.7186295344419724, |
| "confident_error_rate": 0.03 |
| }, |
| "sst2": { |
| "count": 100, |
| "accuracy": 0.91, |
| "brier": 0.15220591437374925, |
| "brier_soft": 0.15220591437374925, |
| "nll": 0.27066277966821617, |
| "nll_soft": 0.27066277966821617, |
| "ece": 0.0584594702815044, |
| "mean_confidence": 0.9584594702815042, |
| "confident_error_rate": 0.05 |
| }, |
| "sst5": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.5692131000511924, |
| "brier_soft": 0.5692131000511924, |
| "nll": 1.0482270625207641, |
| "nll_soft": 1.0482270625207641, |
| "ece": 0.14522475165356238, |
| "mean_confidence": 0.45230109896773846, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.579621455718889 |
| }, |
| "stance_abortion": { |
| "count": 60, |
| "accuracy": 0.6166666666666667, |
| "brier": 0.5206953339555322, |
| "brier_soft": 0.5206953339555322, |
| "nll": 0.8491090429716367, |
| "nll_soft": 0.8491090429716367, |
| "ece": 0.17060325898737938, |
| "mean_confidence": 0.6450970316412057, |
| "confident_error_rate": 0.0 |
| }, |
| "stance_atheism": { |
| "count": 60, |
| "accuracy": 0.7166666666666667, |
| "brier": 0.36611641733676126, |
| "brier_soft": 0.36611641733676126, |
| "nll": 0.6162604565119774, |
| "nll_soft": 0.6162604565119774, |
| "ece": 0.16375666437063566, |
| "mean_confidence": 0.7601322048943361, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stance_feminist": { |
| "count": 60, |
| "accuracy": 0.6, |
| "brier": 0.4934844463221146, |
| "brier_soft": 0.4934844463221146, |
| "nll": 0.7795586115289036, |
| "nll_soft": 0.7795586115289036, |
| "ece": 0.14911099146600104, |
| "mean_confidence": 0.6214280468288923, |
| "confident_error_rate": 0.0 |
| }, |
| "stance_hillary": { |
| "count": 60, |
| "accuracy": 0.65, |
| "brier": 0.47389946504890085, |
| "brier_soft": 0.47389946504890085, |
| "nll": 0.8025726614249472, |
| "nll_soft": 0.8025726614249472, |
| "ece": 0.0704082089812596, |
| "mean_confidence": 0.6720211955303614, |
| "confident_error_rate": 0.0 |
| }, |
| "stsb": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.6192029968172934, |
| "brier_soft": 0.3359260941784888, |
| "nll": 1.1508243053734646, |
| "nll_soft": 1.1679874778521313, |
| "ece": 0.11060553235669754, |
| "mean_confidence": 0.47404663542874836, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.646867863215795 |
| }, |
| "subjectivity": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.09541205703283434, |
| "brier_soft": 0.09541205703283434, |
| "nll": 0.15057303108214104, |
| "nll_soft": 0.15057303108214104, |
| "ece": 0.04798439182160856, |
| "mean_confidence": 0.9302849687736618, |
| "confident_error_rate": 0.0 |
| }, |
| "swag": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.5463578569460535, |
| "brier_soft": 0.5463578569460535, |
| "nll": 0.9887063359616937, |
| "nll_soft": 0.9887063359616937, |
| "ece": 0.08753140963152978, |
| "mean_confidence": 0.6505128614902675, |
| "confident_error_rate": 0.0 |
| }, |
| "trec": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.08204722818910742, |
| "brier_soft": 0.08204722818910742, |
| "nll": 0.17559393951295343, |
| "nll_soft": 0.17559393951295343, |
| "ece": 0.02264209411714494, |
| "mean_confidence": 0.9500299020554932, |
| "confident_error_rate": 0.01 |
| }, |
| "tweet_hate": { |
| "count": 100, |
| "accuracy": 0.51, |
| "brier": 0.5928009007159826, |
| "brier_soft": 0.5928009007159826, |
| "nll": 0.8151179633209827, |
| "nll_soft": 0.8151179633209827, |
| "ece": 0.21335525395842544, |
| "mean_confidence": 0.7233552539584255, |
| "confident_error_rate": 0.0 |
| }, |
| "tweet_sentiment": { |
| "count": 100, |
| "accuracy": 0.69, |
| "brier": 0.43698091004984074, |
| "brier_soft": 0.43698091004984074, |
| "nll": 0.7345815927827489, |
| "nll_soft": 0.7345815927827489, |
| "ece": 0.13993783321704267, |
| "mean_confidence": 0.5500621667829574, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.3860652139710619 |
| }, |
| "vitaminc": { |
| "count": 100, |
| "accuracy": 0.71, |
| "brier": 0.40905484004391957, |
| "brier_soft": 0.40905484004391957, |
| "nll": 0.7271544604990626, |
| "nll_soft": 0.7271544604990626, |
| "ece": 0.10385044249306388, |
| "mean_confidence": 0.672572792332177, |
| "confident_error_rate": 0.01 |
| }, |
| "wic": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.43751322905741524, |
| "brier_soft": 0.43751322905741524, |
| "nll": 0.6263084664151242, |
| "nll_soft": 0.6263084664151242, |
| "ece": 0.05545881046082848, |
| "mean_confidence": 0.6287045484026507, |
| "confident_error_rate": 0.0 |
| }, |
| "winogrande": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.6104376541011685, |
| "brier_soft": 0.6104376541011685, |
| "nll": 0.8506372185576656, |
| "nll_soft": 0.8506372185576656, |
| "ece": 0.2290845274269612, |
| "mean_confidence": 0.7190845274269612, |
| "confident_error_rate": 0.05 |
| }, |
| "yelp": { |
| "count": 100, |
| "accuracy": 0.57, |
| "brier": 0.5254606229663639, |
| "brier_soft": 0.5254606229663639, |
| "nll": 0.8989841552777672, |
| "nll_soft": 0.8989841552777672, |
| "ece": 0.10490958010393049, |
| "mean_confidence": 0.5356457346476723, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5123057010793745 |
| } |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "count": 3, |
| "confidence": 0.06146846781302175, |
| "accuracy": 0.0 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "count": 97, |
| "confidence": 0.16869521945608792, |
| "accuracy": 0.23711340206185566 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "count": 171, |
| "confidence": 0.24612851557121382, |
| "accuracy": 0.25146198830409355 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "count": 293, |
| "confidence": 0.3580441095895981, |
| "accuracy": 0.4129692832764505 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "count": 590, |
| "confidence": 0.4550973981822978, |
| "accuracy": 0.4745762711864407 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "count": 775, |
| "confidence": 0.5489073426772118, |
| "accuracy": 0.52 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "count": 690, |
| "confidence": 0.6466276273117183, |
| "accuracy": 0.6260869565217392 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "count": 714, |
| "confidence": 0.753869082660997, |
| "accuracy": 0.6988795518207283 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "count": 818, |
| "confidence": 0.8513946637045477, |
| "accuracy": 0.812958435207824 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "count": 1962, |
| "confidence": 0.9664467037858544, |
| "accuracy": 0.9398572884811417 |
| } |
| ], |
| "seconds": 16.7630455493927, |
| "model": "runs/assay-0.6b-v4", |
| "data": "data/v4/dev.jsonl" |
| } |
