| { |
| "overall": { |
| "count": 6113, |
| "accuracy": 0.705054801243252, |
| "brier": 0.39538385697778006, |
| "brier_soft": 0.3704977485354012, |
| "nll": 0.7609112228277006, |
| "nll_soft": 0.7734174605563418, |
| "ece": 0.05680640827126576, |
| "mean_confidence": 0.7593619540719595, |
| "confident_error_rate": 0.02977261573695403, |
| "score_mae": 0.5969017740151777, |
| "evidence_auc": 0.9963101774488741 |
| }, |
| "by_type": { |
| "bool": { |
| "count": 1898, |
| "accuracy": 0.8108535300316122, |
| "brier": 0.2740322208531036, |
| "brier_soft": 0.2701714806229827, |
| "nll": 0.46085836553903575, |
| "nll_soft": 0.46528754041151366, |
| "ece": 0.06815199844484307, |
| "mean_confidence": 0.8790055284764552, |
| "confident_error_rate": 0.04741833508956796, |
| "evidence_auc": 0.9915676170846216 |
| }, |
| "choice": { |
| "count": 3248, |
| "accuracy": 0.6736453201970444, |
| "brier": 0.4347351332684781, |
| "brier_soft": 0.41022688214923, |
| "nll": 0.889917829552499, |
| "nll_soft": 0.9023419845071041, |
| "ece": 0.07908039213636706, |
| "mean_confidence": 0.7495591550204898, |
| "confident_error_rate": 0.02832512315270936, |
| "evidence_auc": 0.9996868622897755 |
| }, |
| "score": { |
| "count": 967, |
| "accuracy": 0.6028955532574974, |
| "brier": 0.501394673908958, |
| "brier_soft": 0.4339714098798214, |
| "nll": 0.9165336266454257, |
| "nll_soft": 0.9451689958643129, |
| "ece": 0.05247221660669864, |
| "mean_confidence": 0.557455012085859, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5969017740151777 |
| } |
| }, |
| "by_source": { |
| "ag_news": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.17413496371407272, |
| "brier_soft": 0.17413496371407272, |
| "nll": 0.3629183763109327, |
| "nll_soft": 0.3629183763109327, |
| "ece": 0.07433688379944284, |
| "mean_confidence": 0.9422972178204269, |
| "confident_error_rate": 0.07 |
| }, |
| "amazon_polarity": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.0886938049925854, |
| "brier_soft": 0.0886938049925854, |
| "nll": 0.13511790796851203, |
| "nll_soft": 0.13511790796851203, |
| "ece": 0.04852081055881998, |
| "mean_confidence": 0.9535724897605337, |
| "confident_error_rate": 0.01 |
| }, |
| "amazon_stars": { |
| "count": 100, |
| "accuracy": 0.56, |
| "brier": 0.5615561430559731, |
| "brier_soft": 0.5615561430559731, |
| "nll": 1.004236111230652, |
| "nll_soft": 1.004236111230652, |
| "ece": 0.11254505423155164, |
| "mean_confidence": 0.5131011088633767, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5926813163957635 |
| }, |
| "anli": { |
| "count": 100, |
| "accuracy": 0.43, |
| "brier": 0.8533102628644142, |
| "brier_soft": 0.8533102628644142, |
| "nll": 1.550810689838082, |
| "nll_soft": 1.550810689838082, |
| "ece": 0.35529536864438105, |
| "mean_confidence": 0.7705493629519586, |
| "confident_error_rate": 0.15 |
| }, |
| "arc_challenge": { |
| "count": 100, |
| "accuracy": 0.66, |
| "brier": 0.4784058798558821, |
| "brier_soft": 0.4784058798558821, |
| "nll": 0.9050353613321852, |
| "nll_soft": 0.9050353613321852, |
| "ece": 0.16081623666523995, |
| "mean_confidence": 0.7630610967687517, |
| "confident_error_rate": 0.02 |
| }, |
| "arc_easy": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.1880681937898805, |
| "brier_soft": 0.1880681937898805, |
| "nll": 0.3720691583875527, |
| "nll_soft": 0.3720691583875527, |
| "ece": 0.07244312598227795, |
| "mean_confidence": 0.8403154234201606, |
| "confident_error_rate": 0.0 |
| }, |
| "banking77": { |
| "count": 100, |
| "accuracy": 0.67, |
| "brier": 0.4914882572564752, |
| "brier_soft": 0.4914882572564752, |
| "nll": 1.623636626602333, |
| "nll_soft": 1.623636626602333, |
| "ece": 0.12364102131734185, |
| "mean_confidence": 0.7796481801825672, |
| "confident_error_rate": 0.04 |
| }, |
| "boolq": { |
| "count": 125, |
| "accuracy": 0.664, |
| "brier": 0.5980653234848732, |
| "brier_soft": 0.5980653234848732, |
| "nll": 1.298392630667349, |
| "nll_soft": 1.298392630667349, |
| "ece": 0.26943362417895406, |
| "mean_confidence": 0.9199519452890829, |
| "confident_error_rate": 0.248, |
| "evidence_auc": 1.0 |
| }, |
| "cb": { |
| "count": 28, |
| "accuracy": 0.8928571428571429, |
| "brier": 0.11690463390189174, |
| "brier_soft": 0.11690463390189174, |
| "nll": 0.21779212204430268, |
| "nll_soft": 0.21779212204430268, |
| "ece": 0.08857779853089791, |
| "mean_confidence": 0.9039091474360438, |
| "confident_error_rate": 0.0 |
| }, |
| "civil_comments_severity": { |
| "count": 100, |
| "accuracy": 0.83, |
| "brier": 0.28633910159951037, |
| "brier_soft": 0.19718952723054037, |
| "nll": 0.5796615094995807, |
| "nll_soft": 0.6880046191240531, |
| "ece": 0.16809728616903127, |
| "mean_confidence": 0.6965753782124113, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4343666025740125 |
| }, |
| "civil_comments_toxic": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.1284382819411095, |
| "brier_soft": 0.05516143237341537, |
| "nll": 0.22256718628056793, |
| "nll_soft": 0.30663292536019726, |
| "ece": 0.03703245365501778, |
| "mean_confidence": 0.9066386896639135, |
| "confident_error_rate": 0.01 |
| }, |
| "clinc150": { |
| "count": 100, |
| "accuracy": 0.69, |
| "brier": 0.48424088398683807, |
| "brier_soft": 0.48424088398683807, |
| "nll": 1.9164021183466793, |
| "nll_soft": 1.9164021183466793, |
| "ece": 0.15278673889388222, |
| "mean_confidence": 0.7785355399993794, |
| "confident_error_rate": 0.05 |
| }, |
| "cola": { |
| "count": 100, |
| "accuracy": 0.64, |
| "brier": 0.4698195691883268, |
| "brier_soft": 0.4698195691883268, |
| "nll": 0.67335594822787, |
| "nll_soft": 0.67335594822787, |
| "ece": 0.14687265203113414, |
| "mean_confidence": 0.786872652031134, |
| "confident_error_rate": 0.02 |
| }, |
| "commonsense_qa": { |
| "count": 100, |
| "accuracy": 0.68, |
| "brier": 0.45596203220310955, |
| "brier_soft": 0.45596203220310955, |
| "nll": 0.903688557041255, |
| "nll_soft": 0.903688557041255, |
| "ece": 0.07379980682596343, |
| "mean_confidence": 0.7272165206966212, |
| "confident_error_rate": 0.02 |
| }, |
| "cosmos_qa": { |
| "count": 120, |
| "accuracy": 0.5833333333333334, |
| "brier": 0.5909838058145874, |
| "brier_soft": 0.5909838058145874, |
| "nll": 1.1383380371275735, |
| "nll_soft": 1.1383380371275735, |
| "ece": 0.13455620680782, |
| "mean_confidence": 0.7119819855535954, |
| "confident_error_rate": 0.008333333333333333, |
| "evidence_auc": 0.9953125 |
| }, |
| "dates": { |
| "count": 300, |
| "accuracy": 0.8466666666666667, |
| "brier": 0.1967509138402891, |
| "brier_soft": 0.1967509138402891, |
| "nll": 0.3009553961368296, |
| "nll_soft": 0.3009553961368296, |
| "ece": 0.04130889149601512, |
| "mean_confidence": 0.8671836176506571, |
| "confident_error_rate": 0.013333333333333334, |
| "score_mae": 0.29974614546487804 |
| }, |
| "dbpedia": { |
| "count": 100, |
| "accuracy": 1.0, |
| "brier": 0.0003368394311318027, |
| "brier_soft": 0.0003368394311318027, |
| "nll": 0.0038888579712315127, |
| "nll_soft": 0.0038888579712315127, |
| "ece": 0.0037882713580731898, |
| "mean_confidence": 0.9962117286419268, |
| "confident_error_rate": 0.0 |
| }, |
| "dynasent": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.5961896210308214, |
| "brier_soft": 0.45106317825494485, |
| "nll": 1.0052915301044587, |
| "nll_soft": 1.0210415305687826, |
| "ece": 0.15163641085600274, |
| "mean_confidence": 0.5972543962018781, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6232283387101343 |
| }, |
| "enron_spam": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.07531883646013675, |
| "brier_soft": 0.07531883646013675, |
| "nll": 0.12900663400642037, |
| "nll_soft": 0.12900663400642037, |
| "ece": 0.022067279667001442, |
| "mean_confidence": 0.953383408793485, |
| "confident_error_rate": 0.02 |
| }, |
| "go_emotions": { |
| "count": 100, |
| "accuracy": 0.47, |
| "brier": 0.6952311397118899, |
| "brier_soft": 0.2896312112266606, |
| "nll": 1.8465549197965925, |
| "nll_soft": 2.2501992644261604, |
| "ece": 0.0813991412373355, |
| "mean_confidence": 0.4202206149885135, |
| "confident_error_rate": 0.0 |
| }, |
| "hellaswag": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.49746280523283204, |
| "brier_soft": 0.49746280523283204, |
| "nll": 0.9473712181945483, |
| "nll_soft": 0.9473712181945483, |
| "ece": 0.12692415052435999, |
| "mean_confidence": 0.7047038532805092, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.08615518546819231, |
| "brier_soft": 0.08615518546819231, |
| "nll": 0.17699309707903643, |
| "nll_soft": 0.17699309707903643, |
| "ece": 0.051551565123817686, |
| "mean_confidence": 0.9526590211967746, |
| "confident_error_rate": 0.03 |
| }, |
| "ledgar": { |
| "count": 100, |
| "accuracy": 0.68, |
| "brier": 0.4142293130680919, |
| "brier_soft": 0.4142293130680919, |
| "nll": 1.4755731736027116, |
| "nll_soft": 1.4755731736027116, |
| "ece": 0.15435695147205214, |
| "mean_confidence": 0.7682162120881288, |
| "confident_error_rate": 0.02 |
| }, |
| "liar": { |
| "count": 100, |
| "accuracy": 0.26, |
| "brier": 0.7907528544471196, |
| "brier_soft": 0.7907528544471196, |
| "nll": 1.658049573925658, |
| "nll_soft": 1.658049573925658, |
| "ece": 0.07532040360970521, |
| "mean_confidence": 0.2625787670151657, |
| "confident_error_rate": 0.0, |
| "score_mae": 1.2346387458918446 |
| }, |
| "measuring_hate_speech": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.3185735935493244, |
| "brier_soft": 0.18106387920950776, |
| "nll": 0.581675618508808, |
| "nll_soft": 0.7139047866727517, |
| "ece": 0.13537234371633203, |
| "mean_confidence": 0.6746276562836679, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6125081186331043 |
| }, |
| "medqa": { |
| "count": 100, |
| "accuracy": 0.46, |
| "brier": 0.6816384464374701, |
| "brier_soft": 0.6816384464374701, |
| "nll": 1.2588469439999672, |
| "nll_soft": 1.2588469439999672, |
| "ece": 0.0997822415272474, |
| "mean_confidence": 0.4361856453821915, |
| "confident_error_rate": 0.0 |
| }, |
| "mnli": { |
| "count": 100, |
| "accuracy": 0.86, |
| "brier": 0.24351423712953862, |
| "brier_soft": 0.24351423712953862, |
| "nll": 0.44430237955026153, |
| "nll_soft": 0.44430237955026153, |
| "ece": 0.09311606611146102, |
| "mean_confidence": 0.8489913180528305, |
| "confident_error_rate": 0.05 |
| }, |
| "mrpc": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.23020609099215872, |
| "brier_soft": 0.23020609099215872, |
| "nll": 0.3654669899777431, |
| "nll_soft": 0.3654669899777431, |
| "ece": 0.03948396446737909, |
| "mean_confidence": 0.8373023804118128, |
| "confident_error_rate": 0.01 |
| }, |
| "multirc": { |
| "count": 125, |
| "accuracy": 0.688, |
| "brier": 0.5292844042129581, |
| "brier_soft": 0.5292844042129581, |
| "nll": 1.0655363765667019, |
| "nll_soft": 1.0655363765667019, |
| "ece": 0.23098717062751162, |
| "mean_confidence": 0.900382901571943, |
| "confident_error_rate": 0.192, |
| "evidence_auc": 0.9813333333333333 |
| }, |
| "openbookqa": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.49150152955300946, |
| "brier_soft": 0.49150152955300946, |
| "nll": 0.8837142098093151, |
| "nll_soft": 0.8837142098093151, |
| "ece": 0.13766437961219358, |
| "mean_confidence": 0.7434096264929048, |
| "confident_error_rate": 0.03 |
| }, |
| "piqa": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.47026935359907074, |
| "brier_soft": 0.47026935359907074, |
| "nll": 0.7289998840168692, |
| "nll_soft": 0.7289998840168692, |
| "ece": 0.12827029316468147, |
| "mean_confidence": 0.7482702931646813, |
| "confident_error_rate": 0.06 |
| }, |
| "policy_application": { |
| "count": 150, |
| "accuracy": 0.8, |
| "brier": 0.2581607926971296, |
| "brier_soft": 0.2581607926971296, |
| "nll": 0.41528686587922886, |
| "nll_soft": 0.41528686587922886, |
| "ece": 0.059914612791464726, |
| "mean_confidence": 0.8307312545326156, |
| "confident_error_rate": 0.02, |
| "evidence_auc": 0.9849574885546108 |
| }, |
| "policy_hard": { |
| "count": 300, |
| "accuracy": 0.7166666666666667, |
| "brier": 0.3682974306593614, |
| "brier_soft": 0.3682974306593614, |
| "nll": 0.581751640425408, |
| "nll_soft": 0.581751640425408, |
| "ece": 0.08515304195728496, |
| "mean_confidence": 0.7982964218132859, |
| "confident_error_rate": 0.03666666666666667, |
| "evidence_auc": 0.9403935185185185 |
| }, |
| "qqp": { |
| "count": 100, |
| "accuracy": 0.87, |
| "brier": 0.20985061313216197, |
| "brier_soft": 0.20985061313216197, |
| "nll": 0.3315568314454423, |
| "nll_soft": 0.3315568314454423, |
| "ece": 0.05314184203433264, |
| "mean_confidence": 0.8452123589462218, |
| "confident_error_rate": 0.01 |
| }, |
| "race": { |
| "count": 125, |
| "accuracy": 0.608, |
| "brier": 0.512913665441083, |
| "brier_soft": 0.512913665441083, |
| "nll": 1.0335832427965328, |
| "nll_soft": 1.0335832427965328, |
| "ece": 0.1794652275820026, |
| "mean_confidence": 0.7874652275820027, |
| "confident_error_rate": 0.048, |
| "evidence_auc": 0.9938666666666667 |
| }, |
| "rotten_tomatoes": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.14514149028488538, |
| "brier_soft": 0.14514149028488538, |
| "nll": 0.23569742847009142, |
| "nll_soft": 0.23569742847009142, |
| "ece": 0.04551142960475208, |
| "mean_confidence": 0.9240298023014982, |
| "confident_error_rate": 0.02 |
| }, |
| "rte": { |
| "count": 100, |
| "accuracy": 0.82, |
| "brier": 0.27430090407063495, |
| "brier_soft": 0.27430090407063495, |
| "nll": 0.4232667992998909, |
| "nll_soft": 0.4232667992998909, |
| "ece": 0.09631272637080687, |
| "mean_confidence": 0.8868213605094737, |
| "confident_error_rate": 0.04 |
| }, |
| "scotus": { |
| "count": 100, |
| "accuracy": 0.23, |
| "brier": 0.8962726116111268, |
| "brier_soft": 0.8962726116111268, |
| "nll": 2.4114340329303783, |
| "nll_soft": 2.4114340329303783, |
| "ece": 0.07696578546960535, |
| "mean_confidence": 0.2232808369779372, |
| "confident_error_rate": 0.0 |
| }, |
| "shp": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.5133731718674572, |
| "brier_soft": 0.12294510399950838, |
| "nll": 0.7070193959535053, |
| "nll_soft": 0.706911604249511, |
| "ece": 0.11601680133326055, |
| "mean_confidence": 0.564691586894939, |
| "confident_error_rate": 0.0 |
| }, |
| "sms_spam": { |
| "count": 100, |
| "accuracy": 0.98, |
| "brier": 0.02781846073892523, |
| "brier_soft": 0.02781846073892523, |
| "nll": 0.05762225915879506, |
| "nll_soft": 0.05762225915879506, |
| "ece": 0.01618236698957672, |
| "mean_confidence": 0.9780270650993808, |
| "confident_error_rate": 0.01 |
| }, |
| "snli": { |
| "count": 100, |
| "accuracy": 0.85, |
| "brier": 0.23107744397914165, |
| "brier_soft": 0.23107744397914165, |
| "nll": 0.3998605316362933, |
| "nll_soft": 0.3998605316362933, |
| "ece": 0.09381460196245499, |
| "mean_confidence": 0.8989207042720484, |
| "confident_error_rate": 0.02 |
| }, |
| "social_iqa": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.4848815820131893, |
| "brier_soft": 0.4848815820131893, |
| "nll": 0.8703980107195213, |
| "nll_soft": 0.8703980107195213, |
| "ece": 0.11820744837916289, |
| "mean_confidence": 0.7576265331761537, |
| "confident_error_rate": 0.04 |
| }, |
| "sst2": { |
| "count": 100, |
| "accuracy": 0.91, |
| "brier": 0.15770746238001465, |
| "brier_soft": 0.15770746238001465, |
| "nll": 0.30265794624945574, |
| "nll_soft": 0.30265794624945574, |
| "ece": 0.07956502250027958, |
| "mean_confidence": 0.9699946245692422, |
| "confident_error_rate": 0.06 |
| }, |
| "sst5": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.5572210951068667, |
| "brier_soft": 0.5572210951068667, |
| "nll": 1.0204819519606052, |
| "nll_soft": 1.0204819519606052, |
| "ece": 0.11556809596653374, |
| "mean_confidence": 0.48768009404633483, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.564475534398079 |
| }, |
| "stance_abortion": { |
| "count": 60, |
| "accuracy": 0.6166666666666667, |
| "brier": 0.5308687882209576, |
| "brier_soft": 0.5308687882209576, |
| "nll": 0.8600162545970034, |
| "nll_soft": 0.8600162545970034, |
| "ece": 0.11677183544136102, |
| "mean_confidence": 0.6911205564726794, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stance_atheism": { |
| "count": 60, |
| "accuracy": 0.7166666666666667, |
| "brier": 0.3727709846617827, |
| "brier_soft": 0.3727709846617827, |
| "nll": 0.629708103318696, |
| "nll_soft": 0.629708103318696, |
| "ece": 0.14890939964121483, |
| "mean_confidence": 0.7972844673470334, |
| "confident_error_rate": 0.05 |
| }, |
| "stance_feminist": { |
| "count": 60, |
| "accuracy": 0.6, |
| "brier": 0.4991315791236882, |
| "brier_soft": 0.4991315791236882, |
| "nll": 0.7790695047472594, |
| "nll_soft": 0.7790695047472594, |
| "ece": 0.16714938414424801, |
| "mean_confidence": 0.6587958364747021, |
| "confident_error_rate": 0.0 |
| }, |
| "stance_hillary": { |
| "count": 60, |
| "accuracy": 0.65, |
| "brier": 0.481593529336794, |
| "brier_soft": 0.481593529336794, |
| "nll": 0.812460304771394, |
| "nll_soft": 0.812460304771394, |
| "ece": 0.10841889152584662, |
| "mean_confidence": 0.7186442555046334, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stsb": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.6188782658365521, |
| "brier_soft": 0.3386810341594646, |
| "nll": 1.1508211209842052, |
| "nll_soft": 1.171402863078105, |
| "ece": 0.08542348846594641, |
| "mean_confidence": 0.5125793797409909, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.633302018694645 |
| }, |
| "subjectivity": { |
| "count": 100, |
| "accuracy": 0.92, |
| "brier": 0.09832629989090316, |
| "brier_soft": 0.09832629989090316, |
| "nll": 0.14857302467798977, |
| "nll_soft": 0.14857302467798977, |
| "ece": 0.0557927543533417, |
| "mean_confidence": 0.9442140077349405, |
| "confident_error_rate": 0.01 |
| }, |
| "swag": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.5633899010369295, |
| "brier_soft": 0.5633899010369295, |
| "nll": 1.0145999029200479, |
| "nll_soft": 1.0145999029200479, |
| "ece": 0.15183891657379386, |
| "mean_confidence": 0.7026384657291138, |
| "confident_error_rate": 0.01 |
| }, |
| "trec": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.08352253816512832, |
| "brier_soft": 0.08352253816512832, |
| "nll": 0.17957868443766156, |
| "nll_soft": 0.17957868443766156, |
| "ece": 0.031543798752471444, |
| "mean_confidence": 0.9679757926768224, |
| "confident_error_rate": 0.03 |
| }, |
| "tweet_hate": { |
| "count": 100, |
| "accuracy": 0.51, |
| "brier": 0.6232443102371671, |
| "brier_soft": 0.6232443102371671, |
| "nll": 0.8701818950225269, |
| "nll_soft": 0.8701818950225269, |
| "ece": 0.246043710359219, |
| "mean_confidence": 0.7542534383625626, |
| "confident_error_rate": 0.03 |
| }, |
| "tweet_sentiment": { |
| "count": 100, |
| "accuracy": 0.69, |
| "brier": 0.42086587193042996, |
| "brier_soft": 0.42086587193042996, |
| "nll": 0.7018752052311069, |
| "nll_soft": 0.7018752052311069, |
| "ece": 0.14639545763505046, |
| "mean_confidence": 0.582871891215126, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.37887902199006457 |
| }, |
| "vitaminc": { |
| "count": 100, |
| "accuracy": 0.71, |
| "brier": 0.40613955772862637, |
| "brier_soft": 0.40613955772862637, |
| "nll": 0.7262548068636012, |
| "nll_soft": 0.7262548068636012, |
| "ece": 0.0879470348809155, |
| "mean_confidence": 0.7137166080790087, |
| "confident_error_rate": 0.02 |
| }, |
| "wic": { |
| "count": 100, |
| "accuracy": 0.65, |
| "brier": 0.43665318483488696, |
| "brier_soft": 0.43665318483488696, |
| "nll": 0.6246472481676419, |
| "nll_soft": 0.6246472481676419, |
| "ece": 0.05619134355360803, |
| "mean_confidence": 0.6504787399551206, |
| "confident_error_rate": 0.0 |
| }, |
| "winogrande": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.6405505365444161, |
| "brier_soft": 0.6405505365444161, |
| "nll": 0.914099327330304, |
| "nll_soft": 0.914099327330304, |
| "ece": 0.2568558046429183, |
| "mean_confidence": 0.7468558046429183, |
| "confident_error_rate": 0.06 |
| }, |
| "yelp": { |
| "count": 100, |
| "accuracy": 0.57, |
| "brier": 0.5235143978590316, |
| "brier_soft": 0.5235143978590316, |
| "nll": 0.874960406158555, |
| "nll_soft": 0.874960406158555, |
| "ece": 0.10105406432593476, |
| "mean_confidence": 0.5762614684529703, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4971305399776537 |
| } |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "count": 2, |
| "confidence": 0.07557000005634865, |
| "accuracy": 0.0 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "count": 50, |
| "confidence": 0.17205340978732103, |
| "accuracy": 0.26 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "count": 167, |
| "confidence": 0.24791877265231055, |
| "accuracy": 0.2275449101796407 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "count": 194, |
| "confidence": 0.35460917877545167, |
| "accuracy": 0.36597938144329895 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "count": 538, |
| "confidence": 0.4580273683780515, |
| "accuracy": 0.4479553903345725 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "count": 747, |
| "confidence": 0.5527424316476833, |
| "accuracy": 0.5140562248995983 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "count": 614, |
| "confidence": 0.6495728910347323, |
| "accuracy": 0.5830618892508144 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "count": 614, |
| "confidence": 0.7548405190885715, |
| "accuracy": 0.6742671009771987 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "count": 809, |
| "confidence": 0.8532398931154401, |
| "accuracy": 0.7354758961681088 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "count": 2378, |
| "confidence": 0.9718801392189761, |
| "accuracy": 0.9234650967199327 |
| } |
| ], |
| "seconds": 16.97616457939148, |
| "model": "runs/assay-0.6b-v4", |
| "data": "data/v4/dev.jsonl" |
| } |
