| { |
| "date": "2026-09-24", |
| "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72", |
| "benchmarks": { |
| "typed-decisions": { |
| "count": 2000, |
| "correct": 1463, |
| "answered": 2000, |
| "accuracy": 0.7315, |
| "coverage": 1.0, |
| "jev_reference": 0.727, |
| "delta": 0.0045000000000000595, |
| "statuses": { |
| "ok": 2000 |
| } |
| }, |
| "agnews": { |
| "count": 100, |
| "correct": 94, |
| "answered": 100, |
| "accuracy": 0.94, |
| "coverage": 1.0, |
| "jev_reference": 0.91, |
| "delta": 0.029999999999999916, |
| "statuses": { |
| "ok": 100 |
| } |
| }, |
| "emotiondair": { |
| "count": 100, |
| "correct": 86, |
| "answered": 100, |
| "accuracy": 0.86, |
| "coverage": 1.0, |
| "jev_reference": 0.48, |
| "delta": 0.38, |
| "statuses": { |
| "ok": 100 |
| } |
| }, |
| "banking77": { |
| "count": 100, |
| "correct": 64, |
| "answered": 99, |
| "accuracy": 0.64, |
| "coverage": 0.99, |
| "jev_reference": 0.87, |
| "delta": -0.22999999999999998, |
| "statuses": { |
| "ok": 99, |
| "abstained": 1 |
| } |
| } |
| }, |
| "typed": { |
| "choice": { |
| "count": 600, |
| "correct": 428, |
| "accuracy": 0.7133333333333334 |
| }, |
| "noul": { |
| "count": 600, |
| "correct": 484, |
| "accuracy": 0.8066666666666666 |
| }, |
| "score": { |
| "count": 800, |
| "correct": 551, |
| "accuracy": 0.68875 |
| } |
| }, |
| "massive": { |
| "per_language": { |
| "mn-MN": { |
| "count": 2974, |
| "correct": 1618, |
| "unsupported": 0, |
| "accuracy": 0.5440484196368527 |
| }, |
| "af-ZA": { |
| "count": 2974, |
| "correct": 2253, |
| "unsupported": 0, |
| "accuracy": 0.7575655682582381 |
| }, |
| "el-GR": { |
| "count": 2974, |
| "correct": 2244, |
| "unsupported": 0, |
| "accuracy": 0.7545393409549428 |
| }, |
| "ta-IN": { |
| "count": 2974, |
| "correct": 1840, |
| "unsupported": 0, |
| "accuracy": 0.6186953597848016 |
| }, |
| "ar-SA": { |
| "count": 2974, |
| "correct": 2118, |
| "unsupported": 0, |
| "accuracy": 0.7121721587088097 |
| }, |
| "ca-ES": { |
| "count": 2974, |
| "correct": 2259, |
| "unsupported": 0, |
| "accuracy": 0.7595830531271015 |
| }, |
| "ur-PK": { |
| "count": 2974, |
| "correct": 2057, |
| "unsupported": 0, |
| "accuracy": 0.691661062542031 |
| }, |
| "pl-PL": { |
| "count": 2974, |
| "correct": 2377, |
| "unsupported": 0, |
| "accuracy": 0.7992602555480834 |
| }, |
| "ko-KR": { |
| "count": 2974, |
| "correct": 2296, |
| "unsupported": 0, |
| "accuracy": 0.7720242098184263 |
| }, |
| "az-AZ": { |
| "count": 2974, |
| "correct": 2166, |
| "unsupported": 0, |
| "accuracy": 0.7283120376597175 |
| }, |
| "da-DK": { |
| "count": 2974, |
| "correct": 2416, |
| "unsupported": 0, |
| "accuracy": 0.812373907195696 |
| }, |
| "kn-IN": { |
| "count": 2974, |
| "correct": 1654, |
| "unsupported": 0, |
| "accuracy": 0.5561533288500337 |
| }, |
| "tl-PH": { |
| "count": 2974, |
| "correct": 2146, |
| "unsupported": 0, |
| "accuracy": 0.7215870880968392 |
| }, |
| "is-IS": { |
| "count": 2974, |
| "correct": 1961, |
| "unsupported": 0, |
| "accuracy": 0.6593813046402152 |
| }, |
| "lv-LV": { |
| "count": 2974, |
| "correct": 1960, |
| "unsupported": 0, |
| "accuracy": 0.6590450571620713 |
| }, |
| "it-IT": { |
| "count": 2974, |
| "correct": 2432, |
| "unsupported": 0, |
| "accuracy": 0.8177538668459986 |
| }, |
| "es-ES": { |
| "count": 2974, |
| "correct": 2531, |
| "unsupported": 0, |
| "accuracy": 0.8510423671822461 |
| }, |
| "fr-FR": { |
| "count": 2974, |
| "correct": 2515, |
| "unsupported": 0, |
| "accuracy": 0.8456624075319435 |
| }, |
| "ml-IN": { |
| "count": 2974, |
| "correct": 1653, |
| "unsupported": 0, |
| "accuracy": 0.5558170813718897 |
| }, |
| "km-KH": { |
| "count": 2974, |
| "correct": 1421, |
| "unsupported": 0, |
| "accuracy": 0.47780766644250167 |
| }, |
| "fa-IR": { |
| "count": 2974, |
| "correct": 2382, |
| "unsupported": 0, |
| "accuracy": 0.800941492938803 |
| }, |
| "sw-KE": { |
| "count": 2974, |
| "correct": 1655, |
| "unsupported": 0, |
| "accuracy": 0.5564895763281775 |
| }, |
| "en-US": { |
| "count": 2974, |
| "correct": 2580, |
| "unsupported": 0, |
| "accuracy": 0.867518493611298 |
| }, |
| "tr-TR": { |
| "count": 2974, |
| "correct": 2388, |
| "unsupported": 0, |
| "accuracy": 0.8029589778076665 |
| }, |
| "bn-BD": { |
| "count": 2974, |
| "correct": 1972, |
| "unsupported": 0, |
| "accuracy": 0.6630800268997983 |
| }, |
| "he-IL": { |
| "count": 2974, |
| "correct": 2046, |
| "unsupported": 0, |
| "accuracy": 0.6879623402824478 |
| }, |
| "te-IN": { |
| "count": 2974, |
| "correct": 1900, |
| "unsupported": 0, |
| "accuracy": 0.6388702084734365 |
| }, |
| "pt-PT": { |
| "count": 2974, |
| "correct": 2565, |
| "unsupported": 0, |
| "accuracy": 0.8624747814391392 |
| }, |
| "ka-GE": { |
| "count": 2974, |
| "correct": 1501, |
| "unsupported": 0, |
| "accuracy": 0.5047074646940148 |
| }, |
| "ja-JP": { |
| "count": 2974, |
| "correct": 2446, |
| "unsupported": 0, |
| "accuracy": 0.8224613315400134 |
| }, |
| "id-ID": { |
| "count": 2974, |
| "correct": 2415, |
| "unsupported": 0, |
| "accuracy": 0.8120376597175522 |
| }, |
| "ru-RU": { |
| "count": 2974, |
| "correct": 2450, |
| "unsupported": 0, |
| "accuracy": 0.8238063214525891 |
| }, |
| "hy-AM": { |
| "count": 2974, |
| "correct": 1818, |
| "unsupported": 0, |
| "accuracy": 0.6112979152656355 |
| }, |
| "nb-NO": { |
| "count": 2974, |
| "correct": 2391, |
| "unsupported": 0, |
| "accuracy": 0.8039677202420982 |
| }, |
| "ms-MY": { |
| "count": 2974, |
| "correct": 2306, |
| "unsupported": 0, |
| "accuracy": 0.7753866845998655 |
| }, |
| "sq-AL": { |
| "count": 2974, |
| "correct": 2035, |
| "unsupported": 0, |
| "accuracy": 0.6842636180228648 |
| }, |
| "sv-SE": { |
| "count": 2974, |
| "correct": 2362, |
| "unsupported": 0, |
| "accuracy": 0.7942165433759247 |
| }, |
| "fi-FI": { |
| "count": 2974, |
| "correct": 2096, |
| "unsupported": 0, |
| "accuracy": 0.7047747141896435 |
| }, |
| "th-TH": { |
| "count": 2974, |
| "correct": 2299, |
| "unsupported": 0, |
| "accuracy": 0.7730329522528581 |
| }, |
| "de-DE": { |
| "count": 2974, |
| "correct": 2426, |
| "unsupported": 0, |
| "accuracy": 0.8157363819771352 |
| }, |
| "vi-VN": { |
| "count": 2974, |
| "correct": 2224, |
| "unsupported": 0, |
| "accuracy": 0.7478143913920645 |
| }, |
| "my-MM": { |
| "count": 2974, |
| "correct": 1513, |
| "unsupported": 0, |
| "accuracy": 0.5087424344317417 |
| }, |
| "jv-ID": { |
| "count": 2974, |
| "correct": 1802, |
| "unsupported": 0, |
| "accuracy": 0.6059179556153329 |
| }, |
| "sl-SL": { |
| "count": 2974, |
| "correct": 2147, |
| "unsupported": 0, |
| "accuracy": 0.7219233355749832 |
| }, |
| "hi-IN": { |
| "count": 2974, |
| "correct": 2237, |
| "unsupported": 0, |
| "accuracy": 0.7521856086079355 |
| }, |
| "nl-NL": { |
| "count": 2974, |
| "correct": 2449, |
| "unsupported": 0, |
| "accuracy": 0.8234700739744452 |
| }, |
| "zh-CN": { |
| "count": 2974, |
| "correct": 2487, |
| "unsupported": 0, |
| "accuracy": 0.8362474781439139 |
| }, |
| "hu-HU": { |
| "count": 2974, |
| "correct": 2161, |
| "unsupported": 0, |
| "accuracy": 0.726630800268998 |
| }, |
| "ro-RO": { |
| "count": 2974, |
| "correct": 2265, |
| "unsupported": 0, |
| "accuracy": 0.761600537995965 |
| }, |
| "zh-TW": { |
| "count": 2974, |
| "correct": 2365, |
| "unsupported": 0, |
| "accuracy": 0.7952252858103565 |
| }, |
| "cy-GB": { |
| "count": 2974, |
| "correct": 1639, |
| "unsupported": 0, |
| "accuracy": 0.551109616677875 |
| }, |
| "am-ET": { |
| "count": 2974, |
| "correct": 1334, |
| "unsupported": 0, |
| "accuracy": 0.4485541358439812 |
| } |
| }, |
| "macro_accuracy": 0.7149979307847499, |
| "count": 154648, |
| "seconds": 192.6575419740002, |
| "examples_per_second": 802.7092965863246 |
| }, |
| "diagnostics": { |
| "diagnostic:pt": { |
| "count": 3, |
| "accuracy": 1.0, |
| "worst_order_accuracy": 1.0, |
| "consistency": 1.0, |
| "brier_regret": 7.886988304767776e-10, |
| "worst_order_brier": 2.467773398956191e-09, |
| "nll": 2.103891552285609e-05, |
| "ece": 2.1038666961015196e-05 |
| }, |
| "diagnostic:en": { |
| "count": 3, |
| "accuracy": 1.0, |
| "worst_order_accuracy": 1.0, |
| "consistency": 1.0, |
| "brier_regret": 1.1595274914276354e-11, |
| "worst_order_brier": 5.446226103608541e-11, |
| "nll": 2.528017219883227e-06, |
| "ece": 2.528013180791433e-06 |
| }, |
| "diagnostic-alias:pt": { |
| "count": 2, |
| "accuracy": 1.0, |
| "worst_order_accuracy": 1.0, |
| "consistency": 1.0, |
| "brier_regret": 2.7625965108563243e-06, |
| "worst_order_brier": 5.561015463157067e-06, |
| "nll": 0.000956396971946862, |
| "ece": 0.0009555747324743735 |
| }, |
| "diagnostic-alias:en": { |
| "count": 2, |
| "accuracy": 1.0, |
| "worst_order_accuracy": 1.0, |
| "consistency": 1.0, |
| "brier_regret": 1.0381946113714448e-06, |
| "worst_order_brier": 2.0757069438387377e-06, |
| "nll": 0.0005331441533180119, |
| "ece": 0.0005328709204919235 |
| } |
| } |
| } |
|
|
