{
  "date": "2026-09-24",
  "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72",
  "benchmarks": {
    "typed-decisions": {
      "count": 2000,
      "correct": 1463,
      "answered": 2000,
      "accuracy": 0.7315,
      "coverage": 1.0,
      "jev_reference": 0.727,
      "delta": 0.0045000000000000595,
      "statuses": {
        "ok": 2000
      }
    },
    "agnews": {
      "count": 100,
      "correct": 94,
      "answered": 100,
      "accuracy": 0.94,
      "coverage": 1.0,
      "jev_reference": 0.91,
      "delta": 0.029999999999999916,
      "statuses": {
        "ok": 100
      }
    },
    "emotiondair": {
      "count": 100,
      "correct": 86,
      "answered": 100,
      "accuracy": 0.86,
      "coverage": 1.0,
      "jev_reference": 0.48,
      "delta": 0.38,
      "statuses": {
        "ok": 100
      }
    },
    "banking77": {
      "count": 100,
      "correct": 64,
      "answered": 99,
      "accuracy": 0.64,
      "coverage": 0.99,
      "jev_reference": 0.87,
      "delta": -0.22999999999999998,
      "statuses": {
        "ok": 99,
        "abstained": 1
      }
    }
  },
  "typed": {
    "choice": {
      "count": 600,
      "correct": 428,
      "accuracy": 0.7133333333333334
    },
    "noul": {
      "count": 600,
      "correct": 484,
      "accuracy": 0.8066666666666666
    },
    "score": {
      "count": 800,
      "correct": 551,
      "accuracy": 0.68875
    }
  },
  "massive": {
    "per_language": {
      "mn-MN": {
        "count": 2974,
        "correct": 1618,
        "unsupported": 0,
        "accuracy": 0.5440484196368527
      },
      "af-ZA": {
        "count": 2974,
        "correct": 2253,
        "unsupported": 0,
        "accuracy": 0.7575655682582381
      },
      "el-GR": {
        "count": 2974,
        "correct": 2244,
        "unsupported": 0,
        "accuracy": 0.7545393409549428
      },
      "ta-IN": {
        "count": 2974,
        "correct": 1840,
        "unsupported": 0,
        "accuracy": 0.6186953597848016
      },
      "ar-SA": {
        "count": 2974,
        "correct": 2118,
        "unsupported": 0,
        "accuracy": 0.7121721587088097
      },
      "ca-ES": {
        "count": 2974,
        "correct": 2259,
        "unsupported": 0,
        "accuracy": 0.7595830531271015
      },
      "ur-PK": {
        "count": 2974,
        "correct": 2057,
        "unsupported": 0,
        "accuracy": 0.691661062542031
      },
      "pl-PL": {
        "count": 2974,
        "correct": 2377,
        "unsupported": 0,
        "accuracy": 0.7992602555480834
      },
      "ko-KR": {
        "count": 2974,
        "correct": 2296,
        "unsupported": 0,
        "accuracy": 0.7720242098184263
      },
      "az-AZ": {
        "count": 2974,
        "correct": 2166,
        "unsupported": 0,
        "accuracy": 0.7283120376597175
      },
      "da-DK": {
        "count": 2974,
        "correct": 2416,
        "unsupported": 0,
        "accuracy": 0.812373907195696
      },
      "kn-IN": {
        "count": 2974,
        "correct": 1654,
        "unsupported": 0,
        "accuracy": 0.5561533288500337
      },
      "tl-PH": {
        "count": 2974,
        "correct": 2146,
        "unsupported": 0,
        "accuracy": 0.7215870880968392
      },
      "is-IS": {
        "count": 2974,
        "correct": 1961,
        "unsupported": 0,
        "accuracy": 0.6593813046402152
      },
      "lv-LV": {
        "count": 2974,
        "correct": 1960,
        "unsupported": 0,
        "accuracy": 0.6590450571620713
      },
      "it-IT": {
        "count": 2974,
        "correct": 2432,
        "unsupported": 0,
        "accuracy": 0.8177538668459986
      },
      "es-ES": {
        "count": 2974,
        "correct": 2531,
        "unsupported": 0,
        "accuracy": 0.8510423671822461
      },
      "fr-FR": {
        "count": 2974,
        "correct": 2515,
        "unsupported": 0,
        "accuracy": 0.8456624075319435
      },
      "ml-IN": {
        "count": 2974,
        "correct": 1653,
        "unsupported": 0,
        "accuracy": 0.5558170813718897
      },
      "km-KH": {
        "count": 2974,
        "correct": 1421,
        "unsupported": 0,
        "accuracy": 0.47780766644250167
      },
      "fa-IR": {
        "count": 2974,
        "correct": 2382,
        "unsupported": 0,
        "accuracy": 0.800941492938803
      },
      "sw-KE": {
        "count": 2974,
        "correct": 1655,
        "unsupported": 0,
        "accuracy": 0.5564895763281775
      },
      "en-US": {
        "count": 2974,
        "correct": 2580,
        "unsupported": 0,
        "accuracy": 0.867518493611298
      },
      "tr-TR": {
        "count": 2974,
        "correct": 2388,
        "unsupported": 0,
        "accuracy": 0.8029589778076665
      },
      "bn-BD": {
        "count": 2974,
        "correct": 1972,
        "unsupported": 0,
        "accuracy": 0.6630800268997983
      },
      "he-IL": {
        "count": 2974,
        "correct": 2046,
        "unsupported": 0,
        "accuracy": 0.6879623402824478
      },
      "te-IN": {
        "count": 2974,
        "correct": 1900,
        "unsupported": 0,
        "accuracy": 0.6388702084734365
      },
      "pt-PT": {
        "count": 2974,
        "correct": 2565,
        "unsupported": 0,
        "accuracy": 0.8624747814391392
      },
      "ka-GE": {
        "count": 2974,
        "correct": 1501,
        "unsupported": 0,
        "accuracy": 0.5047074646940148
      },
      "ja-JP": {
        "count": 2974,
        "correct": 2446,
        "unsupported": 0,
        "accuracy": 0.8224613315400134
      },
      "id-ID": {
        "count": 2974,
        "correct": 2415,
        "unsupported": 0,
        "accuracy": 0.8120376597175522
      },
      "ru-RU": {
        "count": 2974,
        "correct": 2450,
        "unsupported": 0,
        "accuracy": 0.8238063214525891
      },
      "hy-AM": {
        "count": 2974,
        "correct": 1818,
        "unsupported": 0,
        "accuracy": 0.6112979152656355
      },
      "nb-NO": {
        "count": 2974,
        "correct": 2391,
        "unsupported": 0,
        "accuracy": 0.8039677202420982
      },
      "ms-MY": {
        "count": 2974,
        "correct": 2306,
        "unsupported": 0,
        "accuracy": 0.7753866845998655
      },
      "sq-AL": {
        "count": 2974,
        "correct": 2035,
        "unsupported": 0,
        "accuracy": 0.6842636180228648
      },
      "sv-SE": {
        "count": 2974,
        "correct": 2362,
        "unsupported": 0,
        "accuracy": 0.7942165433759247
      },
      "fi-FI": {
        "count": 2974,
        "correct": 2096,
        "unsupported": 0,
        "accuracy": 0.7047747141896435
      },
      "th-TH": {
        "count": 2974,
        "correct": 2299,
        "unsupported": 0,
        "accuracy": 0.7730329522528581
      },
      "de-DE": {
        "count": 2974,
        "correct": 2426,
        "unsupported": 0,
        "accuracy": 0.8157363819771352
      },
      "vi-VN": {
        "count": 2974,
        "correct": 2224,
        "unsupported": 0,
        "accuracy": 0.7478143913920645
      },
      "my-MM": {
        "count": 2974,
        "correct": 1513,
        "unsupported": 0,
        "accuracy": 0.5087424344317417
      },
      "jv-ID": {
        "count": 2974,
        "correct": 1802,
        "unsupported": 0,
        "accuracy": 0.6059179556153329
      },
      "sl-SL": {
        "count": 2974,
        "correct": 2147,
        "unsupported": 0,
        "accuracy": 0.7219233355749832
      },
      "hi-IN": {
        "count": 2974,
        "correct": 2237,
        "unsupported": 0,
        "accuracy": 0.7521856086079355
      },
      "nl-NL": {
        "count": 2974,
        "correct": 2449,
        "unsupported": 0,
        "accuracy": 0.8234700739744452
      },
      "zh-CN": {
        "count": 2974,
        "correct": 2487,
        "unsupported": 0,
        "accuracy": 0.8362474781439139
      },
      "hu-HU": {
        "count": 2974,
        "correct": 2161,
        "unsupported": 0,
        "accuracy": 0.726630800268998
      },
      "ro-RO": {
        "count": 2974,
        "correct": 2265,
        "unsupported": 0,
        "accuracy": 0.761600537995965
      },
      "zh-TW": {
        "count": 2974,
        "correct": 2365,
        "unsupported": 0,
        "accuracy": 0.7952252858103565
      },
      "cy-GB": {
        "count": 2974,
        "correct": 1639,
        "unsupported": 0,
        "accuracy": 0.551109616677875
      },
      "am-ET": {
        "count": 2974,
        "correct": 1334,
        "unsupported": 0,
        "accuracy": 0.4485541358439812
      }
    },
    "macro_accuracy": 0.7149979307847499,
    "count": 154648,
    "seconds": 192.6575419740002,
    "examples_per_second": 802.7092965863246
  },
  "diagnostics": {
    "diagnostic:pt": {
      "count": 3,
      "accuracy": 1.0,
      "worst_order_accuracy": 1.0,
      "consistency": 1.0,
      "brier_regret": 7.886988304767776e-10,
      "worst_order_brier": 2.467773398956191e-09,
      "nll": 2.103891552285609e-05,
      "ece": 2.1038666961015196e-05
    },
    "diagnostic:en": {
      "count": 3,
      "accuracy": 1.0,
      "worst_order_accuracy": 1.0,
      "consistency": 1.0,
      "brier_regret": 1.1595274914276354e-11,
      "worst_order_brier": 5.446226103608541e-11,
      "nll": 2.528017219883227e-06,
      "ece": 2.528013180791433e-06
    },
    "diagnostic-alias:pt": {
      "count": 2,
      "accuracy": 1.0,
      "worst_order_accuracy": 1.0,
      "consistency": 1.0,
      "brier_regret": 2.7625965108563243e-06,
      "worst_order_brier": 5.561015463157067e-06,
      "nll": 0.000956396971946862,
      "ece": 0.0009555747324743735
    },
    "diagnostic-alias:en": {
      "count": 2,
      "accuracy": 1.0,
      "worst_order_accuracy": 1.0,
      "consistency": 1.0,
      "brier_regret": 1.0381946113714448e-06,
      "worst_order_brier": 2.0757069438387377e-06,
      "nll": 0.0005331441533180119,
      "ece": 0.0005328709204919235
    }
  }
}