Julia-1 / metrics /accuracy-20260924.json
kleeedolinux
Publish Julia 1 model and Python runtime
5278d6b
Raw
History Blame Contribute Delete
10.1 kB
{
"date": "2026-09-24",
"weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72",
"benchmarks": {
"typed-decisions": {
"count": 2000,
"correct": 1463,
"answered": 2000,
"accuracy": 0.7315,
"coverage": 1.0,
"jev_reference": 0.727,
"delta": 0.0045000000000000595,
"statuses": {
"ok": 2000
}
},
"agnews": {
"count": 100,
"correct": 94,
"answered": 100,
"accuracy": 0.94,
"coverage": 1.0,
"jev_reference": 0.91,
"delta": 0.029999999999999916,
"statuses": {
"ok": 100
}
},
"emotiondair": {
"count": 100,
"correct": 86,
"answered": 100,
"accuracy": 0.86,
"coverage": 1.0,
"jev_reference": 0.48,
"delta": 0.38,
"statuses": {
"ok": 100
}
},
"banking77": {
"count": 100,
"correct": 64,
"answered": 99,
"accuracy": 0.64,
"coverage": 0.99,
"jev_reference": 0.87,
"delta": -0.22999999999999998,
"statuses": {
"ok": 99,
"abstained": 1
}
}
},
"typed": {
"choice": {
"count": 600,
"correct": 428,
"accuracy": 0.7133333333333334
},
"noul": {
"count": 600,
"correct": 484,
"accuracy": 0.8066666666666666
},
"score": {
"count": 800,
"correct": 551,
"accuracy": 0.68875
}
},
"massive": {
"per_language": {
"mn-MN": {
"count": 2974,
"correct": 1618,
"unsupported": 0,
"accuracy": 0.5440484196368527
},
"af-ZA": {
"count": 2974,
"correct": 2253,
"unsupported": 0,
"accuracy": 0.7575655682582381
},
"el-GR": {
"count": 2974,
"correct": 2244,
"unsupported": 0,
"accuracy": 0.7545393409549428
},
"ta-IN": {
"count": 2974,
"correct": 1840,
"unsupported": 0,
"accuracy": 0.6186953597848016
},
"ar-SA": {
"count": 2974,
"correct": 2118,
"unsupported": 0,
"accuracy": 0.7121721587088097
},
"ca-ES": {
"count": 2974,
"correct": 2259,
"unsupported": 0,
"accuracy": 0.7595830531271015
},
"ur-PK": {
"count": 2974,
"correct": 2057,
"unsupported": 0,
"accuracy": 0.691661062542031
},
"pl-PL": {
"count": 2974,
"correct": 2377,
"unsupported": 0,
"accuracy": 0.7992602555480834
},
"ko-KR": {
"count": 2974,
"correct": 2296,
"unsupported": 0,
"accuracy": 0.7720242098184263
},
"az-AZ": {
"count": 2974,
"correct": 2166,
"unsupported": 0,
"accuracy": 0.7283120376597175
},
"da-DK": {
"count": 2974,
"correct": 2416,
"unsupported": 0,
"accuracy": 0.812373907195696
},
"kn-IN": {
"count": 2974,
"correct": 1654,
"unsupported": 0,
"accuracy": 0.5561533288500337
},
"tl-PH": {
"count": 2974,
"correct": 2146,
"unsupported": 0,
"accuracy": 0.7215870880968392
},
"is-IS": {
"count": 2974,
"correct": 1961,
"unsupported": 0,
"accuracy": 0.6593813046402152
},
"lv-LV": {
"count": 2974,
"correct": 1960,
"unsupported": 0,
"accuracy": 0.6590450571620713
},
"it-IT": {
"count": 2974,
"correct": 2432,
"unsupported": 0,
"accuracy": 0.8177538668459986
},
"es-ES": {
"count": 2974,
"correct": 2531,
"unsupported": 0,
"accuracy": 0.8510423671822461
},
"fr-FR": {
"count": 2974,
"correct": 2515,
"unsupported": 0,
"accuracy": 0.8456624075319435
},
"ml-IN": {
"count": 2974,
"correct": 1653,
"unsupported": 0,
"accuracy": 0.5558170813718897
},
"km-KH": {
"count": 2974,
"correct": 1421,
"unsupported": 0,
"accuracy": 0.47780766644250167
},
"fa-IR": {
"count": 2974,
"correct": 2382,
"unsupported": 0,
"accuracy": 0.800941492938803
},
"sw-KE": {
"count": 2974,
"correct": 1655,
"unsupported": 0,
"accuracy": 0.5564895763281775
},
"en-US": {
"count": 2974,
"correct": 2580,
"unsupported": 0,
"accuracy": 0.867518493611298
},
"tr-TR": {
"count": 2974,
"correct": 2388,
"unsupported": 0,
"accuracy": 0.8029589778076665
},
"bn-BD": {
"count": 2974,
"correct": 1972,
"unsupported": 0,
"accuracy": 0.6630800268997983
},
"he-IL": {
"count": 2974,
"correct": 2046,
"unsupported": 0,
"accuracy": 0.6879623402824478
},
"te-IN": {
"count": 2974,
"correct": 1900,
"unsupported": 0,
"accuracy": 0.6388702084734365
},
"pt-PT": {
"count": 2974,
"correct": 2565,
"unsupported": 0,
"accuracy": 0.8624747814391392
},
"ka-GE": {
"count": 2974,
"correct": 1501,
"unsupported": 0,
"accuracy": 0.5047074646940148
},
"ja-JP": {
"count": 2974,
"correct": 2446,
"unsupported": 0,
"accuracy": 0.8224613315400134
},
"id-ID": {
"count": 2974,
"correct": 2415,
"unsupported": 0,
"accuracy": 0.8120376597175522
},
"ru-RU": {
"count": 2974,
"correct": 2450,
"unsupported": 0,
"accuracy": 0.8238063214525891
},
"hy-AM": {
"count": 2974,
"correct": 1818,
"unsupported": 0,
"accuracy": 0.6112979152656355
},
"nb-NO": {
"count": 2974,
"correct": 2391,
"unsupported": 0,
"accuracy": 0.8039677202420982
},
"ms-MY": {
"count": 2974,
"correct": 2306,
"unsupported": 0,
"accuracy": 0.7753866845998655
},
"sq-AL": {
"count": 2974,
"correct": 2035,
"unsupported": 0,
"accuracy": 0.6842636180228648
},
"sv-SE": {
"count": 2974,
"correct": 2362,
"unsupported": 0,
"accuracy": 0.7942165433759247
},
"fi-FI": {
"count": 2974,
"correct": 2096,
"unsupported": 0,
"accuracy": 0.7047747141896435
},
"th-TH": {
"count": 2974,
"correct": 2299,
"unsupported": 0,
"accuracy": 0.7730329522528581
},
"de-DE": {
"count": 2974,
"correct": 2426,
"unsupported": 0,
"accuracy": 0.8157363819771352
},
"vi-VN": {
"count": 2974,
"correct": 2224,
"unsupported": 0,
"accuracy": 0.7478143913920645
},
"my-MM": {
"count": 2974,
"correct": 1513,
"unsupported": 0,
"accuracy": 0.5087424344317417
},
"jv-ID": {
"count": 2974,
"correct": 1802,
"unsupported": 0,
"accuracy": 0.6059179556153329
},
"sl-SL": {
"count": 2974,
"correct": 2147,
"unsupported": 0,
"accuracy": 0.7219233355749832
},
"hi-IN": {
"count": 2974,
"correct": 2237,
"unsupported": 0,
"accuracy": 0.7521856086079355
},
"nl-NL": {
"count": 2974,
"correct": 2449,
"unsupported": 0,
"accuracy": 0.8234700739744452
},
"zh-CN": {
"count": 2974,
"correct": 2487,
"unsupported": 0,
"accuracy": 0.8362474781439139
},
"hu-HU": {
"count": 2974,
"correct": 2161,
"unsupported": 0,
"accuracy": 0.726630800268998
},
"ro-RO": {
"count": 2974,
"correct": 2265,
"unsupported": 0,
"accuracy": 0.761600537995965
},
"zh-TW": {
"count": 2974,
"correct": 2365,
"unsupported": 0,
"accuracy": 0.7952252858103565
},
"cy-GB": {
"count": 2974,
"correct": 1639,
"unsupported": 0,
"accuracy": 0.551109616677875
},
"am-ET": {
"count": 2974,
"correct": 1334,
"unsupported": 0,
"accuracy": 0.4485541358439812
}
},
"macro_accuracy": 0.7149979307847499,
"count": 154648,
"seconds": 192.6575419740002,
"examples_per_second": 802.7092965863246
},
"diagnostics": {
"diagnostic:pt": {
"count": 3,
"accuracy": 1.0,
"worst_order_accuracy": 1.0,
"consistency": 1.0,
"brier_regret": 7.886988304767776e-10,
"worst_order_brier": 2.467773398956191e-09,
"nll": 2.103891552285609e-05,
"ece": 2.1038666961015196e-05
},
"diagnostic:en": {
"count": 3,
"accuracy": 1.0,
"worst_order_accuracy": 1.0,
"consistency": 1.0,
"brier_regret": 1.1595274914276354e-11,
"worst_order_brier": 5.446226103608541e-11,
"nll": 2.528017219883227e-06,
"ece": 2.528013180791433e-06
},
"diagnostic-alias:pt": {
"count": 2,
"accuracy": 1.0,
"worst_order_accuracy": 1.0,
"consistency": 1.0,
"brier_regret": 2.7625965108563243e-06,
"worst_order_brier": 5.561015463157067e-06,
"nll": 0.000956396971946862,
"ece": 0.0009555747324743735
},
"diagnostic-alias:en": {
"count": 2,
"accuracy": 1.0,
"worst_order_accuracy": 1.0,
"consistency": 1.0,
"brier_regret": 1.0381946113714448e-06,
"worst_order_brier": 2.0757069438387377e-06,
"nll": 0.0005331441533180119,
"ece": 0.0005328709204919235
}
}
}