| { |
| "family": "Julia-1", |
| "variant": "posttrained-candidate", |
| "parent_weights_sha256": "e0fce3f70bc1add0fc5ea1555728129775f313dd8beb40bb4d0284b9cc4582e0", |
| "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72", |
| "step": 500, |
| "validation": { |
| "agnews": { |
| "count": 500, |
| "correct": 461, |
| "accuracy": 0.922, |
| "nll": 0.2361866206965157 |
| }, |
| "banking-joint": { |
| "count": 500, |
| "correct": 453, |
| "accuracy": 0.906, |
| "nll": 0.4732417627487641 |
| }, |
| "banking-rank": { |
| "count": 500, |
| "correct": 483, |
| "accuracy": 0.966, |
| "nll": 0.12442472780195588 |
| }, |
| "bilingual:en": { |
| "count": 409, |
| "correct": 396, |
| "accuracy": 0.9682151589242054, |
| "nll": 0.1019394528733742 |
| }, |
| "bilingual:pt": { |
| "count": 409, |
| "correct": 399, |
| "accuracy": 0.9755501222493888, |
| "nll": 0.07665246521243735 |
| }, |
| "emotiondair": { |
| "count": 500, |
| "correct": 457, |
| "accuracy": 0.914, |
| "nll": 0.4869592022011608 |
| }, |
| "jev:arc_challenge": { |
| "count": 200, |
| "correct": 57, |
| "accuracy": 0.285, |
| "nll": 4.898485339760228 |
| }, |
| "jev:mmlu": { |
| "count": 198, |
| "correct": 52, |
| "accuracy": 0.26262626262626265, |
| "nll": 5.022009487458182 |
| }, |
| "jev:mnli": { |
| "count": 200, |
| "correct": 121, |
| "accuracy": 0.605, |
| "nll": 2.405353136634182 |
| }, |
| "jev:sst5": { |
| "count": 200, |
| "correct": 81, |
| "accuracy": 0.405, |
| "nll": 3.058030146173687 |
| }, |
| "massive:af-ZA": { |
| "count": 200, |
| "correct": 160, |
| "accuracy": 0.8, |
| "nll": 0.7254367060870702 |
| }, |
| "massive:am-ET": { |
| "count": 200, |
| "correct": 91, |
| "accuracy": 0.455, |
| "nll": 1.8078553098358898 |
| }, |
| "massive:ar-SA": { |
| "count": 200, |
| "correct": 143, |
| "accuracy": 0.715, |
| "nll": 0.9127131833431383 |
| }, |
| "massive:az-AZ": { |
| "count": 200, |
| "correct": 150, |
| "accuracy": 0.75, |
| "nll": 0.8805768398220111 |
| }, |
| "massive:bn-BD": { |
| "count": 200, |
| "correct": 135, |
| "accuracy": 0.675, |
| "nll": 1.0898693716287062 |
| }, |
| "massive:ca-ES": { |
| "count": 200, |
| "correct": 149, |
| "accuracy": 0.745, |
| "nll": 0.9042317060852588 |
| }, |
| "massive:cy-GB": { |
| "count": 200, |
| "correct": 107, |
| "accuracy": 0.535, |
| "nll": 1.5896150314416113 |
| }, |
| "massive:da-DK": { |
| "count": 200, |
| "correct": 155, |
| "accuracy": 0.775, |
| "nll": 0.7864874714409028 |
| }, |
| "massive:de-DE": { |
| "count": 200, |
| "correct": 163, |
| "accuracy": 0.815, |
| "nll": 0.6609104527319621 |
| }, |
| "massive:el-GR": { |
| "count": 200, |
| "correct": 146, |
| "accuracy": 0.73, |
| "nll": 0.9561585124287475 |
| }, |
| "massive:en-US": { |
| "count": 200, |
| "correct": 171, |
| "accuracy": 0.855, |
| "nll": 0.4239840084077537 |
| }, |
| "massive:es-ES": { |
| "count": 200, |
| "correct": 170, |
| "accuracy": 0.85, |
| "nll": 0.4821026878847341 |
| }, |
| "massive:fa-IR": { |
| "count": 200, |
| "correct": 153, |
| "accuracy": 0.765, |
| "nll": 0.7513568875009335 |
| }, |
| "massive:fi-FI": { |
| "count": 200, |
| "correct": 136, |
| "accuracy": 0.68, |
| "nll": 1.0470849014139052 |
| }, |
| "massive:fr-FR": { |
| "count": 200, |
| "correct": 172, |
| "accuracy": 0.86, |
| "nll": 0.5079110052088927 |
| }, |
| "massive:he-IL": { |
| "count": 200, |
| "correct": 137, |
| "accuracy": 0.685, |
| "nll": 1.1409396331540986 |
| }, |
| "massive:hi-IN": { |
| "count": 200, |
| "correct": 148, |
| "accuracy": 0.74, |
| "nll": 0.8160677297670518 |
| }, |
| "massive:hu-HU": { |
| "count": 200, |
| "correct": 142, |
| "accuracy": 0.71, |
| "nll": 0.9946744119686869 |
| }, |
| "massive:hy-AM": { |
| "count": 200, |
| "correct": 117, |
| "accuracy": 0.585, |
| "nll": 1.3989824168977554 |
| }, |
| "massive:id-ID": { |
| "count": 200, |
| "correct": 162, |
| "accuracy": 0.81, |
| "nll": 0.7148022617771757 |
| }, |
| "massive:is-IS": { |
| "count": 200, |
| "correct": 140, |
| "accuracy": 0.7, |
| "nll": 1.0531424324651082 |
| }, |
| "massive:it-IT": { |
| "count": 200, |
| "correct": 152, |
| "accuracy": 0.76, |
| "nll": 0.7013908650180337 |
| }, |
| "massive:ja-JP": { |
| "count": 200, |
| "correct": 157, |
| "accuracy": 0.785, |
| "nll": 0.7928279227610233 |
| }, |
| "massive:jv-ID": { |
| "count": 200, |
| "correct": 124, |
| "accuracy": 0.62, |
| "nll": 1.349707674896094 |
| }, |
| "massive:ka-GE": { |
| "count": 200, |
| "correct": 96, |
| "accuracy": 0.48, |
| "nll": 1.7132205322599072 |
| }, |
| "massive:km-KH": { |
| "count": 200, |
| "correct": 96, |
| "accuracy": 0.48, |
| "nll": 1.6912345944473137 |
| }, |
| "massive:kn-IN": { |
| "count": 200, |
| "correct": 122, |
| "accuracy": 0.61, |
| "nll": 1.5155959311408833 |
| }, |
| "massive:ko-KR": { |
| "count": 200, |
| "correct": 158, |
| "accuracy": 0.79, |
| "nll": 0.677267845838551 |
| }, |
| "massive:lv-LV": { |
| "count": 200, |
| "correct": 130, |
| "accuracy": 0.65, |
| "nll": 1.0922244557466885 |
| }, |
| "massive:ml-IN": { |
| "count": 200, |
| "correct": 113, |
| "accuracy": 0.565, |
| "nll": 1.5063557911660717 |
| }, |
| "massive:mn-MN": { |
| "count": 200, |
| "correct": 100, |
| "accuracy": 0.5, |
| "nll": 1.578224376673286 |
| }, |
| "massive:ms-MY": { |
| "count": 200, |
| "correct": 158, |
| "accuracy": 0.79, |
| "nll": 0.7296966203279017 |
| }, |
| "massive:my-MM": { |
| "count": 200, |
| "correct": 100, |
| "accuracy": 0.5, |
| "nll": 1.6915851091994594 |
| }, |
| "massive:nb-NO": { |
| "count": 200, |
| "correct": 156, |
| "accuracy": 0.78, |
| "nll": 0.7617846100042545 |
| }, |
| "massive:nl-NL": { |
| "count": 200, |
| "correct": 167, |
| "accuracy": 0.835, |
| "nll": 0.5973099173809344 |
| }, |
| "massive:pl-PL": { |
| "count": 200, |
| "correct": 154, |
| "accuracy": 0.77, |
| "nll": 0.6892870971949061 |
| }, |
| "massive:pt-PT": { |
| "count": 200, |
| "correct": 169, |
| "accuracy": 0.845, |
| "nll": 0.5603875439414431 |
| }, |
| "massive:ro-RO": { |
| "count": 200, |
| "correct": 153, |
| "accuracy": 0.765, |
| "nll": 0.8143788994751282 |
| }, |
| "massive:ru-RU": { |
| "count": 200, |
| "correct": 169, |
| "accuracy": 0.845, |
| "nll": 0.4346364665858552 |
| }, |
| "massive:sl-SL": { |
| "count": 200, |
| "correct": 139, |
| "accuracy": 0.695, |
| "nll": 0.9863898763288126 |
| }, |
| "massive:sq-AL": { |
| "count": 200, |
| "correct": 144, |
| "accuracy": 0.72, |
| "nll": 1.008332237904707 |
| }, |
| "massive:sv-SE": { |
| "count": 200, |
| "correct": 157, |
| "accuracy": 0.785, |
| "nll": 0.775370210089804 |
| }, |
| "massive:sw-KE": { |
| "count": 200, |
| "correct": 97, |
| "accuracy": 0.485, |
| "nll": 1.7013354790948956 |
| }, |
| "massive:ta-IN": { |
| "count": 200, |
| "correct": 131, |
| "accuracy": 0.655, |
| "nll": 1.1463096412760028 |
| }, |
| "massive:te-IN": { |
| "count": 200, |
| "correct": 124, |
| "accuracy": 0.62, |
| "nll": 1.2817266688502815 |
| }, |
| "massive:th-TH": { |
| "count": 200, |
| "correct": 147, |
| "accuracy": 0.735, |
| "nll": 0.8718785998410745 |
| }, |
| "massive:tl-PH": { |
| "count": 200, |
| "correct": 140, |
| "accuracy": 0.7, |
| "nll": 0.9638123165387646 |
| }, |
| "massive:tr-TR": { |
| "count": 200, |
| "correct": 157, |
| "accuracy": 0.785, |
| "nll": 0.6703527633025136 |
| }, |
| "massive:ur-PK": { |
| "count": 200, |
| "correct": 131, |
| "accuracy": 0.655, |
| "nll": 1.1953078716722099 |
| }, |
| "massive:vi-VN": { |
| "count": 200, |
| "correct": 144, |
| "accuracy": 0.72, |
| "nll": 0.9832935351908162 |
| }, |
| "massive:zh-CN": { |
| "count": 200, |
| "correct": 156, |
| "accuracy": 0.78, |
| "nll": 0.6844688390764435 |
| }, |
| "massive:zh-TW": { |
| "count": 200, |
| "correct": 154, |
| "accuracy": 0.77, |
| "nll": 0.7012105530022036 |
| }, |
| "open-jev:customer-control-v1": { |
| "count": 96, |
| "correct": 93, |
| "accuracy": 0.96875, |
| "nll": 0.11922412518095221 |
| }, |
| "open-jev:painting-geometry-v1": { |
| "count": 192, |
| "correct": 153, |
| "accuracy": 0.796875, |
| "nll": 0.4879071857538497 |
| }, |
| "open-jev:reasoning-control-v1": { |
| "count": 131, |
| "correct": 68, |
| "accuracy": 0.5190839694656488, |
| "nll": 1.5929976853600538 |
| }, |
| "open-jev:snake-v1": { |
| "count": 200, |
| "correct": 158, |
| "accuracy": 0.79, |
| "nll": 0.4846087664592145 |
| }, |
| "open-jev:tic_tac_toe-v1": { |
| "count": 115, |
| "correct": 47, |
| "accuracy": 0.40869565217391307, |
| "nll": 1.2158676274170932 |
| }, |
| "open-jev:tile_platformer-v1": { |
| "count": 22, |
| "correct": 20, |
| "accuracy": 0.9090909090909091, |
| "nll": 0.09021685084244535 |
| }, |
| "open-jev:trex_runner-v1": { |
| "count": 2, |
| "correct": 1, |
| "accuracy": 0.5, |
| "nll": 1.2663163314336559 |
| }, |
| "open-jev:vizdoom-basic-v1": { |
| "count": 153, |
| "correct": 149, |
| "accuracy": 0.9738562091503268, |
| "nll": 0.03614709415145343 |
| }, |
| "open-jev:workflow-controls-v1/agent_trace_observability": { |
| "count": 71, |
| "correct": 71, |
| "accuracy": 1.0, |
| "nll": 0.0011360287241133358 |
| }, |
| "open-jev:workflow-controls-v1/customer_service": { |
| "count": 137, |
| "correct": 136, |
| "accuracy": 0.9927007299270073, |
| "nll": 0.030294906363817344 |
| }, |
| "open-jev:workflow-controls-v1/invoice_processing": { |
| "count": 112, |
| "correct": 110, |
| "accuracy": 0.9821428571428571, |
| "nll": 0.10505136013108395 |
| }, |
| "open-jev:workflow-controls-v1/security_incidents": { |
| "count": 121, |
| "correct": 121, |
| "accuracy": 1.0, |
| "nll": 0.009351040906769742 |
| }, |
| "replay:agnews": { |
| "count": 29, |
| "correct": 23, |
| "accuracy": 0.7931034482758621, |
| "nll": 0.5144041118900763 |
| }, |
| "replay:banking77": { |
| "count": 52, |
| "correct": 50, |
| "accuracy": 0.9615384615384616, |
| "nll": 0.1474100871587149 |
| }, |
| "replay:emotion": { |
| "count": 15, |
| "correct": 12, |
| "accuracy": 0.8, |
| "nll": 0.5748594660688843 |
| }, |
| "replay:gsm8k": { |
| "count": 200, |
| "correct": 196, |
| "accuracy": 0.98, |
| "nll": 0.047936887727627886 |
| }, |
| "replay:hellaswag": { |
| "count": 119, |
| "correct": 46, |
| "accuracy": 0.3865546218487395, |
| "nll": 3.136717304436293 |
| }, |
| "replay:mmlu": { |
| "count": 182, |
| "correct": 93, |
| "accuracy": 0.510989010989011, |
| "nll": 2.17472550274021 |
| }, |
| "replay:winogrande": { |
| "count": 200, |
| "correct": 107, |
| "accuracy": 0.535, |
| "nll": 1.151156462092322 |
| }, |
| "typed": { |
| "count": 500, |
| "correct": 365, |
| "accuracy": 0.73, |
| "nll": 2.027869497306916 |
| }, |
| "typed:choice": { |
| "count": 155, |
| "correct": 108, |
| "accuracy": 0.6967741935483871, |
| "nll": 2.5421524021866317 |
| }, |
| "typed:noul": { |
| "count": 142, |
| "correct": 115, |
| "accuracy": 0.8098591549295775, |
| "nll": 1.5555020525764138 |
| }, |
| "typed:score": { |
| "count": 203, |
| "correct": 142, |
| "accuracy": 0.6995073891625616, |
| "nll": 1.9656149499935025 |
| }, |
| "banking77": { |
| "count": 100, |
| "correct": 83, |
| "accuracy": 0.83, |
| "recall_at_16": 0.99, |
| "unsupported": 0 |
| } |
| }, |
| "quality_gate": false |
| } |
|
|
