{
  "family": "Julia-1",
  "variant": "posttrained-candidate",
  "parent_weights_sha256": "e0fce3f70bc1add0fc5ea1555728129775f313dd8beb40bb4d0284b9cc4582e0",
  "weights_sha256": "df853bf7fe424420011f3d0c47a05d7341aa9eefa7fb9f203ea4aada4ad95b72",
  "step": 500,
  "validation": {
    "agnews": {
      "count": 500,
      "correct": 461,
      "accuracy": 0.922,
      "nll": 0.2361866206965157
    },
    "banking-joint": {
      "count": 500,
      "correct": 453,
      "accuracy": 0.906,
      "nll": 0.4732417627487641
    },
    "banking-rank": {
      "count": 500,
      "correct": 483,
      "accuracy": 0.966,
      "nll": 0.12442472780195588
    },
    "bilingual:en": {
      "count": 409,
      "correct": 396,
      "accuracy": 0.9682151589242054,
      "nll": 0.1019394528733742
    },
    "bilingual:pt": {
      "count": 409,
      "correct": 399,
      "accuracy": 0.9755501222493888,
      "nll": 0.07665246521243735
    },
    "emotiondair": {
      "count": 500,
      "correct": 457,
      "accuracy": 0.914,
      "nll": 0.4869592022011608
    },
    "jev:arc_challenge": {
      "count": 200,
      "correct": 57,
      "accuracy": 0.285,
      "nll": 4.898485339760228
    },
    "jev:mmlu": {
      "count": 198,
      "correct": 52,
      "accuracy": 0.26262626262626265,
      "nll": 5.022009487458182
    },
    "jev:mnli": {
      "count": 200,
      "correct": 121,
      "accuracy": 0.605,
      "nll": 2.405353136634182
    },
    "jev:sst5": {
      "count": 200,
      "correct": 81,
      "accuracy": 0.405,
      "nll": 3.058030146173687
    },
    "massive:af-ZA": {
      "count": 200,
      "correct": 160,
      "accuracy": 0.8,
      "nll": 0.7254367060870702
    },
    "massive:am-ET": {
      "count": 200,
      "correct": 91,
      "accuracy": 0.455,
      "nll": 1.8078553098358898
    },
    "massive:ar-SA": {
      "count": 200,
      "correct": 143,
      "accuracy": 0.715,
      "nll": 0.9127131833431383
    },
    "massive:az-AZ": {
      "count": 200,
      "correct": 150,
      "accuracy": 0.75,
      "nll": 0.8805768398220111
    },
    "massive:bn-BD": {
      "count": 200,
      "correct": 135,
      "accuracy": 0.675,
      "nll": 1.0898693716287062
    },
    "massive:ca-ES": {
      "count": 200,
      "correct": 149,
      "accuracy": 0.745,
      "nll": 0.9042317060852588
    },
    "massive:cy-GB": {
      "count": 200,
      "correct": 107,
      "accuracy": 0.535,
      "nll": 1.5896150314416113
    },
    "massive:da-DK": {
      "count": 200,
      "correct": 155,
      "accuracy": 0.775,
      "nll": 0.7864874714409028
    },
    "massive:de-DE": {
      "count": 200,
      "correct": 163,
      "accuracy": 0.815,
      "nll": 0.6609104527319621
    },
    "massive:el-GR": {
      "count": 200,
      "correct": 146,
      "accuracy": 0.73,
      "nll": 0.9561585124287475
    },
    "massive:en-US": {
      "count": 200,
      "correct": 171,
      "accuracy": 0.855,
      "nll": 0.4239840084077537
    },
    "massive:es-ES": {
      "count": 200,
      "correct": 170,
      "accuracy": 0.85,
      "nll": 0.4821026878847341
    },
    "massive:fa-IR": {
      "count": 200,
      "correct": 153,
      "accuracy": 0.765,
      "nll": 0.7513568875009335
    },
    "massive:fi-FI": {
      "count": 200,
      "correct": 136,
      "accuracy": 0.68,
      "nll": 1.0470849014139052
    },
    "massive:fr-FR": {
      "count": 200,
      "correct": 172,
      "accuracy": 0.86,
      "nll": 0.5079110052088927
    },
    "massive:he-IL": {
      "count": 200,
      "correct": 137,
      "accuracy": 0.685,
      "nll": 1.1409396331540986
    },
    "massive:hi-IN": {
      "count": 200,
      "correct": 148,
      "accuracy": 0.74,
      "nll": 0.8160677297670518
    },
    "massive:hu-HU": {
      "count": 200,
      "correct": 142,
      "accuracy": 0.71,
      "nll": 0.9946744119686869
    },
    "massive:hy-AM": {
      "count": 200,
      "correct": 117,
      "accuracy": 0.585,
      "nll": 1.3989824168977554
    },
    "massive:id-ID": {
      "count": 200,
      "correct": 162,
      "accuracy": 0.81,
      "nll": 0.7148022617771757
    },
    "massive:is-IS": {
      "count": 200,
      "correct": 140,
      "accuracy": 0.7,
      "nll": 1.0531424324651082
    },
    "massive:it-IT": {
      "count": 200,
      "correct": 152,
      "accuracy": 0.76,
      "nll": 0.7013908650180337
    },
    "massive:ja-JP": {
      "count": 200,
      "correct": 157,
      "accuracy": 0.785,
      "nll": 0.7928279227610233
    },
    "massive:jv-ID": {
      "count": 200,
      "correct": 124,
      "accuracy": 0.62,
      "nll": 1.349707674896094
    },
    "massive:ka-GE": {
      "count": 200,
      "correct": 96,
      "accuracy": 0.48,
      "nll": 1.7132205322599072
    },
    "massive:km-KH": {
      "count": 200,
      "correct": 96,
      "accuracy": 0.48,
      "nll": 1.6912345944473137
    },
    "massive:kn-IN": {
      "count": 200,
      "correct": 122,
      "accuracy": 0.61,
      "nll": 1.5155959311408833
    },
    "massive:ko-KR": {
      "count": 200,
      "correct": 158,
      "accuracy": 0.79,
      "nll": 0.677267845838551
    },
    "massive:lv-LV": {
      "count": 200,
      "correct": 130,
      "accuracy": 0.65,
      "nll": 1.0922244557466885
    },
    "massive:ml-IN": {
      "count": 200,
      "correct": 113,
      "accuracy": 0.565,
      "nll": 1.5063557911660717
    },
    "massive:mn-MN": {
      "count": 200,
      "correct": 100,
      "accuracy": 0.5,
      "nll": 1.578224376673286
    },
    "massive:ms-MY": {
      "count": 200,
      "correct": 158,
      "accuracy": 0.79,
      "nll": 0.7296966203279017
    },
    "massive:my-MM": {
      "count": 200,
      "correct": 100,
      "accuracy": 0.5,
      "nll": 1.6915851091994594
    },
    "massive:nb-NO": {
      "count": 200,
      "correct": 156,
      "accuracy": 0.78,
      "nll": 0.7617846100042545
    },
    "massive:nl-NL": {
      "count": 200,
      "correct": 167,
      "accuracy": 0.835,
      "nll": 0.5973099173809344
    },
    "massive:pl-PL": {
      "count": 200,
      "correct": 154,
      "accuracy": 0.77,
      "nll": 0.6892870971949061
    },
    "massive:pt-PT": {
      "count": 200,
      "correct": 169,
      "accuracy": 0.845,
      "nll": 0.5603875439414431
    },
    "massive:ro-RO": {
      "count": 200,
      "correct": 153,
      "accuracy": 0.765,
      "nll": 0.8143788994751282
    },
    "massive:ru-RU": {
      "count": 200,
      "correct": 169,
      "accuracy": 0.845,
      "nll": 0.4346364665858552
    },
    "massive:sl-SL": {
      "count": 200,
      "correct": 139,
      "accuracy": 0.695,
      "nll": 0.9863898763288126
    },
    "massive:sq-AL": {
      "count": 200,
      "correct": 144,
      "accuracy": 0.72,
      "nll": 1.008332237904707
    },
    "massive:sv-SE": {
      "count": 200,
      "correct": 157,
      "accuracy": 0.785,
      "nll": 0.775370210089804
    },
    "massive:sw-KE": {
      "count": 200,
      "correct": 97,
      "accuracy": 0.485,
      "nll": 1.7013354790948956
    },
    "massive:ta-IN": {
      "count": 200,
      "correct": 131,
      "accuracy": 0.655,
      "nll": 1.1463096412760028
    },
    "massive:te-IN": {
      "count": 200,
      "correct": 124,
      "accuracy": 0.62,
      "nll": 1.2817266688502815
    },
    "massive:th-TH": {
      "count": 200,
      "correct": 147,
      "accuracy": 0.735,
      "nll": 0.8718785998410745
    },
    "massive:tl-PH": {
      "count": 200,
      "correct": 140,
      "accuracy": 0.7,
      "nll": 0.9638123165387646
    },
    "massive:tr-TR": {
      "count": 200,
      "correct": 157,
      "accuracy": 0.785,
      "nll": 0.6703527633025136
    },
    "massive:ur-PK": {
      "count": 200,
      "correct": 131,
      "accuracy": 0.655,
      "nll": 1.1953078716722099
    },
    "massive:vi-VN": {
      "count": 200,
      "correct": 144,
      "accuracy": 0.72,
      "nll": 0.9832935351908162
    },
    "massive:zh-CN": {
      "count": 200,
      "correct": 156,
      "accuracy": 0.78,
      "nll": 0.6844688390764435
    },
    "massive:zh-TW": {
      "count": 200,
      "correct": 154,
      "accuracy": 0.77,
      "nll": 0.7012105530022036
    },
    "open-jev:customer-control-v1": {
      "count": 96,
      "correct": 93,
      "accuracy": 0.96875,
      "nll": 0.11922412518095221
    },
    "open-jev:painting-geometry-v1": {
      "count": 192,
      "correct": 153,
      "accuracy": 0.796875,
      "nll": 0.4879071857538497
    },
    "open-jev:reasoning-control-v1": {
      "count": 131,
      "correct": 68,
      "accuracy": 0.5190839694656488,
      "nll": 1.5929976853600538
    },
    "open-jev:snake-v1": {
      "count": 200,
      "correct": 158,
      "accuracy": 0.79,
      "nll": 0.4846087664592145
    },
    "open-jev:tic_tac_toe-v1": {
      "count": 115,
      "correct": 47,
      "accuracy": 0.40869565217391307,
      "nll": 1.2158676274170932
    },
    "open-jev:tile_platformer-v1": {
      "count": 22,
      "correct": 20,
      "accuracy": 0.9090909090909091,
      "nll": 0.09021685084244535
    },
    "open-jev:trex_runner-v1": {
      "count": 2,
      "correct": 1,
      "accuracy": 0.5,
      "nll": 1.2663163314336559
    },
    "open-jev:vizdoom-basic-v1": {
      "count": 153,
      "correct": 149,
      "accuracy": 0.9738562091503268,
      "nll": 0.03614709415145343
    },
    "open-jev:workflow-controls-v1/agent_trace_observability": {
      "count": 71,
      "correct": 71,
      "accuracy": 1.0,
      "nll": 0.0011360287241133358
    },
    "open-jev:workflow-controls-v1/customer_service": {
      "count": 137,
      "correct": 136,
      "accuracy": 0.9927007299270073,
      "nll": 0.030294906363817344
    },
    "open-jev:workflow-controls-v1/invoice_processing": {
      "count": 112,
      "correct": 110,
      "accuracy": 0.9821428571428571,
      "nll": 0.10505136013108395
    },
    "open-jev:workflow-controls-v1/security_incidents": {
      "count": 121,
      "correct": 121,
      "accuracy": 1.0,
      "nll": 0.009351040906769742
    },
    "replay:agnews": {
      "count": 29,
      "correct": 23,
      "accuracy": 0.7931034482758621,
      "nll": 0.5144041118900763
    },
    "replay:banking77": {
      "count": 52,
      "correct": 50,
      "accuracy": 0.9615384615384616,
      "nll": 0.1474100871587149
    },
    "replay:emotion": {
      "count": 15,
      "correct": 12,
      "accuracy": 0.8,
      "nll": 0.5748594660688843
    },
    "replay:gsm8k": {
      "count": 200,
      "correct": 196,
      "accuracy": 0.98,
      "nll": 0.047936887727627886
    },
    "replay:hellaswag": {
      "count": 119,
      "correct": 46,
      "accuracy": 0.3865546218487395,
      "nll": 3.136717304436293
    },
    "replay:mmlu": {
      "count": 182,
      "correct": 93,
      "accuracy": 0.510989010989011,
      "nll": 2.17472550274021
    },
    "replay:winogrande": {
      "count": 200,
      "correct": 107,
      "accuracy": 0.535,
      "nll": 1.151156462092322
    },
    "typed": {
      "count": 500,
      "correct": 365,
      "accuracy": 0.73,
      "nll": 2.027869497306916
    },
    "typed:choice": {
      "count": 155,
      "correct": 108,
      "accuracy": 0.6967741935483871,
      "nll": 2.5421524021866317
    },
    "typed:noul": {
      "count": 142,
      "correct": 115,
      "accuracy": 0.8098591549295775,
      "nll": 1.5555020525764138
    },
    "typed:score": {
      "count": 203,
      "correct": 142,
      "accuracy": 0.6995073891625616,
      "nll": 1.9656149499935025
    },
    "banking77": {
      "count": 100,
      "correct": 83,
      "accuracy": 0.83,
      "recall_at_16": 0.99,
      "unsupported": 0
    }
  },
  "quality_gate": false
}