{ "baseline_test": { "count": 512, "hard_count": 470, "accuracy": 0.6468085106382979, "expected_accuracy": 0.615909443204365, "nll": 0.8817175317461541, "brier": 0.441505162267834, "expected_brier": 0.49279716995632633, "multiclass_ece": 0.060929629228140346, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 378, "coverage": 0.73828125, "accuracy": 0.7235772357723578, "expected_accuracy": 0.7178130511463844, "expected_risk": 0.2821869488536156 }, { "threshold": 0.7, "selected": 229, "coverage": 0.447265625, "accuracy": 0.7991071428571429, "expected_accuracy": 0.791848617176128, "expected_risk": 0.20815138282387202 }, { "threshold": 0.8, "selected": 129, "coverage": 0.251953125, "accuracy": 0.8984375, "expected_accuracy": 0.8940568475452197, "expected_risk": 0.10594315245478025 }, { "threshold": 0.9, "selected": 82, "coverage": 0.16015625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 59, "coverage": 0.115234375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 0.048828125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "mean_latency_seconds": 0.08172164261850412, "by_kind": { "choice": { "count": 197, "hard_count": 158, "accuracy": 0.4873417721518987, "expected_accuracy": 0.440840786399162, "nll": 1.2488974501919257, "brier": 0.5142610511870647, "expected_brier": 0.6399539848749219, "multiclass_ece": 0.17116037995721942, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 104, "coverage": 0.5279187817258884, "accuracy": 0.6632653061224489, "expected_accuracy": 0.6522435897435899, "expected_risk": 0.34775641025641013 }, { "threshold": 0.7, "selected": 65, "coverage": 0.3299492385786802, "accuracy": 0.7580645161290323, "expected_accuracy": 0.7435897435897435, "expected_risk": 0.2564102564102565 }, { "threshold": 0.8, "selected": 33, "coverage": 0.16751269035532995, "accuracy": 0.90625, "expected_accuracy": 0.888888888888889, "expected_risk": 0.11111111111111105 }, { "threshold": 0.9, "selected": 20, "coverage": 0.10152284263959391, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 16, "coverage": 0.08121827411167512, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 0.01015228426395939, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "noul": { "count": 219, "hard_count": 219, "accuracy": 0.7899543378995434, "expected_accuracy": 0.7899543378995434, "nll": 0.4765508481066932, "brier": 0.3232662622711765, "expected_brier": 0.3232662622711765, "multiclass_ece": 0.08988485189392803, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 219, "coverage": 1.0, "accuracy": 0.7899543378995434, "expected_accuracy": 0.7899543378995434, "expected_risk": 0.2100456621004566 }, { "threshold": 0.7, "selected": 141, "coverage": 0.6438356164383562, "accuracy": 0.8226950354609929, "expected_accuracy": 0.8226950354609929, "expected_risk": 0.17730496453900713 }, { "threshold": 0.8, "selected": 84, "coverage": 0.3835616438356164, "accuracy": 0.9047619047619048, "expected_accuracy": 0.9047619047619048, "expected_risk": 0.09523809523809523 }, { "threshold": 0.9, "selected": 57, "coverage": 0.2602739726027397, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 42, "coverage": 0.1917808219178082, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 23, "coverage": 0.1050228310502283, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "score": { "count": 96, "hard_count": 93, "accuracy": 0.5806451612903226, "expected_accuracy": 0.578125, "nll": 1.0525202378214116, "brier": 0.5619365058322072, "expected_brier": 0.5775615058322073, "multiclass_ece": 0.1927334020311011, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 55, "coverage": 0.5729166666666666, "accuracy": 0.5576923076923077, "expected_accuracy": 0.5545454545454546, "expected_risk": 0.44545454545454544 }, { "threshold": 0.7, "selected": 23, "coverage": 0.23958333333333334, "accuracy": 0.7619047619047619, "expected_accuracy": 0.7391304347826086, "expected_risk": 0.26086956521739135 }, { "threshold": 0.8, "selected": 12, "coverage": 0.125, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 5, "coverage": 0.052083333333333336, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.010416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "ordinal_mae": 0.7143754983162888 } }, "by_source": { "customer-control-v1": { "count": 72, "hard_count": 68, "accuracy": 0.8529411764705882, "expected_accuracy": 0.8333333333333334, "nll": 0.33323271456468384, "brier": 0.17967632263664385, "expected_brier": 0.2074541004144216, "multiclass_ece": 0.08249298308036307, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 71, "coverage": 0.9861111111111112, "accuracy": 0.8507462686567164, "expected_accuracy": 0.8309859154929577, "expected_risk": 0.16901408450704225 }, { "threshold": 0.7, "selected": 62, "coverage": 0.8611111111111112, "accuracy": 0.9152542372881356, "expected_accuracy": 0.8951612903225806, "expected_risk": 0.10483870967741937 }, { "threshold": 0.8, "selected": 52, "coverage": 0.7222222222222222, "accuracy": 0.9615384615384616, "expected_accuracy": 0.9615384615384616, "expected_risk": 0.038461538461538436 }, { "threshold": 0.9, "selected": 42, "coverage": 0.5833333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 39, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 74, "hard_count": 74, "accuracy": 0.8243243243243243, "expected_accuracy": 0.8243243243243243, "nll": 0.6160462615772585, "brier": 0.3586155608549776, "expected_brier": 0.3586155608549776, "multiclass_ece": 0.23631098070437034, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 62, "coverage": 0.8378378378378378, "accuracy": 0.7903225806451613, "expected_accuracy": 0.7903225806451613, "expected_risk": 0.20967741935483875 }, { "threshold": 0.7, "selected": 38, "coverage": 0.5135135135135135, "accuracy": 0.8421052631578947, "expected_accuracy": 0.8421052631578947, "expected_risk": 0.1578947368421053 }, { "threshold": 0.8, "selected": 21, "coverage": 0.28378378378378377, "accuracy": 0.8095238095238095, "expected_accuracy": 0.8095238095238095, "expected_risk": 0.19047619047619047 }, { "threshold": 0.9, "selected": 12, "coverage": 0.16216216216216217, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 0.08108108108108109, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1": { "count": 72, "hard_count": 72, "accuracy": 0.6111111111111112, "expected_accuracy": 0.6111111111111112, "nll": 0.8780569298313221, "brier": 0.5137561460359871, "expected_brier": 0.5137561460359868, "multiclass_ece": 0.14355872021062377, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 49, "coverage": 0.6805555555555556, "accuracy": 0.6938775510204082, "expected_accuracy": 0.6938775510204082, "expected_risk": 0.30612244897959184 }, { "threshold": 0.7, "selected": 26, "coverage": 0.3611111111111111, "accuracy": 0.6923076923076923, "expected_accuracy": 0.6923076923076923, "expected_risk": 0.3076923076923077 }, { "threshold": 0.8, "selected": 11, "coverage": 0.1527777777777778, "accuracy": 0.7272727272727273, "expected_accuracy": 0.7272727272727273, "expected_risk": 0.2727272727272727 }, { "threshold": 0.9, "selected": 7, "coverage": 0.09722222222222222, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1": { "count": 48, "hard_count": 43, "accuracy": 0.5581395348837209, "expected_accuracy": 0.5243055555555556, "nll": 0.8808281205783343, "brier": 0.4890597244881951, "expected_brier": 0.548087502265973, "multiclass_ece": 0.1101612127905519, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 0.5208333333333334, "accuracy": 0.76, "expected_accuracy": 0.76, "expected_risk": 0.24 }, { "threshold": 0.7, "selected": 24, "coverage": 0.5, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tic_tac_toe-v1": { "count": 24, "hard_count": 7, "accuracy": 0.42857142857142855, "expected_accuracy": 0.3531746031746032, "nll": 1.2553168489114566, "brier": 0.2765893588184592, "expected_brier": 0.6942480889771891, "multiclass_ece": 0.09437864627388504, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1": { "count": 24, "hard_count": 24, "accuracy": 0.041666666666666664, "expected_accuracy": 0.041666666666666664, "nll": 1.7829095333554825, "brier": 0.8865678023062865, "expected_brier": 0.8865678023062865, "multiclass_ece": 0.30010726031768137, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1": { "count": 4, "hard_count": 4, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7307347607476473, "brier": 0.4520387085722736, "expected_brier": 0.4520387085722736, "multiclass_ece": 0.3520615203273855, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1": { "count": 73, "hard_count": 73, "accuracy": 0.3561643835616438, "expected_accuracy": 0.3561643835616438, "nll": 1.3425165420127338, "brier": 0.8023158966716812, "expected_brier": 0.8023158966716812, "multiclass_ece": 0.27768183802177016, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 61, "coverage": 0.8356164383561644, "accuracy": 0.3442622950819672, "expected_accuracy": 0.3442622950819672, "expected_risk": 0.6557377049180328 }, { "threshold": 0.7, "selected": 18, "coverage": 0.2465753424657534, "accuracy": 0.16666666666666666, "expected_accuracy": 0.16666666666666666, "expected_risk": 0.8333333333333334 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.1959490740740741, "nll": 2.5412964840563768, "brier": 0.47697892684702387, "expected_brier": 0.952160805153902, "multiclass_ece": 0.2919900261050552, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 0.5, "expected_accuracy": 0.4583333333333333, "expected_risk": 0.5416666666666667 }, { "threshold": 0.7, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.41666666666666663, "expected_risk": 0.5833333333333334 }, { "threshold": 0.8, "selected": 1, "coverage": 0.041666666666666664, "accuracy": null, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.5416666666666666, "optimal_set_probability_mass": 0.43454189807943405 }, "workflow-controls-v1/agent_trace_observability": { "count": 24, "hard_count": 24, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.43867556001813907, "brier": 0.2919941474824955, "expected_brier": 0.29199414748249536, "multiclass_ece": 0.1346704838183649, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 0.8421052631578947, "expected_accuracy": 0.8421052631578947, "expected_risk": 0.1578947368421053 }, { "threshold": 0.8, "selected": 15, "coverage": 0.625, "accuracy": 0.8666666666666667, "expected_accuracy": 0.8666666666666667, "expected_risk": 0.1333333333333333 }, { "threshold": 0.9, "selected": 8, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service": { "count": 24, "hard_count": 24, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "nll": 0.44603624362346334, "brier": 0.2886856133300345, "expected_brier": 0.28868561333003445, "multiclass_ece": 0.1476082864128875, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "expected_risk": 0.20833333333333337 }, { "threshold": 0.7, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.8, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8461538461538461, "expected_risk": 0.15384615384615385 }, { "threshold": 0.9, "selected": 7, "coverage": 0.2916666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.4247406840388844, "brier": 0.2550697606761862, "expected_brier": 0.2550697606761861, "multiclass_ece": 0.29220395912883795, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 9, "coverage": 0.375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents": { "count": 25, "hard_count": 25, "accuracy": 0.84, "expected_accuracy": 0.84, "nll": 0.40389174428118824, "brier": 0.25250468558506645, "expected_brier": 0.2525046855850665, "multiclass_ece": 0.13450269089458863, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 0.84, "expected_accuracy": 0.84, "expected_risk": 0.16000000000000003 }, { "threshold": 0.7, "selected": 13, "coverage": 0.52, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 0.24, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.08, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 0.375, "expected_accuracy": 0.375, "nll": 1.5154957829505873, "brier": 0.9581639412101648, "expected_brier": 0.958163941210165, "multiclass_ece": 0.5978255635844472, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.875, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.7, "selected": 5, "coverage": 0.625, "accuracy": 0.2, "expected_accuracy": 0.2, "expected_risk": 0.8 }, { "threshold": 0.8, "selected": 3, "coverage": 0.375, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 1, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/category": { "count": 24, "hard_count": 23, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "nll": 0.13146410030162184, "brier": 0.033835821313346295, "expected_brier": 0.05466915464667962, "multiclass_ece": 0.07354489809755994, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "expected_risk": 0.02083333333333337 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 0.9782608695652174, "expected_risk": 0.021739130434782594 }, { "threshold": 0.8, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 7, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.12867993345138773, "brier": 0.03344012324824742, "expected_brier": 0.033440123248247425, "multiclass_ece": 0.1187657793452654, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 0.8571428571428571, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.42857142857142855, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.14285714285714285, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/frustration": { "count": 9, "hard_count": 6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3888888888888889, "nll": 0.7776272583293182, "brier": 0.4076220810767673, "expected_brier": 0.574288747743434, "multiclass_ece": 0.3124813193980322, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 1.0, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3888888888888889, "expected_risk": 0.6111111111111112 }, { "threshold": 0.7, "selected": 4, "coverage": 0.4444444444444444, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 1, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/has_reproducible_steps": { "count": 14, "hard_count": 14, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "nll": 0.05447111036787444, "brier": 0.039684919522046794, "expected_brier": 0.039684919522046815, "multiclass_ece": 0.038634232472595176, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 14, "coverage": 1.0, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "expected_risk": 0.0714285714285714 }, { "threshold": 0.7, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.005165037353980109, "brier": 0.0001055522899432388, "expected_brier": 0.00010555228994320043, "multiclass_ece": 0.0051383770001155685, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 9, "coverage": 0.9, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.25061169361067015, "brier": 0.07707358531359691, "expected_brier": 0.07707358531359687, "multiclass_ece": 0.22167545871970484, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.290960929385389, "brier": 0.9115423551722488, "expected_brier": 0.9115423551722488, "multiclass_ece": 0.6106084337804977, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3758215703154072, "brier": 0.11003950313618463, "expected_brier": 0.11003950313618471, "multiclass_ece": 0.3132751457895919, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1067467091068113, "brier": 0.5662601356755865, "expected_brier": 0.5662601356755865, "multiclass_ece": 0.6693671418553664, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6573043642884194, "brier": 0.4641729525158496, "expected_brier": 0.46417295251584956, "multiclass_ece": 0.481753543067329, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7637593823939963, "brier": 1.3731582545974015, "expected_brier": 1.3731582545974015, "multiclass_ece": 0.8286007043798, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9486113240167164, "brier": 0.4910052789864623, "expected_brier": 0.4910052789864623, "multiclass_ece": 0.6127215455020518, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.028215473463322507, "brier": 0.0015480310515196916, "expected_brier": 0.0015480310515196827, "multiclass_ece": 0.02782113451604451, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.4311176670488925, "brier": 1.0151875912238948, "expected_brier": 1.0151875912238948, "multiclass_ece": 0.6512199138997508, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.24088219617816786, "brier": 0.07122856906572506, "expected_brier": 0.07122856906572506, "multiclass_ece": 0.21406579301164397, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07163475944340203, "brier": 0.009557685531945985, "expected_brier": 0.009557685531945914, "multiclass_ece": 0.06912917449219969, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0916722610682434, "brier": 0.5723584956456659, "expected_brier": 0.5723584956456658, "multiclass_ece": 0.6643452782006345, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.22663139252875392, "brier": 0.04618732358692956, "expected_brier": 0.04618732358692945, "multiclass_ece": 0.20278541260946603, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0921189959161661, "brier": 0.5737165203359809, "expected_brier": 0.573716520335981, "multiclass_ece": 0.6643233327731036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07616195579801274, "brier": 0.010755717244793217, "expected_brier": 0.010755717244793184, "multiclass_ece": 0.0733338845445719, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y2": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19447096688535614, "brier": 0.054376989683661336, "expected_brier": 0.0543769896836613, "multiclass_ece": 0.16497119003016303, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.09509565847398439, "brier": 0.00990127514549919, "expected_brier": 0.009901275145499211, "multiclass_ece": 0.09071405059846682, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5001144314952748, "brier": 0.23237117114838718, "expected_brier": 0.23237117114838712, "multiclass_ece": 0.3935387425267107, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0691269229189364, "brier": 0.5534891688481363, "expected_brier": 0.5534891688481363, "multiclass_ece": 0.6566918789366191, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4045142554248069, "brier": 0.1368900796175709, "expected_brier": 0.13689007961757094, "multiclass_ece": 0.3326991300752765, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.27240777267452226, "brier": 0.08899423851235795, "expected_brier": 0.08899423851235788, "multiclass_ece": 0.23845633892839935, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.34879056335891045, "brier": 0.1734123456311483, "expected_brier": 0.17341234563114827, "multiclass_ece": 0.29445911909053546, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.03729321769884203, "brier": 0.0015410747254854325, "expected_brier": 0.0015410747254853696, "multiclass_ece": 0.03660639012702338, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y1": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3231745756917025, "brier": 0.11880343227055283, "expected_brier": 0.11880343227055279, "multiclass_ece": 0.24380020857690077, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0292318261968079, "brier": 0.5309228900531984, "expected_brier": 0.5309228900531984, "multiclass_ece": 0.6427186906759138, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y3_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2613302198322449, "brier": 0.06713786517445684, "expected_brier": 0.06713786517445675, "multiclass_ece": 0.2299734004279056, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6325781692267881, "brier": 0.33566482417193577, "expected_brier": 0.33566482417193577, "multiclass_ece": 0.46877954224780427, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y4_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.865233210851233, "brier": 1.4285229328543014, "expected_brier": 1.4285229328543014, "multiclass_ece": 0.845139909380187, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.015559082159477978, "brier": 0.0004767047525142958, "expected_brier": 0.0004767047525142676, "multiclass_ece": 0.015438664976517469, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y5_green": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 1.0350787047166359, "brier": 0.8003716092905087, "expected_brier": 0.8003716092905088, "multiclass_ece": 0.6147861600566995, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.18888147856982543, "brier": 0.059247406034326364, "expected_brier": 0.05924740603432643, "multiclass_ece": 0.17211537705028912, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4970591239534609, "brier": 0.26061331660758946, "expected_brier": 0.26061331660758946, "multiclass_ece": 0.39168298335905527, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6633131574320936, "brier": 0.47017502916064685, "expected_brier": 0.4701750291606468, "multiclass_ece": 0.4848582417370292, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y4": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2830191120023805, "brier": 0.07164516537518219, "expected_brier": 0.07164516537518217, "multiclass_ece": 0.2464946132746444, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y5_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5088332264630842, "brier": 1.2131589413426385, "expected_brier": 1.2131589413426385, "multiclass_ece": 0.7788321196967413, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4187206439226338, "brier": 0.18479680062195844, "expected_brier": 0.18479680062195847, "multiclass_ece": 0.342112045402178, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07870817257858351, "brier": 0.006581016106193623, "expected_brier": 0.00658101610619366, "multiclass_ece": 0.07569037601552997, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3879594449276739, "brier": 0.2068018512820064, "expected_brier": 0.20680185128200645, "multiclass_ece": 0.3215601431163433, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.10648074761089, "brier": 0.7340055582944687, "expected_brier": 0.7340055582944687, "multiclass_ece": 0.510854803637153, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4039733922578492, "brier": 0.14441646063092856, "expected_brier": 0.1444164606309286, "multiclass_ece": 0.3323381139921051, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.7962504499575063, "brier": 0.40341720086662797, "expected_brier": 0.4034172008666279, "multiclass_ece": 0.5489830919086633, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3456154920395685, "brier": 0.14176918611668743, "expected_brier": 0.14176918611668743, "multiclass_ece": 0.29221541640182, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8717973596726329, "brier": 0.4520777517244417, "expected_brier": 0.45207775172444165, "multiclass_ece": 0.5818007810651498, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.330368818809002, "brier": 0.9122471539230859, "expected_brier": 0.9122471539230859, "multiclass_ece": 0.5920214086168877, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2737743994823805, "brier": 0.07168886649901243, "expected_brier": 0.07168886649901252, "multiclass_ece": 0.23949637407914515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6606651046668591, "brier": 0.46752963046255797, "expected_brier": 0.467529630462558, "multiclass_ece": 0.4834923114500157, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4282462482368157, "brier": 0.2426941531139567, "expected_brier": 0.2426941531139568, "multiclass_ece": 0.3483490728521871, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.42199907551539395, "brier": 0.1851740966510652, "expected_brier": 0.18517409665106532, "multiclass_ece": 0.34426535439025296, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y6_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08793756417513526, "brier": 0.014173199797330598, "expected_brier": 0.014173199797330671, "multiclass_ece": 0.08418194520599587, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.829340862136269, "brier": 0.433892444166338, "expected_brier": 0.4338924441663381, "multiclass_ece": 0.5636632023390014, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2667785149151063, "brier": 0.0680737433219249, "expected_brier": 0.06807374332192495, "multiclass_ece": 0.23415732458771277, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9088779443473529, "brier": 0.7128749976842488, "expected_brier": 0.7128749976842488, "multiclass_ece": 0.5970238678998725, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.321082152379276, "brier": 0.9141924016910878, "expected_brier": 0.9141924016910878, "multiclass_ece": 0.5988708597435247, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y1_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1292269155296309, "brier": 0.029391226568710314, "expected_brier": 0.029391226568710405, "multiclass_ece": 0.12122546466957818, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08895125424329564, "brier": 0.014487366495094196, "expected_brier": 0.01448736649509419, "multiclass_ece": 0.08510983049887422, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.25118498326607547, "brier": 0.07073080011453821, "expected_brier": 0.07073080011453814, "multiclass_ece": 0.22212153624973485, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9693334580396843, "brier": 0.5251366446175078, "expected_brier": 0.5251366446175076, "multiclass_ece": 0.6206642029173866, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y4": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1946376383278648, "brier": 0.03742475249683832, "expected_brier": 0.0374247524968383, "multiclass_ece": 0.17558280105013574, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.13929559416433523, "brier": 0.019888018944366582, "expected_brier": 0.019888018944366648, "multiclass_ece": 0.13002916785386442, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2823687184795314, "brier": 0.10455442253629812, "expected_brier": 0.10455442253629821, "multiclass_ece": 0.2460043788463191, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1423616423312868, "brier": 0.7707382889509069, "expected_brier": 0.7707382889509069, "multiclass_ece": 0.534412002649892, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4197833990335743, "brier": 0.1801329123434367, "expected_brier": 0.18013291234343676, "multiclass_ece": 0.3428108477946742, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_green": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 2.044767669078705, "brier": 1.515853016945333, "expected_brier": 1.515853016945333, "multiclass_ece": 0.870589747511804, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y2_green": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 2.0986600445997086, "brier": 1.5395891374968458, "expected_brier": 1.5395891374968458, "multiclass_ece": 0.8773793756114985, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.32105176835164906, "brier": 0.13064503674386727, "expected_brier": 0.13064503674386718, "multiclass_ece": 0.2746143020037738, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3166184941516057, "brier": 0.6803546060400019, "expected_brier": 0.6803546060400019, "multiclass_ece": 0.7319598494350382, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.16964859523871106, "brier": 0.04082000338779008, "expected_brier": 0.04082000338779013, "multiclass_ece": 0.14703345065406742, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/decision": { "count": 24, "hard_count": 24, "accuracy": 0.5416666666666666, "expected_accuracy": 0.5416666666666666, "nll": 1.0577458553895966, "brier": 0.6353700925846094, "expected_brier": 0.6353700925846093, "multiclass_ece": 0.3025464417616207, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.5882352941176471, "expected_accuracy": 0.5882352941176471, "expected_risk": 0.4117647058823529 }, { "threshold": 0.7, "selected": 11, "coverage": 0.4583333333333333, "accuracy": 0.45454545454545453, "expected_accuracy": 0.45454545454545453, "expected_risk": 0.5454545454545454 }, { "threshold": 0.8, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.9, "selected": 2, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/ordinal": { "count": 24, "hard_count": 24, "accuracy": 0.5833333333333334, "expected_accuracy": 0.5833333333333334, "nll": 1.0303934767932423, "brier": 0.5257811830322919, "expected_brier": 0.5257811830322919, "multiclass_ece": 0.17836184515480924, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 0.3333333333333333, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.7, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/verification": { "count": 24, "hard_count": 24, "accuracy": 0.7083333333333334, "expected_accuracy": 0.7083333333333334, "nll": 0.5460314573111277, "brier": 0.3801171624910597, "expected_brier": 0.3801171624910597, "multiclass_ece": 0.11341563968644845, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.7083333333333334, "expected_accuracy": 0.7083333333333334, "expected_risk": 0.29166666666666663 }, { "threshold": 0.7, "selected": 11, "coverage": 0.4583333333333333, "accuracy": 0.8181818181818182, "expected_accuracy": 0.8181818181818182, "expected_risk": 0.18181818181818177 }, { "threshold": 0.8, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/choice": { "count": 24, "hard_count": 19, "accuracy": 0.3157894736842105, "expected_accuracy": 0.2986111111111111, "nll": 1.2100159754401705, "brier": 0.610597546828379, "expected_brier": 0.7286531023839342, "multiclass_ece": 0.12711028040371114, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/noul": { "count": 24, "hard_count": 24, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.5516402657164982, "brier": 0.3675219021480114, "expected_brier": 0.36752190214801145, "multiclass_ece": 0.09321214517739262, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tic_tac_toe-v1/choice": { "count": 24, "hard_count": 7, "accuracy": 0.42857142857142855, "expected_accuracy": 0.3531746031746032, "nll": 1.2553168489114566, "brier": 0.2765893588184592, "expected_brier": 0.6942480889771891, "multiclass_ece": 0.09437864627388504, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.041666666666666664, "expected_accuracy": 0.041666666666666664, "nll": 1.7829095333554825, "brier": 0.8865678023062865, "expected_brier": 0.8865678023062865, "multiclass_ece": 0.30010726031768137, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1/choice": { "count": 4, "hard_count": 4, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7307347607476473, "brier": 0.4520387085722736, "expected_brier": 0.4520387085722736, "multiclass_ece": 0.3520615203273855, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.08333333333333333, "expected_accuracy": 0.08333333333333333, "nll": 1.8548766439101059, "brier": 1.131016697661235, "expected_brier": 1.1310166976612352, "multiclass_ece": 0.5685037869066162, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.08333333333333333, "expected_accuracy": 0.08333333333333333, "expected_risk": 0.9166666666666666 }, { "threshold": 0.7, "selected": 9, "coverage": 0.375, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 25, "hard_count": 25, "accuracy": 0.56, "expected_accuracy": 0.56, "nll": 0.7808257803513436, "brier": 0.5749328594640268, "expected_brier": 0.5749328594640268, "multiclass_ece": 0.2164648946302959, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 0.56, "expected_accuracy": 0.56, "expected_risk": 0.43999999999999995 }, { "threshold": 0.7, "selected": 8, "coverage": 0.32, "accuracy": 0.25, "expected_accuracy": 0.25, "expected_risk": 0.75 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/score": { "count": 24, "hard_count": 24, "accuracy": 0.4166666666666667, "expected_accuracy": 0.4166666666666667, "nll": 1.4152509835126432, "brier": 0.7104724261067669, "expected_brier": 0.7104724261067666, "multiclass_ece": 0.1896669689663131, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 0.5, "accuracy": 0.4166666666666667, "expected_accuracy": 0.4166666666666667, "expected_risk": 0.5833333333333333 }, { "threshold": 0.7, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1/choice": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.1959490740740741, "nll": 2.5412964840563768, "brier": 0.47697892684702387, "expected_brier": 0.952160805153902, "multiclass_ece": 0.2919900261050552, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 0.5, "expected_accuracy": 0.4583333333333333, "expected_risk": 0.5416666666666667 }, { "threshold": 0.7, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.41666666666666663, "expected_risk": 0.5833333333333334 }, { "threshold": 0.8, "selected": 1, "coverage": 0.041666666666666664, "accuracy": null, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.06924441901940935, "brier": 0.019831004335831858, "expected_brier": 0.019831004335831848, "multiclass_ece": 0.06363803750705706, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.10331980759397992, "brier": 0.024635972228959534, "expected_brier": 0.024635972228959524, "multiclass_ece": 0.09649700098022485, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.17503803702120002, "brier": 0.09269725211451219, "expected_brier": 0.09269725211451214, "multiclass_ece": 0.1417895203368496, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.6, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.4, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.506675338369359, "brier": 0.32221747477258456, "expected_brier": 0.32221747477258456, "multiclass_ece": 0.3907830056400119, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08173881676989282, "brier": 0.012320540492919991, "expected_brier": 0.012320540492920085, "multiclass_ece": 0.07848738909187891, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 0.4, "expected_accuracy": 0.4, "nll": 1.1165927217186744, "brier": 0.8308819447505298, "expected_brier": 0.8308819447505298, "multiclass_ece": 0.4221753256685233, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.8, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.4813089322161565, "brier": 0.31845739532568257, "expected_brier": 0.3184573953256826, "multiclass_ece": 0.34738593863982153, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.05222616044691446, "brier": 0.005436009898980887, "expected_brier": 0.005436009898980965, "multiclass_ece": 0.05081420797321734, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3713785984147519, "brier": 0.2273835402628462, "expected_brier": 0.22738354026284613, "multiclass_ece": 0.2918469407272709, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 3, "hard_count": 3, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "nll": 0.6342122263674833, "brier": 0.46035903374880477, "expected_brier": 0.46035903374880477, "multiclass_ece": 0.44188089532548264, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.048110371162522654, "brier": 0.00448087585370835, "expected_brier": 0.004480875853708414, "multiclass_ece": 0.04695261822060356, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5552974957095779, "brier": 0.3631198590361685, "expected_brier": 0.3631198590361686, "multiclass_ece": 0.4260984974370178, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.14036871513091584, "brier": 0.03439812964338595, "expected_brier": 0.034398129643385955, "multiclass_ece": 0.13092976146035842, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3231783695754461, "brier": 0.15410360771267828, "expected_brier": 0.15410360771267823, "multiclass_ece": 0.2752593128725131, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 5, "hard_count": 5, "accuracy": 0.4, "expected_accuracy": 0.4, "nll": 1.0340393637661816, "brier": 0.7516367343419497, "expected_brier": 0.7516367343419497, "multiclass_ece": 0.523173600509939, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.7, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.8, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 1, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19207911128761262, "brier": 0.06108101073213613, "expected_brier": 0.06108101073213623, "multiclass_ece": 0.17475842001479658, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2097875711959973, "brier": 0.07162623811647786, "expected_brier": 0.07162623811647784, "multiclass_ece": 0.18924354429739187, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.39641847219956006, "brier": 0.2143225873762089, "expected_brier": 0.2143225873762089, "multiclass_ece": 0.32719953576600913, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6523502905122536, "brier": 0.459260054127935, "expected_brier": 0.4592600541279351, "multiclass_ece": 0.47901470193893114, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.42448276609786173, "brier": 0.23928251743916004, "expected_brier": 0.23928251743915996, "multiclass_ece": 0.34589197550619766, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.23626104550177165, "brier": 0.10596583520449858, "expected_brier": 0.10596583520449854, "multiclass_ece": 0.20357623611332823, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3245393404673471, "brier": 0.15392248477089354, "expected_brier": 0.15392248477089343, "multiclass_ece": 0.27696613317103075, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.6072850750173204, "brier": 0.4152110721516099, "expected_brier": 0.4152110721516098, "multiclass_ece": 0.21397334003328225, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.52378637764638, "brier": 0.3325517966616923, "expected_brier": 0.3325517966616923, "multiclass_ece": 0.40763100179712747, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.16888852235122379, "brier": 0.051831783419505936, "expected_brier": 0.051831783419505895, "multiclass_ece": 0.15411692148710543, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.3852726003244243, "brier": 0.25630643842997664, "expected_brier": 0.25630643842997664, "multiclass_ece": 0.281634465836278, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.396162361691784, "brier": 0.21399217697263284, "expected_brier": 0.21399217697263273, "multiclass_ece": 0.3271025656981559, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1753151166206895, "brier": 0.06921618080664277, "expected_brier": 0.06921618080664282, "multiclass_ece": 0.1544191829813078, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4267856220863001, "brier": 0.24202728360338416, "expected_brier": 0.2420272836033842, "multiclass_ece": 0.3468580837869641, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.7262262440576691, "brier": 0.5281165012334273, "expected_brier": 0.5281165012334273, "multiclass_ece": 0.49714542908172393, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3645911020444213, "brier": 0.1941280995064558, "expected_brier": 0.19412809950645582, "multiclass_ece": 0.30027631926313214, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/KILL PROCESS": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.7771179737322875, "brier": 0.5837853878521752, "expected_brier": 0.583785387852175, "multiclass_ece": 0.5402709449212382, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2301313965070047, "brier": 0.08805377997817938, "expected_brier": 0.08805377997817943, "multiclass_ece": 0.20407377578145758, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.23256380764653556, "brier": 0.08611318041386751, "expected_brier": 0.08611318041386751, "multiclass_ece": 0.2075008197741246, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2689532353320694, "brier": 0.153118044468375, "expected_brier": 0.15311804446837507, "multiclass_ece": 0.2166904933512851, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5004547537421054, "brier": 0.3159653096114085, "expected_brier": 0.31596530961140856, "multiclass_ece": 0.38702288067568047, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.05988167751789632, "brier": 0.006756850220352334, "expected_brier": 0.006756850220352328, "multiclass_ece": 0.05812401518161292, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9401360210674187, "brier": 0.7427983795017221, "expected_brier": 0.7427983795017221, "multiclass_ece": 0.6094252946431261, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } } }, "baseline_ood": { "count": 512, "hard_count": 484, "accuracy": 0.6673553719008265, "expected_accuracy": 0.637868923611111, "nll": 0.8627883604907501, "brier": 0.42613352720162223, "expected_brier": 0.4633459837988445, "multiclass_ece": 0.03578670685937321, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 359, "coverage": 0.701171875, "accuracy": 0.768361581920904, "expected_accuracy": 0.7623026926648095, "expected_risk": 0.23769730733519046 }, { "threshold": 0.7, "selected": 205, "coverage": 0.400390625, "accuracy": 0.8669950738916257, "expected_accuracy": 0.8617886178861788, "expected_risk": 0.13821138211382122 }, { "threshold": 0.8, "selected": 134, "coverage": 0.26171875, "accuracy": 0.9398496240601504, "expected_accuracy": 0.9353233830845772, "expected_risk": 0.06467661691542281 }, { "threshold": 0.9, "selected": 95, "coverage": 0.185546875, "accuracy": 0.9787234042553191, "expected_accuracy": 0.9719298245614036, "expected_risk": 0.028070175438596356 }, { "threshold": 0.95, "selected": 67, "coverage": 0.130859375, "accuracy": 0.9701492537313433, "expected_accuracy": 0.9701492537313433, "expected_risk": 0.02985074626865669 }, { "threshold": 0.99, "selected": 26, "coverage": 0.05078125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "mean_latency_seconds": 0.06099348470888799, "by_kind": { "choice": { "count": 162, "hard_count": 136, "accuracy": 0.5147058823529411, "expected_accuracy": 0.4501371742112483, "nll": 1.3488087343285642, "brier": 0.5435014439324403, "expected_brier": 0.6528807306265416, "multiclass_ece": 0.13969090241586624, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 67, "coverage": 0.41358024691358025, "accuracy": 0.625, "expected_accuracy": 0.6119402985074627, "expected_risk": 0.3880597014925373 }, { "threshold": 0.7, "selected": 32, "coverage": 0.19753086419753085, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.8, "selected": 23, "coverage": 0.1419753086419753, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 19, "coverage": 0.11728395061728394, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 0.09259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 0.012345679012345678, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "noul": { "count": 243, "hard_count": 243, "accuracy": 0.8106995884773662, "expected_accuracy": 0.8106995884773662, "nll": 0.4402324317187243, "brier": 0.28658961352911105, "expected_brier": 0.28658961352911105, "multiclass_ece": 0.060235461644253384, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 243, "coverage": 1.0, "accuracy": 0.8106995884773662, "expected_accuracy": 0.8106995884773662, "expected_risk": 0.18930041152263377 }, { "threshold": 0.7, "selected": 146, "coverage": 0.6008230452674898, "accuracy": 0.8835616438356164, "expected_accuracy": 0.8835616438356164, "expected_risk": 0.11643835616438358 }, { "threshold": 0.8, "selected": 91, "coverage": 0.37448559670781895, "accuracy": 0.9560439560439561, "expected_accuracy": 0.9560439560439561, "expected_risk": 0.04395604395604391 }, { "threshold": 0.9, "selected": 64, "coverage": 0.26337448559670784, "accuracy": 0.96875, "expected_accuracy": 0.96875, "expected_risk": 0.03125 }, { "threshold": 0.95, "selected": 47, "coverage": 0.1934156378600823, "accuracy": 0.9574468085106383, "expected_accuracy": 0.9574468085106383, "expected_risk": 0.04255319148936165 }, { "threshold": 0.99, "selected": 20, "coverage": 0.0823045267489712, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "score": { "count": 107, "hard_count": 105, "accuracy": 0.5333333333333333, "expected_accuracy": 0.5295950155763239, "nll": 1.0865807916110943, "brier": 0.5653444478747786, "expected_brier": 0.5778055070648097, "multiclass_ece": 0.13514481498846603, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 49, "coverage": 0.45794392523364486, "accuracy": 0.7446808510638298, "expected_accuracy": 0.7278911564625852, "expected_risk": 0.2721088435374148 }, { "threshold": 0.7, "selected": 27, "coverage": 0.2523364485981308, "accuracy": 0.76, "expected_accuracy": 0.7283950617283951, "expected_risk": 0.2716049382716049 }, { "threshold": 0.8, "selected": 20, "coverage": 0.18691588785046728, "accuracy": 0.7894736842105263, "expected_accuracy": 0.7666666666666666, "expected_risk": 0.2333333333333334 }, { "threshold": 0.9, "selected": 12, "coverage": 0.11214953271028037, "accuracy": 1.0, "expected_accuracy": 0.9444444444444443, "expected_risk": 0.05555555555555569 }, { "threshold": 0.95, "selected": 5, "coverage": 0.04672897196261682, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 0.037383177570093455, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "ordinal_mae": 0.6960043527895335 } }, "by_source": { "customer-control-v1": { "count": 81, "hard_count": 77, "accuracy": 0.8311688311688312, "expected_accuracy": 0.8106995884773661, "nll": 0.3822530553263359, "brier": 0.19691152151155858, "expected_brier": 0.22571810587369848, "multiclass_ece": 0.08781684993632687, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.8421052631578947, "expected_accuracy": 0.8208333333333332, "expected_risk": 0.1791666666666668 }, { "threshold": 0.7, "selected": 69, "coverage": 0.8518518518518519, "accuracy": 0.9104477611940298, "expected_accuracy": 0.893719806763285, "expected_risk": 0.106280193236715 }, { "threshold": 0.8, "selected": 62, "coverage": 0.7654320987654321, "accuracy": 0.9344262295081968, "expected_accuracy": 0.9247311827956989, "expected_risk": 0.07526881720430112 }, { "threshold": 0.9, "selected": 51, "coverage": 0.6296296296296297, "accuracy": 1.0, "expected_accuracy": 0.9869281045751633, "expected_risk": 0.013071895424836666 }, { "threshold": 0.95, "selected": 42, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 26, "coverage": 0.32098765432098764, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 81, "hard_count": 81, "accuracy": 0.5679012345679012, "expected_accuracy": 0.5679012345679012, "nll": 1.0554203678729608, "brier": 0.5660242295394857, "expected_brier": 0.566024229539486, "multiclass_ece": 0.12919611404462586, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 30, "coverage": 0.37037037037037035, "accuracy": 0.7, "expected_accuracy": 0.7, "expected_risk": 0.30000000000000004 }, { "threshold": 0.7, "selected": 13, "coverage": 0.16049382716049382, "accuracy": 0.7692307692307693, "expected_accuracy": 0.7692307692307693, "expected_risk": 0.23076923076923073 }, { "threshold": 0.8, "selected": 6, "coverage": 0.07407407407407407, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1": { "count": 80, "hard_count": 80, "accuracy": 0.5625, "expected_accuracy": 0.5625, "nll": 1.0338404676107387, "brier": 0.5792595979337627, "expected_brier": 0.5792595979337626, "multiclass_ece": 0.16002899319251784, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 50, "coverage": 0.625, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 19, "coverage": 0.2375, "accuracy": 0.7368421052631579, "expected_accuracy": 0.7368421052631579, "expected_risk": 0.26315789473684215 }, { "threshold": 0.8, "selected": 14, "coverage": 0.175, "accuracy": 0.8571428571428571, "expected_accuracy": 0.8571428571428571, "expected_risk": 0.1428571428571429 }, { "threshold": 0.9, "selected": 12, "coverage": 0.15, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.95, "selected": 5, "coverage": 0.0625, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1": { "count": 54, "hard_count": 48, "accuracy": 0.5625, "expected_accuracy": 0.5185185185185185, "nll": 0.8597370239012662, "brier": 0.46443902629760125, "expected_brier": 0.5199945818531568, "multiclass_ece": 0.1546633247551369, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 0.5, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.7, "selected": 27, "coverage": 0.5, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.8, "selected": 6, "coverage": 0.1111111111111111, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1": { "count": 81, "hard_count": 81, "accuracy": 0.49382716049382713, "expected_accuracy": 0.49382716049382713, "nll": 1.0140032232991911, "brier": 0.6113740032527482, "expected_brier": 0.6113740032527482, "multiclass_ece": 0.13574135926315373, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 63, "coverage": 0.7777777777777778, "accuracy": 0.5555555555555556, "expected_accuracy": 0.5555555555555556, "expected_risk": 0.4444444444444444 }, { "threshold": 0.7, "selected": 12, "coverage": 0.14814814814814814, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1": { "count": 27, "hard_count": 9, "accuracy": 0.2222222222222222, "expected_accuracy": 0.10823045267489712, "nll": 2.8591546846934452, "brier": 0.49474953777578484, "expected_brier": 1.002877109792246, "multiclass_ece": 0.19076155939447956, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.037037037037037035, "accuracy": null, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.3333333333333333, "optimal_set_probability_mass": 0.3222542921873315 }, "workflow-controls-v1/agent_trace_observability": { "count": 27, "hard_count": 27, "accuracy": 0.7777777777777778, "expected_accuracy": 0.7777777777777778, "nll": 0.3791801211044954, "brier": 0.25071706728040266, "expected_brier": 0.25071706728040266, "multiclass_ece": 0.1663607698839319, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.7777777777777778, "expected_accuracy": 0.7777777777777778, "expected_risk": 0.2222222222222222 }, { "threshold": 0.7, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 15, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 8, "coverage": 0.2962962962962963, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 0.18518518518518517, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.3335077224347973, "brier": 0.18787280080402338, "expected_brier": 0.18787280080402338, "multiclass_ece": 0.20962026312310614, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 14, "coverage": 0.5185185185185185, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "expected_risk": 0.0714285714285714 }, { "threshold": 0.8, "selected": 9, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 7, "coverage": 0.25925925925925924, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.3770202653642815, "brier": 0.22070082190711432, "expected_brier": 0.22070082190711432, "multiclass_ece": 0.2507417242662642, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 8, "coverage": 0.2962962962962963, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 0.14814814814814814, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.27442592882369776, "brier": 0.15858201570867314, "expected_brier": 0.15858201570867314, "multiclass_ece": 0.1692598106928884, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 17, "coverage": 0.6296296296296297, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 14, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.25, "nll": 1.5660310662037447, "brier": 1.0161625656776057, "expected_brier": 1.0161625656776057, "multiclass_ece": 0.5989770139456254, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.875, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.7, "selected": 6, "coverage": 0.75, "accuracy": 0.16666666666666666, "expected_accuracy": 0.16666666666666666, "expected_risk": 0.8333333333333334 }, { "threshold": 0.8, "selected": 4, "coverage": 0.5, "accuracy": 0.25, "expected_accuracy": 0.25, "expected_risk": 0.75 }, { "threshold": 0.9, "selected": 1, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/category": { "count": 27, "hard_count": 25, "accuracy": 0.96, "expected_accuracy": 0.9259259259259259, "nll": 0.18771819173154947, "brier": 0.057470149786313644, "expected_brier": 0.09450718682335069, "multiclass_ece": 0.04463202237097918, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.96, "expected_accuracy": 0.9259259259259259, "expected_risk": 0.07407407407407407 }, { "threshold": 0.7, "selected": 23, "coverage": 0.8518518518518519, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 22, "coverage": 0.8148148148148148, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 9, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "nll": 0.6067044727825608, "brier": 0.09860061154244454, "expected_brier": 0.24674875969059273, "multiclass_ece": 0.12556770407251705, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "expected_risk": 0.14814814814814825 }, { "threshold": 0.7, "selected": 9, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "expected_risk": 0.14814814814814825 }, { "threshold": 0.8, "selected": 8, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 0.9166666666666666, "expected_risk": 0.08333333333333337 }, { "threshold": 0.9, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 0.9047619047619048, "expected_risk": 0.09523809523809523 }, { "threshold": 0.95, "selected": 5, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 0.4444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/frustration": { "count": 10, "hard_count": 10, "accuracy": 0.7, "expected_accuracy": 0.7, "nll": 0.4614085355095604, "brier": 0.3093678852148495, "expected_brier": 0.3093678852148494, "multiclass_ece": 0.27012850764119767, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 0.7, "expected_accuracy": 0.7, "expected_risk": 0.30000000000000004 }, { "threshold": 0.7, "selected": 8, "coverage": 0.8, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.8, "selected": 6, "coverage": 0.6, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 3, "coverage": 0.3, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/has_reproducible_steps": { "count": 15, "hard_count": 15, "accuracy": 0.8, "expected_accuracy": 0.8, "nll": 0.1680519571707833, "brier": 0.12840646623967353, "expected_brier": 0.12840646623967353, "multiclass_ece": 0.11384934628042337, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 15, "coverage": 1.0, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.7, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 12, "hard_count": 12, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.06422106727925218, "brier": 0.030138110263277312, "expected_brier": 0.03013811026327734, "multiclass_ece": 0.05458602948785693, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 11, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 8, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.11619462244379324, "brier": 0.024067331501258088, "expected_brier": 0.02406733150125806, "multiclass_ece": 0.10969806630305312, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5979866080936935, "brier": 0.8164254392357402, "expected_brier": 0.8164254392357402, "multiclass_ece": 0.25333438151892756, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2069864765010877, "brier": 0.06991507946888714, "expected_brier": 0.0699150794688872, "multiclass_ece": 0.18696935506773182, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4124796185898986, "brier": 0.22847895631386125, "expected_brier": 0.22847895631386117, "multiclass_ece": 0.33799331081684236, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0917034636138534, "brier": 0.8827371286212814, "expected_brier": 0.8827371286212813, "multiclass_ece": 0.6643557513190058, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5609076933601433, "brier": 0.7875817962159596, "expected_brier": 0.7875817962159597, "multiclass_ece": 0.26131188331042654, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y11": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5823295566064173, "brier": 0.7009186508302253, "expected_brier": 0.7009186508302254, "multiclass_ece": 0.7945041739833069, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y4_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.32168341978386195, "brier": 0.15132959353342645, "expected_brier": 0.15132959353342645, "multiclass_ece": 0.2750723482408096, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.7801965510915061, "brier": 0.44251640590182656, "expected_brier": 0.4425164059018265, "multiclass_ece": 0.5416840800425851, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9750516208115132, "brier": 0.5919857486925515, "expected_brier": 0.5919857486925515, "multiclass_ece": 0.382530793902746, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3501031926963336, "brier": 0.7009587303306608, "expected_brier": 0.7009587303306608, "multiclass_ece": 0.2808450371949719, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.31431629696680785, "brier": 0.14548912184802776, "expected_brier": 0.14548912184802765, "multiclass_ece": 0.2697119962552905, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5984823951247364, "brier": 0.7795964574870716, "expected_brier": 0.7795964574870715, "multiclass_ece": 0.27935646171677875, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8524944610982905, "brier": 0.4103055445766403, "expected_brier": 0.41030554457664037, "multiclass_ece": 0.5736499093171412, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y10": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2970073320528113, "brier": 0.6020279961325188, "expected_brier": 0.6020279961325188, "multiclass_ece": 0.7266513881803085, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5248338046714471, "brier": 0.784350141158733, "expected_brier": 0.784350141158733, "multiclass_ece": 0.2709148679407224, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9344160574820933, "brier": 0.42778096073224886, "expected_brier": 0.4277809607322489, "multiclass_ece": 0.6071848199429339, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y10_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.375181237823471, "brier": 1.1166342827901166, "expected_brier": 1.1166342827901166, "multiclass_ece": 0.7472062241410053, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6716011223243692, "brier": 0.3720369448267427, "expected_brier": 0.37203694482674265, "multiclass_ece": 0.48911007468465806, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1014902790274064, "brier": 0.6941973037697226, "expected_brier": 0.6941973037697227, "multiclass_ece": 0.4469789086941433, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.683282969617768, "brier": 0.8022735949742987, "expected_brier": 0.8022735949742987, "multiclass_ece": 0.30339226812502235, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.461394402550489, "brier": 0.7415192097268062, "expected_brier": 0.7415192097268062, "multiclass_ece": 0.7680873304536602, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y7_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9813522231738355, "brier": 0.7817402332282721, "expected_brier": 0.7817402332282721, "multiclass_ece": 0.6251960625388936, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y8_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3960283873910748, "brier": 0.7094531346385042, "expected_brier": 0.7094531346385042, "multiclass_ece": 0.7524217009956666, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3150911236982465, "brier": 1.0703312147247281, "expected_brier": 1.0703312147247281, "multiclass_ece": 0.7315501400193729, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1423781471601424, "brier": 0.5585868980547724, "expected_brier": 0.5585868980547724, "multiclass_ece": 0.6809406512170217, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y10_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7172021654965495, "brier": 0.8532439796246628, "expected_brier": 0.8532439796246627, "multiclass_ece": 0.2510331445748322, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.6503414818149467, "brier": 1.3057786192162348, "expected_brier": 1.3057786192162348, "multiclass_ece": 0.808015661734423, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.71640668743629, "brier": 0.8466572638363632, "expected_brier": 0.8466572638363632, "multiclass_ece": 0.2394043261226122, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1884920811822792, "brier": 0.966938828344222, "expected_brier": 0.966938828344222, "multiclass_ece": 0.6953196489184748, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8062478588462251, "brier": 0.47579286919373626, "expected_brier": 0.4757928691937363, "multiclass_ece": 0.5534696281177233, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y8": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 1.339909026867815, "brier": 0.6397726069454341, "expected_brier": 0.6397726069454341, "multiclass_ece": 0.19670196647055582, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3113187421837182, "brier": 0.675633500270597, "expected_brier": 0.6756335002705969, "multiclass_ece": 0.7305355321893214, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19499119625187203, "brier": 0.06276998329105495, "expected_brier": 0.06276998329105488, "multiclass_ece": 0.17715809788301373, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.20133587274354767, "brier": 0.0665119708280496, "expected_brier": 0.06651197082804972, "multiclass_ece": 0.1823622368091179, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y6": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3772674953419028, "brier": 0.6742362020984567, "expected_brier": 0.6742362020984567, "multiclass_ece": 0.3149423817635471, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.23242744872411938, "brier": 0.08602350390553817, "expected_brier": 0.08602350390553815, "multiclass_ece": 0.2073927480717903, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8547041417777487, "brier": 0.3745921325266455, "expected_brier": 0.37459213252664547, "multiclass_ece": 0.5745909667740604, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y10_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3033591013565045, "brier": 0.641563395930331, "expected_brier": 0.6415633959303308, "multiclass_ece": 0.7283821330371443, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3886660014463743, "brier": 0.7093966263540221, "expected_brier": 0.7093966263540221, "multiclass_ece": 0.7505922075523831, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1038046649447282, "brier": 0.5340735666740868, "expected_brier": 0.5340735666740868, "multiclass_ece": 0.6683929730641364, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1908998623838112, "brier": 0.7791628562562608, "expected_brier": 0.7791628562562609, "multiclass_ece": 0.5099358873938062, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8048153523633544, "brier": 0.4781114869366342, "expected_brier": 0.4781114869366343, "multiclass_ece": 0.552829512089487, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9700559384037347, "brier": 0.599776666868785, "expected_brier": 0.599776666868785, "multiclass_ece": 0.41398322907740237, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y6_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.8015834253576947, "brier": 0.6080441134728564, "expected_brier": 0.6080441134728565, "multiclass_ece": 0.5513819517688515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.6809870656033277, "brier": 0.8307394221079653, "expected_brier": 0.8307394221079654, "multiclass_ece": 0.23346478677725835, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.936935482940027, "brier": 0.42177984652663314, "expected_brier": 0.42177984652663314, "multiclass_ece": 0.6081732428560724, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8661073168514726, "brier": 0.4013036392806466, "expected_brier": 0.4013036392806466, "multiclass_ece": 0.579414426811186, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5677599760292622, "brier": 0.7861493044587168, "expected_brier": 0.7861493044587169, "multiclass_ece": 0.22430937819377048, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 1.2929656941383443, "brier": 0.5655920365738002, "expected_brier": 0.5655920365738003, "multiclass_ece": 0.39320320645600515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5790620788224962, "brier": 0.3864545530657001, "expected_brier": 0.3864545530657, "multiclass_ece": 0.4395762465521198, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.984076864274597, "brier": 0.9270621610423749, "expected_brier": 0.9270621610423748, "multiclass_ece": 0.39219083583545505, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3694975055943467, "brier": 0.19086126940611783, "expected_brier": 0.1908612694061178, "multiclass_ece": 0.3089184920056728, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3976224807095219, "brier": 0.7150376188107014, "expected_brier": 0.7150376188107015, "multiclass_ece": 0.7528160495102281, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7350269408325192, "brier": 0.8249914310458247, "expected_brier": 0.8249914310458246, "multiclass_ece": 0.3148353432076466, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7182304728660438, "brier": 0.8136016683450649, "expected_brier": 0.8136016683450649, "multiclass_ece": 0.3479687535884655, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.39395363073272577, "brier": 0.21204983149802914, "expected_brier": 0.21204983149802903, "multiclass_ece": 0.325614673731106, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3377675080664388, "brier": 0.6825922370672443, "expected_brier": 0.6825922370672441, "multiclass_ece": 0.7375691100688209, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.294450025482778, "brier": 0.6735382063527955, "expected_brier": 0.6735382063527955, "multiclass_ece": 0.2755957918093992, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y8_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0605014521077971, "brier": 0.8546941257186581, "expected_brier": 0.854694125718658, "multiclass_ece": 0.653717877114684, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_lightness": { "count": 2, "hard_count": 2, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5294003088839445, "brier": 0.7727480695303084, "expected_brier": 0.7727480695303083, "multiclass_ece": 0.2643392653414206, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4092545767361447, "brier": 0.2255969808250614, "expected_brier": 0.22559698082506152, "multiclass_ece": 0.33585486510177387, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4273101286837049, "brier": 0.24184449606141828, "expected_brier": 0.24184449606141833, "multiclass_ece": 0.34773876406105364, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6828597899230511, "brier": 0.3683819384989202, "expected_brier": 0.3683819384989202, "multiclass_ece": 0.4948297561196473, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3090922318714596, "brier": 0.1413917315517783, "expected_brier": 0.1413917315517783, "multiclass_ece": 0.2658869417175074, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4071316214995624, "brier": 0.2237047983751595, "expected_brier": 0.22370479837515944, "multiclass_ece": 0.3344434170193513, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6678204776274697, "brier": 0.4746788163589236, "expected_brier": 0.4746788163589235, "multiclass_ece": 0.4871749256473098, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y9_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9856583770810657, "brier": 0.6225547737053427, "expected_brier": 0.6225547737053427, "multiclass_ece": 0.4422929319313738, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.16490121090949827, "brier": 0.04622169872413302, "expected_brier": 0.046221698724133065, "multiclass_ece": 0.15202252912666103, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2505869995149011, "brier": 0.09826297609791855, "expected_brier": 0.09826297609791856, "multiclass_ece": 0.22165623846163063, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y3": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.915527517240037, "brier": 0.397068550044559, "expected_brier": 0.397068550044559, "multiclass_ece": 0.5996945976173935, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9440414705665002, "brier": 0.428448032264833, "expected_brier": 0.42844803226483297, "multiclass_ece": 0.6109476896770232, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.329685235215226, "brier": 0.6542851447985824, "expected_brier": 0.6542851447985822, "multiclass_ece": 0.735439477468781, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3546667905056022, "brier": 0.6385741447976025, "expected_brier": 0.6385741447976025, "multiclass_ece": 0.741966740739259, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y2": { "count": 2, "hard_count": 2, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.4625855628148416, "brier": 0.7192922756180997, "expected_brier": 0.7192922756180997, "multiclass_ece": 0.3301458519750476, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4670284618874761, "brier": 0.2784635632252332, "expected_brier": 0.27846356322523325, "multiclass_ece": 0.3731377515243085, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5428923403143775, "brier": 0.7612741903637107, "expected_brier": 0.7612741903637108, "multiclass_ece": 0.33214062977286224, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/decision": { "count": 27, "hard_count": 27, "accuracy": 0.6296296296296297, "expected_accuracy": 0.6296296296296297, "nll": 1.1467973654815216, "brier": 0.605648762293523, "expected_brier": 0.605648762293523, "multiclass_ece": 0.3347311532110949, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 15, "coverage": 0.5555555555555556, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.8, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/ordinal": { "count": 26, "hard_count": 26, "accuracy": 0.5384615384615384, "expected_accuracy": 0.5384615384615384, "nll": 1.1639128133315753, "brier": 0.5843111231808523, "expected_brier": 0.5843111231808522, "multiclass_ece": 0.2289689625659578, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 0.3076923076923077, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.7, "selected": 3, "coverage": 0.11538461538461539, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.07692307692307693, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.038461538461538464, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/verification": { "count": 27, "hard_count": 27, "accuracy": 0.5185185185185185, "expected_accuracy": 0.5185185185185185, "nll": 0.7956287183050771, "brier": 0.5480060018545826, "expected_brier": 0.5480060018545826, "multiclass_ece": 0.26913371512364564, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.5185185185185185, "expected_accuracy": 0.5185185185185185, "expected_risk": 0.4814814814814815 }, { "threshold": 0.7, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 0.6923076923076923, "expected_accuracy": 0.6923076923076923, "expected_risk": 0.3076923076923077 }, { "threshold": 0.8, "selected": 11, "coverage": 0.4074074074074074, "accuracy": 0.8181818181818182, "expected_accuracy": 0.8181818181818182, "expected_risk": 0.18181818181818177 }, { "threshold": 0.9, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.95, "selected": 5, "coverage": 0.18518518518518517, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/choice": { "count": 27, "hard_count": 21, "accuracy": 0.19047619047619047, "expected_accuracy": 0.18518518518518517, "nll": 1.268094519462248, "brier": 0.6549523896349503, "expected_brier": 0.7660635007460613, "multiclass_ece": 0.23580802374143012, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "nll": 0.45137952834028433, "brier": 0.27392566296025217, "expected_brier": 0.2739256629602523, "multiclass_ece": 0.07351862576884362, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.8, "selected": 6, "coverage": 0.2222222222222222, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/choice": { "count": 27, "hard_count": 27, "accuracy": 0.25925925925925924, "expected_accuracy": 0.25925925925925924, "nll": 1.3256195956466352, "brier": 0.8324967853336331, "expected_brier": 0.832496785333633, "multiclass_ece": 0.41554316833754046, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.2916666666666667, "expected_accuracy": 0.2916666666666667, "expected_risk": 0.7083333333333333 }, { "threshold": 0.7, "selected": 6, "coverage": 0.2222222222222222, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.6397743222229317, "brier": 0.4495477394413823, "expected_brier": 0.4495477394413823, "multiclass_ece": 0.19535796711812048, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 5, "coverage": 0.18518518518518517, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/score": { "count": 27, "hard_count": 27, "accuracy": 0.5555555555555556, "expected_accuracy": 0.5555555555555556, "nll": 1.0766157520280073, "brier": 0.5520774849832285, "expected_brier": 0.5520774849832286, "multiclass_ece": 0.20561717800685894, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 0.4444444444444444, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1/choice": { "count": 27, "hard_count": 9, "accuracy": 0.2222222222222222, "expected_accuracy": 0.10823045267489712, "nll": 2.8591546846934452, "brier": 0.49474953777578484, "expected_brier": 1.002877109792246, "multiclass_ece": 0.19076155939447956, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.037037037037037035, "accuracy": null, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 4, "hard_count": 4, "accuracy": 0.25, "expected_accuracy": 0.25, "nll": 0.742022059936596, "brier": 0.5777575818466366, "expected_brier": 0.5777575818466367, "multiclass_ece": 0.4652681397956143, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.25, "expected_accuracy": 0.25, "expected_risk": 0.75 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.06414951516105503, "brier": 0.009847974413922083, "expected_brier": 0.00984797441392212, "multiclass_ece": 0.06152545795287464, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0818071448652209, "brier": 0.015835112549222544, "expected_brier": 0.01583511254922254, "multiclass_ece": 0.07751508322090378, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2977167627354933, "brier": 0.13260053483566778, "expected_brier": 0.1326005348356678, "multiclass_ece": 0.25748838307355515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 6, "hard_count": 6, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.3342306367408157, "brier": 0.20924345500034303, "expected_brier": 0.20924345500034294, "multiclass_ece": 0.23237794790963026, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 4, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 0.6, "expected_accuracy": 0.6, "nll": 0.6952723321085037, "brier": 0.503131901198782, "expected_brier": 0.5031319011987819, "multiclass_ece": 0.23890562910641036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.8, "selected": 1, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.31393380188510456, "brier": 0.15461721559023678, "expected_brier": 0.15461721559023678, "multiclass_ece": 0.2643006525711456, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.13072788270534222, "brier": 0.04810013390185639, "expected_brier": 0.04810013390185647, "multiclass_ece": 0.11592803858131343, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1941146814521919, "brier": 0.10660032556919237, "expected_brier": 0.10660032556919237, "multiclass_ece": 0.157509434293308, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.39706728050400053, "brier": 0.2156884721905102, "expected_brier": 0.2156884721905102, "multiclass_ece": 0.32706395544284456, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.21082583704231572, "brier": 0.09798898649144361, "expected_brier": 0.09798898649144361, "multiclass_ece": 0.1787973890493537, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.599639416815768, "brier": 0.4094796858535742, "expected_brier": 0.4094796858535742, "multiclass_ece": 0.2516049044085984, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.14529420627049117, "brier": 0.03657547659753377, "expected_brier": 0.036575476597533685, "multiclass_ece": 0.1352321644386678, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4019245139131147, "brier": 0.23087194704671835, "expected_brier": 0.23087194704671835, "multiclass_ece": 0.3225291681972606, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3830034004104313, "brier": 0.20521118195369004, "expected_brier": 0.20521118195369004, "multiclass_ece": 0.3163638233291547, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19967880967211568, "brier": 0.06614547912588008, "expected_brier": 0.06614547912588013, "multiclass_ece": 0.18076502558611418, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.29917981546210576, "brier": 0.1408176642890942, "expected_brier": 0.14081766428909426, "multiclass_ece": 0.2549270918146309, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.43423568913148786, "brier": 0.24856828210047346, "expected_brier": 0.24856828210047346, "multiclass_ece": 0.3518917008185785, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6046763418824027, "brier": 0.4118681702699163, "expected_brier": 0.4118681702699163, "multiclass_ece": 0.45350478240448466, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.409549814337095, "brier": 0.22655870115775806, "expected_brier": 0.22655870115775806, "multiclass_ece": 0.33552413380910284, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.21393768003072122, "brier": 0.07419052112504426, "expected_brier": 0.0741905211250442, "multiclass_ece": 0.1926012994829529, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.38782054478950273, "brier": 0.2090609285341778, "expected_brier": 0.2090609285341778, "multiclass_ece": 0.31978204225337825, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0213050388154639, "brier": 0.8188809098873693, "expected_brier": 0.8188809098873693, "multiclass_ece": 0.6398753432846781, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.603364878565443, "brier": 0.41050828631712405, "expected_brier": 0.4105082863171241, "multiclass_ece": 0.45294579514876865, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.05305795745557958, "brier": 0.005869193346775597, "expected_brier": 0.005869193346775603, "multiclass_ece": 0.051525875707719354, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/AUTO CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07983862395943388, "brier": 0.012202454186091886, "expected_brier": 0.012202454186091819, "multiclass_ece": 0.0766099305724232, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.27961946715979347, "brier": 0.17200525041381434, "expected_brier": 0.17200525041381431, "multiclass_ece": 0.2192451406113623, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.18405306501146246, "brier": 0.10633435199954197, "expected_brier": 0.106334351999542, "multiclass_ece": 0.14530125402493502, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3834583161152853, "brier": 0.2414104499653037, "expected_brier": 0.24141044996530373, "multiclass_ece": 0.29705769555840505, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3775361109428338, "brier": 0.19775956046046997, "expected_brier": 0.19775956046047005, "multiclass_ece": 0.3144515546634091, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7502189036459033, "brier": 0.554428369509535, "expected_brier": 0.554428369509535, "multiclass_ece": 0.12643191914034202, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4016259844674533, "brier": 0.21881626050306216, "expected_brier": 0.21881626050306213, "multiclass_ece": 0.33076899832289464, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1651175790044262, "brier": 0.046333323465005995, "expected_brier": 0.04633332346500607, "multiclass_ece": 0.15220598454890988, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19082282665091152, "brier": 0.09017521330925375, "expected_brier": 0.09017521330925382, "multiclass_ece": 0.1624910796873042, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2995682349707719, "brier": 0.1340189156567592, "expected_brier": 0.13401891565675927, "multiclass_ece": 0.2588618508555859, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.04260553136841881, "brier": 0.003515978971941086, "expected_brier": 0.003515978971941003, "multiclass_ece": 0.04170073607514535, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.06366744642276558, "brier": 0.007609587859236321, "expected_brier": 0.0076095878592363775, "multiclass_ece": 0.06168301167759371, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2637814880910121, "brier": 0.14674152464913867, "expected_brier": 0.1467415246491387, "multiclass_ece": 0.21415745099648326, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4608926041647519, "brier": 0.2786339910149602, "expected_brier": 0.27863399101496034, "multiclass_ece": 0.36375530884373014, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } } }, "baseline_calibrated_test": { "count": 512, "hard_count": 470, "accuracy": 0.6468085106382979, "expected_accuracy": 0.615909443204365, "nll": 0.8763029697640415, "brier": 0.43757033588547545, "expected_brier": 0.4888623435739674, "multiclass_ece": 0.03803263605517923, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 367, "coverage": 0.716796875, "accuracy": 0.729050279329609, "expected_accuracy": 0.7229791099000907, "expected_risk": 0.27702089009990927 }, { "threshold": 0.7, "selected": 186, "coverage": 0.36328125, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8378136200716845, "expected_risk": 0.16218637992831553 }, { "threshold": 0.8, "selected": 104, "coverage": 0.203125, "accuracy": 0.9134615384615384, "expected_accuracy": 0.9134615384615384, "expected_risk": 0.08653846153846156 }, { "threshold": 0.9, "selected": 69, "coverage": 0.134765625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 45, "coverage": 0.087890625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.037109375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "mean_latency_seconds": 0.08172164261850412, "by_kind": { "choice": { "count": 197, "hard_count": 158, "accuracy": 0.4873417721518987, "expected_accuracy": 0.440840786399162, "nll": 1.2301818414097745, "brier": 0.503344630600222, "expected_brier": 0.6290375642880797, "multiclass_ece": 0.10823337361620336, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 98, "coverage": 0.49746192893401014, "accuracy": 0.6630434782608695, "expected_accuracy": 0.6513605442176871, "expected_risk": 0.3486394557823129 }, { "threshold": 0.7, "selected": 39, "coverage": 0.19796954314720813, "accuracy": 0.918918918918919, "expected_accuracy": 0.8931623931623932, "expected_risk": 0.10683760683760679 }, { "threshold": 0.8, "selected": 23, "coverage": 0.116751269035533, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.9, "selected": 17, "coverage": 0.08629441624365482, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 0.06091370558375635, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "noul": { "count": 219, "hard_count": 219, "accuracy": 0.7899543378995434, "expected_accuracy": 0.7899543378995434, "nll": 0.4774366279452144, "brier": 0.3233082518915342, "expected_brier": 0.3233082518915342, "multiclass_ece": 0.09186999825973236, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 219, "coverage": 1.0, "accuracy": 0.7899543378995434, "expected_accuracy": 0.7899543378995434, "expected_risk": 0.2100456621004566 }, { "threshold": 0.7, "selected": 129, "coverage": 0.589041095890411, "accuracy": 0.8217054263565892, "expected_accuracy": 0.8217054263565892, "expected_risk": 0.17829457364341084 }, { "threshold": 0.8, "selected": 73, "coverage": 0.3333333333333333, "accuracy": 0.9041095890410958, "expected_accuracy": 0.9041095890410958, "expected_risk": 0.09589041095890416 }, { "threshold": 0.9, "selected": 51, "coverage": 0.2328767123287671, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 33, "coverage": 0.1506849315068493, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.0867579908675799, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "score": { "count": 96, "hard_count": 93, "accuracy": 0.5806451612903226, "expected_accuracy": 0.578125, "nll": 1.0600278775152285, "brier": 0.5632563810507684, "expected_brier": 0.5788813810507684, "multiclass_ece": 0.1742922651967985, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 50, "coverage": 0.5208333333333334, "accuracy": 0.574468085106383, "expected_accuracy": 0.57, "expected_risk": 0.43000000000000005 }, { "threshold": 0.7, "selected": 18, "coverage": 0.1875, "accuracy": 0.875, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 8, "coverage": 0.08333333333333333, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.9, "selected": 1, "coverage": 0.010416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "ordinal_mae": 0.7141080706798255 } }, "by_source": { "customer-control-v1": { "count": 72, "hard_count": 68, "accuracy": 0.8529411764705882, "expected_accuracy": 0.8333333333333334, "nll": 0.33466537905082433, "brier": 0.17576688900364557, "expected_brier": 0.2035446667814233, "multiclass_ece": 0.07505337939577306, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 71, "coverage": 0.9861111111111112, "accuracy": 0.8507462686567164, "expected_accuracy": 0.8309859154929577, "expected_risk": 0.16901408450704225 }, { "threshold": 0.7, "selected": 58, "coverage": 0.8055555555555556, "accuracy": 0.9636363636363636, "expected_accuracy": 0.9396551724137931, "expected_risk": 0.06034482758620685 }, { "threshold": 0.8, "selected": 46, "coverage": 0.6388888888888888, "accuracy": 0.9782608695652174, "expected_accuracy": 0.9782608695652174, "expected_risk": 0.021739130434782594 }, { "threshold": 0.9, "selected": 39, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 35, "coverage": 0.4861111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.2638888888888889, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 74, "hard_count": 74, "accuracy": 0.8243243243243243, "expected_accuracy": 0.8243243243243243, "nll": 0.6592332651613164, "brier": 0.375833842293872, "expected_brier": 0.375833842293872, "multiclass_ece": 0.25779504868999725, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 60, "coverage": 0.8108108108108109, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.7, "selected": 27, "coverage": 0.36486486486486486, "accuracy": 0.7777777777777778, "expected_accuracy": 0.7777777777777778, "expected_risk": 0.2222222222222222 }, { "threshold": 0.8, "selected": 16, "coverage": 0.21621621621621623, "accuracy": 0.8125, "expected_accuracy": 0.8125, "expected_risk": 0.1875 }, { "threshold": 0.9, "selected": 7, "coverage": 0.0945945945945946, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 0.06756756756756757, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1": { "count": 72, "hard_count": 72, "accuracy": 0.6111111111111112, "expected_accuracy": 0.6111111111111112, "nll": 0.8834356742861179, "brier": 0.5145067677774604, "expected_brier": 0.5145067677774604, "multiclass_ece": 0.1135965070509722, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 47, "coverage": 0.6527777777777778, "accuracy": 0.6808510638297872, "expected_accuracy": 0.6808510638297872, "expected_risk": 0.3191489361702128 }, { "threshold": 0.7, "selected": 19, "coverage": 0.2638888888888889, "accuracy": 0.7368421052631579, "expected_accuracy": 0.7368421052631579, "expected_risk": 0.26315789473684215 }, { "threshold": 0.8, "selected": 8, "coverage": 0.1111111111111111, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.9, "selected": 4, "coverage": 0.05555555555555555, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.027777777777777776, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1": { "count": 48, "hard_count": 43, "accuracy": 0.5581395348837209, "expected_accuracy": 0.5243055555555556, "nll": 0.8702447761097233, "brier": 0.48375444662121736, "expected_brier": 0.5427822243989952, "multiclass_ece": 0.1079420678476908, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 0.5, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 20, "coverage": 0.4166666666666667, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tic_tac_toe-v1": { "count": 24, "hard_count": 7, "accuracy": 0.42857142857142855, "expected_accuracy": 0.3531746031746032, "nll": 1.255800529131762, "brier": 0.2765412764144371, "expected_brier": 0.6942000065731669, "multiclass_ece": 0.0849307307977768, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1": { "count": 24, "hard_count": 24, "accuracy": 0.041666666666666664, "expected_accuracy": 0.041666666666666664, "nll": 1.764217525720681, "brier": 0.8628098090544826, "expected_brier": 0.8628098090544828, "multiclass_ece": 0.27056221095781013, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1": { "count": 4, "hard_count": 4, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7471285595952405, "brier": 0.43871328284801203, "expected_brier": 0.43871328284801203, "multiclass_ece": 0.3602701624597008, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1": { "count": 73, "hard_count": 73, "accuracy": 0.3561643835616438, "expected_accuracy": 0.3561643835616438, "nll": 1.2816606587154418, "brier": 0.7688306061202909, "expected_brier": 0.7688306061202909, "multiclass_ece": 0.23428267433815733, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 57, "coverage": 0.7808219178082192, "accuracy": 0.3508771929824561, "expected_accuracy": 0.3508771929824561, "expected_risk": 0.6491228070175439 }, { "threshold": 0.7, "selected": 7, "coverage": 0.0958904109589041, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.1959490740740741, "nll": 2.4385308499338874, "brier": 0.4501102000587825, "expected_brier": 0.9252920783656607, "multiclass_ece": 0.18975573278293156, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.41666666666666663, "expected_risk": 0.5833333333333334 }, { "threshold": 0.7, "selected": 1, "coverage": 0.041666666666666664, "accuracy": null, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.5416666666666666, "optimal_set_probability_mass": 0.4244638135379455 }, "workflow-controls-v1/agent_trace_observability": { "count": 24, "hard_count": 24, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.4382765209950806, "brier": 0.2897293275725255, "expected_brier": 0.2897293275725254, "multiclass_ece": 0.16770566865941738, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 0.8421052631578947, "expected_accuracy": 0.8421052631578947, "expected_risk": 0.1578947368421053 }, { "threshold": 0.8, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8461538461538461, "expected_risk": 0.15384615384615385 }, { "threshold": 0.9, "selected": 6, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service": { "count": 24, "hard_count": 24, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "nll": 0.4501296071912735, "brier": 0.29138615746381247, "expected_brier": 0.29138615746381247, "multiclass_ece": 0.17222498340081047, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "expected_risk": 0.20833333333333337 }, { "threshold": 0.7, "selected": 15, "coverage": 0.625, "accuracy": 0.8666666666666667, "expected_accuracy": 0.8666666666666667, "expected_risk": 0.1333333333333333 }, { "threshold": 0.8, "selected": 12, "coverage": 0.5, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 7, "coverage": 0.2916666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.4533678207150369, "brier": 0.277702349610057, "expected_brier": 0.277702349610057, "multiclass_ece": 0.31259443330916176, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 6, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents": { "count": 25, "hard_count": 25, "accuracy": 0.84, "expected_accuracy": 0.84, "nll": 0.42605371962448196, "brier": 0.2666481862082871, "expected_brier": 0.2666481862082872, "multiclass_ece": 0.17000359018439912, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 0.84, "expected_accuracy": 0.84, "expected_risk": 0.16000000000000003 }, { "threshold": 0.7, "selected": 12, "coverage": 0.48, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 0.24, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 0.375, "expected_accuracy": 0.375, "nll": 1.3968551702656242, "brier": 0.8921208817155377, "expected_brier": 0.8921208817155376, "multiclass_ece": 0.5920780945540082, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.875, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.7, "selected": 3, "coverage": 0.375, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.8, "selected": 2, "coverage": 0.25, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/category": { "count": 24, "hard_count": 23, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "nll": 0.1670543821040239, "brier": 0.04600238211641114, "expected_brier": 0.0668357154497445, "multiclass_ece": 0.10593632496525253, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "expected_risk": 0.02083333333333337 }, { "threshold": 0.7, "selected": 22, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 0.9772727272727273, "expected_risk": 0.022727272727272707 }, { "threshold": 0.8, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/churn_likelihood_level": { "count": 7, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.17342994770395262, "brier": 0.053028498584162566, "expected_brier": 0.05302849858416254, "multiclass_ece": 0.15689348194752176, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 0.7142857142857143, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.14285714285714285, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/frustration": { "count": 9, "hard_count": 6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3888888888888889, "nll": 0.7565718587447946, "brier": 0.3879899658847263, "expected_brier": 0.5546566325513927, "multiclass_ece": 0.2471424317689477, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 1.0, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3888888888888889, "expected_risk": 0.6111111111111112 }, { "threshold": 0.7, "selected": 3, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/has_reproducible_steps": { "count": 14, "hard_count": 14, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "nll": 0.05559666981857294, "brier": 0.03908001953056985, "expected_brier": 0.039080019530569846, "multiclass_ece": 0.0400834715956282, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 14, "coverage": 1.0, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "expected_risk": 0.0714285714285714 }, { "threshold": 0.7, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 13, "coverage": 0.9285714285714286, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.011025101894694089, "brier": 0.00039623272646610515, "expected_brier": 0.00039623272646611697, "multiclass_ece": 0.010924285594151683, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 0.6, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.34601990374302266, "brier": 0.12377351834438345, "expected_brier": 0.12377351834438355, "multiclass_ece": 0.2925015949000842, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.2353547133769227, "brier": 0.8480755908519118, "expected_brier": 0.8480755908519118, "multiclass_ece": 0.5708321717431446, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5560310782165047, "brier": 0.20138494037074348, "expected_brier": 0.2013849403707435, "multiclass_ece": 0.42651934715712747, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1758638662792829, "brier": 0.6022687602464171, "expected_brier": 0.602268760246417, "multiclass_ece": 0.6914476834177192, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y2_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6627430112033167, "brier": 0.4696054160178698, "expected_brier": 0.46960541601786976, "multiclass_ece": 0.48456445186263397, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.566895400560929, "brier": 1.2523364390849077, "expected_brier": 1.2523364390849077, "multiclass_ece": 0.7913079170224786, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0295213682150712, "brier": 0.5334962254023893, "expected_brier": 0.5334962254023892, "multiclass_ece": 0.6428221236524843, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.04830825086259792, "brier": 0.004448125909869939, "expected_brier": 0.004448125909869871, "multiclass_ece": 0.04715997195646926, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.348569775161871, "brier": 0.933480582900835, "expected_brier": 0.933480582900835, "multiclass_ece": 0.606009360525036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3131210038686904, "brier": 0.11128406454212035, "expected_brier": 0.11128406454212025, "multiclass_ece": 0.268838566146277, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.10515240715275168, "brier": 0.019925143317410993, "expected_brier": 0.01992514331741102, "multiclass_ece": 0.09981268285496336, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1570127994022004, "brier": 0.6032122052799056, "expected_brier": 0.6032122052799056, "multiclass_ece": 0.6855759729405255, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3754545486919703, "brier": 0.10839406037984879, "expected_brier": 0.10839406037984878, "multiclass_ece": 0.31302305666049424, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1560772668702688, "brier": 0.604179376585817, "expected_brier": 0.604179376585817, "multiclass_ece": 0.6851105962609545, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.11065138778372995, "brier": 0.0219447940343179, "expected_brier": 0.021944794034317816, "multiclass_ece": 0.10474921010279237, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y2": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2875658566291677, "brier": 0.09572317877617785, "expected_brier": 0.0957231787761778, "multiclass_ece": 0.2294431298537569, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.17327931969466012, "brier": 0.029876387378521645, "expected_brier": 0.029876387378521718, "multiclass_ece": 0.15909729864761257, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5720130305722009, "brier": 0.2846909164311833, "expected_brier": 0.28469091643118327, "multiclass_ece": 0.4356118362292746, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1396428918814943, "brier": 0.5890525702507625, "expected_brier": 0.5890525702507625, "multiclass_ece": 0.6800667478194717, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.542575689465163, "brier": 0.21244409863985692, "expected_brier": 0.2124440986398568, "multiclass_ece": 0.4187507947917507, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.345783508366425, "brier": 0.13227309360102196, "expected_brier": 0.1322730936010219, "multiclass_ece": 0.2923343257781238, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3903756028632871, "brier": 0.20891309578616216, "expected_brier": 0.20891309578616224, "multiclass_ece": 0.3231973822497346, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08204241477867935, "brier": 0.007094683092112937, "expected_brier": 0.00709468309211303, "multiclass_ece": 0.07876711602120434, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y1": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.43785182394176814, "brier": 0.17607318029248473, "expected_brier": 0.17607318029248475, "multiclass_ece": 0.30389138283343625, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1036823518105074, "brier": 0.5700034585598057, "expected_brier": 0.5700034585598057, "multiclass_ece": 0.6683524106887371, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y3_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3778944184421238, "brier": 0.12281169922037827, "expected_brier": 0.12281169922037827, "multiclass_ece": 0.31469714781093905, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6918774529241388, "brier": 0.37899801241400993, "expected_brier": 0.37899801241401, "multiclass_ece": 0.49936473295938655, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y4_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.648893269300428, "brier": 1.3048794993350745, "expected_brier": 1.3048794993350743, "multiclass_ece": 0.8077374261896852, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.02932142810440297, "brier": 0.0016699259517416474, "expected_brier": 0.0016699259517416465, "multiclass_ece": 0.028895725910086156, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y5_green": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.9724400696440427, "brier": 0.7542565305959927, "expected_brier": 0.754256530595993, "multiclass_ece": 0.6003310226540968, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2349581651066827, "brier": 0.08769343348295694, "expected_brier": 0.08769343348295688, "multiclass_ece": 0.20939607623228873, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5835649910971529, "brier": 0.30605021131483423, "expected_brier": 0.30605021131483423, "multiclass_ece": 0.44209411243154373, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6678525416201178, "brier": 0.4747108592827403, "expected_brier": 0.47471085928274026, "multiclass_ece": 0.48719136860310874, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y4": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4338237239639619, "brier": 0.1420932374253576, "expected_brier": 0.14209323742535762, "multiclass_ece": 0.35197352304359364, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y5_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3610477937194683, "brier": 1.1059057443449705, "expected_brier": 1.1059057443449705, "multiclass_ece": 0.7436080097554659, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5234233413459157, "brier": 0.24119816772160088, "expected_brier": 0.24119816772160085, "multiclass_ece": 0.4075112191023217, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.15175555941065774, "brier": 0.022646367185360835, "expected_brier": 0.022646367185360727, "multiclass_ece": 0.14080172199241192, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.42643433203539727, "brier": 0.2410502206571369, "expected_brier": 0.241050220657137, "multiclass_ece": 0.34716726563512357, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0922947356223056, "brier": 0.7088285238584322, "expected_brier": 0.7088285238584322, "multiclass_ece": 0.48456003026316463, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5429324273384957, "brier": 0.21925808806709107, "expected_brier": 0.21925808806709113, "multiclass_ece": 0.41895811141604855, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8877704096830917, "brier": 0.45512311901602875, "expected_brier": 0.45512311901602875, "multiclass_ece": 0.5884276316988728, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4388461405004222, "brier": 0.19283710094677695, "expected_brier": 0.19283710094677686, "multiclass_ece": 0.35522002247698403, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9560041414634272, "brier": 0.4974315842551159, "expected_brier": 0.497431584255116, "multiclass_ece": 0.6155740673388588, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.2737789693329942, "brier": 0.8527147015360317, "expected_brier": 0.8527147015360317, "multiclass_ece": 0.5532755913793204, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.40433951030677856, "brier": 0.1337517635919014, "expected_brier": 0.13375176359190144, "multiclass_ece": 0.3325825123171059, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6656014309881316, "brier": 0.472461363176453, "expected_brier": 0.47246136317645293, "multiclass_ece": 0.48603567933663727, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.462947609955955, "brier": 0.2746507610800844, "expected_brier": 0.27465076108008435, "multiclass_ece": 0.3705743927203311, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4945928240836845, "brier": 0.235327476953064, "expected_brier": 0.235327476953064, "multiclass_ece": 0.39018083957303495, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y6_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.12469557499752333, "brier": 0.027487810210755904, "expected_brier": 0.02748781021075586, "multiclass_ece": 0.1172344023969839, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9137327907975775, "brier": 0.47883744770538555, "expected_brier": 0.4788374477053856, "multiclass_ece": 0.5989755138403603, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3882523029611703, "brier": 0.12598440433575223, "expected_brier": 0.1259844043357523, "multiclass_ece": 0.32175880058803774, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.8731435578992864, "brier": 0.6782942158546762, "expected_brier": 0.6782942158546761, "multiclass_ece": 0.5823633813413565, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.2643300794796768, "brier": 0.8529971108370309, "expected_brier": 0.8529971108370309, "multiclass_ece": 0.559576330917459, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y1_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.17160891385360222, "brier": 0.04973320323339839, "expected_brier": 0.04973320323339836, "multiclass_ece": 0.15769147604325096, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.12588843478183329, "brier": 0.027983529460764055, "expected_brier": 0.02798352946076399, "multiclass_ece": 0.1182867901770186, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3481676120381354, "brier": 0.11864891037677, "expected_brier": 0.11864891037677006, "multiclass_ece": 0.2940194645394385, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0378139216405164, "brier": 0.556397247107488, "expected_brier": 0.5563972471074878, "multiclass_ece": 0.6457717932250533, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y4": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3268658277947037, "brier": 0.08903331269950415, "expected_brier": 0.08903331269950421, "multiclass_ece": 0.27659984561413375, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.24869956005121285, "brier": 0.05532819874885563, "expected_brier": 0.05532819874885564, "multiclass_ece": 0.22018577446042975, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3657594027424609, "brier": 0.14972518458441886, "expected_brier": 0.14972518458441897, "multiclass_ece": 0.30633032384415526, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1199418104326047, "brier": 0.7371946251292734, "expected_brier": 0.7371946251292734, "multiclass_ece": 0.504756277714049, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4940445967859731, "brier": 0.23158169046001487, "expected_brier": 0.23158169046001476, "multiclass_ece": 0.3898464284043315, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_green": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7941680943070757, "brier": 1.3902257252340244, "expected_brier": 1.3902257252340244, "multiclass_ece": 0.8337342877781939, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y2_green": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.8378381921971725, "brier": 1.414020009814757, "expected_brier": 1.4140200098147568, "multiclass_ece": 0.8408388697648191, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.40570973796693427, "brier": 0.177894707884683, "expected_brier": 0.1778947078846831, "multiclass_ece": 0.3334963999595505, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3561076436124062, "brier": 0.6967740222971983, "expected_brier": 0.6967740222971983, "multiclass_ece": 0.7423382610451552, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.26707115435308393, "brier": 0.08413595927782934, "expected_brier": 0.08413595927782935, "multiclass_ece": 0.21486105115673854, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/decision": { "count": 24, "hard_count": 24, "accuracy": 0.5416666666666666, "expected_accuracy": 0.5416666666666666, "nll": 1.036218726779147, "brier": 0.6189308675627149, "expected_brier": 0.618930867562715, "multiclass_ece": 0.2710675963627027, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 0.5625, "expected_accuracy": 0.5625, "expected_risk": 0.4375 }, { "threshold": 0.7, "selected": 6, "coverage": 0.25, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 3, "coverage": 0.125, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.9, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/ordinal": { "count": 24, "hard_count": 24, "accuracy": 0.5833333333333334, "expected_accuracy": 0.5833333333333334, "nll": 1.0647112252882824, "brier": 0.5442581908768647, "expected_brier": 0.5442581908768648, "multiclass_ece": 0.16885435938019264, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.2916666666666667, "accuracy": 0.8571428571428571, "expected_accuracy": 0.8571428571428571, "expected_risk": 0.1428571428571429 }, { "threshold": 0.7, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/verification": { "count": 24, "hard_count": 24, "accuracy": 0.7083333333333334, "expected_accuracy": 0.7083333333333334, "nll": 0.5493770707909253, "brier": 0.380331244892802, "expected_brier": 0.38033124489280185, "multiclass_ece": 0.07196784051703685, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.7083333333333334, "expected_accuracy": 0.7083333333333334, "expected_risk": 0.29166666666666663 }, { "threshold": 0.7, "selected": 10, "coverage": 0.4166666666666667, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.8, "selected": 4, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.08333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/choice": { "count": 24, "hard_count": 19, "accuracy": 0.3157894736842105, "expected_accuracy": 0.2986111111111111, "nll": 1.187392967756167, "brier": 0.5995280937360439, "expected_brier": 0.7175836492915995, "multiclass_ece": 0.1551506834473081, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/noul": { "count": 24, "hard_count": 24, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.5530965844632794, "brier": 0.3679807995063907, "expected_brier": 0.3679807995063908, "multiclass_ece": 0.060733452248073506, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 20, "coverage": 0.8333333333333334, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tic_tac_toe-v1/choice": { "count": 24, "hard_count": 7, "accuracy": 0.42857142857142855, "expected_accuracy": 0.3531746031746032, "nll": 1.255800529131762, "brier": 0.2765412764144371, "expected_brier": 0.6942000065731669, "multiclass_ece": 0.0849307307977768, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 0.20833333333333334, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.041666666666666664, "expected_accuracy": 0.041666666666666664, "nll": 1.764217525720681, "brier": 0.8628098090544826, "expected_brier": 0.8628098090544828, "multiclass_ece": 0.27056221095781013, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1/choice": { "count": 4, "hard_count": 4, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7471285595952405, "brier": 0.43871328284801203, "expected_brier": 0.43871328284801203, "multiclass_ece": 0.3602701624597008, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.08333333333333333, "expected_accuracy": 0.08333333333333333, "nll": 1.7034300962167699, "brier": 1.0517708355030433, "expected_brier": 1.0517708355030433, "multiclass_ece": 0.5254071180681376, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.08695652173913043, "expected_accuracy": 0.08695652173913043, "expected_risk": 0.9130434782608696 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 25, "hard_count": 25, "accuracy": 0.56, "expected_accuracy": 0.56, "nll": 0.7592922354308388, "brier": 0.5588078485784894, "expected_brier": 0.5588078485784894, "multiclass_ece": 0.24021070512596793, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 0.56, "expected_accuracy": 0.56, "expected_risk": 0.43999999999999995 }, { "threshold": 0.7, "selected": 7, "coverage": 0.28, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/score": { "count": 24, "hard_count": 24, "accuracy": 0.4166666666666667, "expected_accuracy": 0.4166666666666667, "nll": 1.404024995468907, "brier": 0.7046640825102483, "expected_brier": 0.7046640825102485, "multiclass_ece": 0.13152899687613628, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 0.375, "accuracy": 0.4444444444444444, "expected_accuracy": 0.4444444444444444, "expected_risk": 0.5555555555555556 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1/choice": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.1959490740740741, "nll": 2.4385308499338874, "brier": 0.4501102000587825, "expected_brier": 0.9252920783656607, "multiclass_ece": 0.18975573278293156, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.41666666666666663, "expected_risk": 0.5833333333333334 }, { "threshold": 0.7, "selected": 1, "coverage": 0.041666666666666664, "accuracy": null, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.09106250518282048, "brier": 0.02970347966355448, "expected_brier": 0.029703479663554466, "multiclass_ece": 0.08242627707879886, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1384403056768878, "brier": 0.04002525947975625, "expected_brier": 0.04002525947975618, "multiclass_ece": 0.1270796912958357, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2009392305409782, "brier": 0.10251763702139938, "expected_brier": 0.10251763702139938, "multiclass_ece": 0.16448167831938224, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.6, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5298473143094675, "brier": 0.3424961172628109, "expected_brier": 0.3424961172628108, "multiclass_ece": 0.4061237715878367, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.11734704713883631, "brier": 0.02451937860787374, "expected_brier": 0.024519378607873676, "multiclass_ece": 0.11072348126724008, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 0.4, "expected_accuracy": 0.4, "nll": 1.0200828790154524, "brier": 0.7725448784940866, "expected_brier": 0.7725448784940866, "multiclass_ece": 0.4813426942556706, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.8, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.49609373971465015, "brier": 0.324833739691377, "expected_brier": 0.324833739691377, "multiclass_ece": 0.36454372197833534, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08049041935078421, "brier": 0.012355273064936063, "expected_brier": 0.012355273064936112, "multiclass_ece": 0.0772199036985508, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.39655205603136934, "brier": 0.24479361443494688, "expected_brier": 0.24479361443494688, "multiclass_ece": 0.309945433675761, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 3, "hard_count": 3, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "nll": 0.6321908433285975, "brier": 0.45301012287060094, "expected_brier": 0.453010122870601, "multiclass_ece": 0.44677297886873363, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07537288775953904, "brier": 0.01065118961519945, "expected_brier": 0.010651189615199508, "multiclass_ece": 0.07257062522877422, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5751164814786862, "brier": 0.38256871494879996, "expected_brier": 0.38256871494879996, "multiclass_ece": 0.4373606720710036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1837262106271971, "brier": 0.05643530626194777, "expected_brier": 0.05643530626194774, "multiclass_ece": 0.16779980928071878, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.36588179357792305, "brier": 0.18893090725189965, "expected_brier": 0.18893090725189965, "multiclass_ece": 0.30566098732849767, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 5, "hard_count": 5, "accuracy": 0.4, "expected_accuracy": 0.4, "nll": 0.9479920169717018, "brier": 0.7071188378994288, "expected_brier": 0.7071188378994288, "multiclass_ece": 0.5169733480998355, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.4, "expected_accuracy": 0.4, "expected_risk": 0.6 }, { "threshold": 0.7, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.8, "selected": 3, "coverage": 0.6, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 1, "coverage": 0.2, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.23824386584585291, "brier": 0.08987909722675287, "expected_brier": 0.08987909722675291, "multiclass_ece": 0.2119895011866777, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2562727424592756, "brier": 0.10221451555797535, "expected_brier": 0.10221451555797545, "multiclass_ece": 0.22606914380115584, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4341073233137374, "brier": 0.2481012978606111, "expected_brier": 0.24810129786061114, "multiclass_ece": 0.352097035441548, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6584788041154462, "brier": 0.46536645763787754, "expected_brier": 0.46536645763787765, "multiclass_ece": 0.4822424920174969, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4595590106628233, "brier": 0.2714929778309564, "expected_brier": 0.2714929778309565, "multiclass_ece": 0.3684379037442784, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.274541924895275, "brier": 0.13211898338717784, "expected_brier": 0.13211898338717787, "multiclass_ece": 0.23278737838508634, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3676214365634232, "brier": 0.18948691443086202, "expected_brier": 0.18948691443086207, "multiclass_ece": 0.3074740649400728, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.6193294946796444, "brier": 0.4268555452623285, "expected_brier": 0.4268555452623285, "multiclass_ece": 0.21937587008429385, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5476204225869988, "brier": 0.3556653567082982, "expected_brier": 0.35566535670829824, "multiclass_ece": 0.421604455627721, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.21249004838547547, "brier": 0.0766575522712804, "expected_brier": 0.07665755227128046, "multiclass_ece": 0.19007131987765408, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.4000097694324862, "brier": 0.26095542788653003, "expected_brier": 0.26095542788653, "multiclass_ece": 0.29694644920862034, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.43391328756809616, "brier": 0.24785243851628874, "expected_brier": 0.2478524385162888, "multiclass_ece": 0.35203156003140457, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.21188318099910708, "brier": 0.09039369718992085, "expected_brier": 0.09039369718992085, "multiclass_ece": 0.1839512326754576, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.46137684588118305, "brier": 0.2736268975696541, "expected_brier": 0.2736268975696541, "multiclass_ece": 0.3691630770165346, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 3, "hard_count": 3, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.7166067438779389, "brier": 0.5205654657305907, "expected_brier": 0.5205654657305907, "multiclass_ece": 0.23730534467799416, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4021875370774497, "brier": 0.22446636737803635, "expected_brier": 0.22446636737803635, "multiclass_ece": 0.3269541214751182, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/KILL PROCESS": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.7637495109127286, "brier": 0.5704910323037041, "expected_brier": 0.5704910323037042, "multiclass_ece": 0.5340838100446895, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.27524561654687973, "brier": 0.11875110934597564, "expected_brier": 0.11875110934597566, "multiclass_ece": 0.2391916241628292, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.27907987106023896, "brier": 0.11860448733779394, "expected_brier": 0.11860448733779405, "multiclass_ece": 0.2435205200160696, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.29609796785065434, "brier": 0.1692478935650418, "expected_brier": 0.16924789356504177, "multiclass_ece": 0.23757353412373172, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5244621845736293, "brier": 0.33700060408167043, "expected_brier": 0.3370006040816703, "multiclass_ece": 0.4030086150736406, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.09057772420538801, "brier": 0.014998028688777088, "expected_brier": 0.014998028688777088, "multiclass_ece": 0.086596642810568, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.8989011297936792, "brier": 0.7032584531818407, "expected_brier": 0.7032584531818407, "multiclass_ece": 0.5929833274139504, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } } }, "baseline_calibrated_ood": { "count": 512, "hard_count": 484, "accuracy": 0.6673553719008265, "expected_accuracy": 0.637868923611111, "nll": 0.8644109045490788, "brier": 0.42761710920381696, "expected_brier": 0.46482956580103896, "multiclass_ece": 0.06931489832084904, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 347, "coverage": 0.677734375, "accuracy": 0.7748538011695907, "expected_accuracy": 0.7684918347742554, "expected_risk": 0.23150816522574458 }, { "threshold": 0.7, "selected": 180, "coverage": 0.3515625, "accuracy": 0.888268156424581, "expected_accuracy": 0.8851851851851852, "expected_risk": 0.11481481481481481 }, { "threshold": 0.8, "selected": 113, "coverage": 0.220703125, "accuracy": 0.9642857142857143, "expected_accuracy": 0.9587020648967552, "expected_risk": 0.041297935103244754 }, { "threshold": 0.9, "selected": 76, "coverage": 0.1484375, "accuracy": 0.9736842105263158, "expected_accuracy": 0.9736842105263158, "expected_risk": 0.02631578947368418 }, { "threshold": 0.95, "selected": 42, "coverage": 0.08203125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.037109375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "mean_latency_seconds": 0.06099348470888799, "by_kind": { "choice": { "count": 162, "hard_count": 136, "accuracy": 0.5147058823529411, "expected_accuracy": 0.4501371742112483, "nll": 1.337039684720497, "brier": 0.5373374017427013, "expected_brier": 0.646716688436803, "multiclass_ece": 0.14066701591410513, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 62, "coverage": 0.38271604938271603, "accuracy": 0.6440677966101694, "expected_accuracy": 0.6290322580645161, "expected_risk": 0.3709677419354839 }, { "threshold": 0.7, "selected": 24, "coverage": 0.14814814814814814, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 22, "coverage": 0.13580246913580246, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 18, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.05555555555555555, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "noul": { "count": 243, "hard_count": 243, "accuracy": 0.8106995884773662, "expected_accuracy": 0.8106995884773662, "nll": 0.44796418557911666, "brier": 0.2920260650801552, "expected_brier": 0.2920260650801552, "multiclass_ece": 0.07162787406657194, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 243, "coverage": 1.0, "accuracy": 0.8106995884773662, "expected_accuracy": 0.8106995884773662, "expected_risk": 0.18930041152263377 }, { "threshold": 0.7, "selected": 133, "coverage": 0.5473251028806584, "accuracy": 0.8872180451127819, "expected_accuracy": 0.8872180451127819, "expected_risk": 0.1127819548872181 }, { "threshold": 0.8, "selected": 75, "coverage": 0.30864197530864196, "accuracy": 0.9733333333333334, "expected_accuracy": 0.9733333333333334, "expected_risk": 0.026666666666666616 }, { "threshold": 0.9, "selected": 53, "coverage": 0.21810699588477367, "accuracy": 0.9622641509433962, "expected_accuracy": 0.9622641509433962, "expected_risk": 0.037735849056603765 }, { "threshold": 0.95, "selected": 28, "coverage": 0.11522633744855967, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.07818930041152264, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "score": { "count": 107, "hard_count": 105, "accuracy": 0.5333333333333333, "expected_accuracy": 0.5295950155763239, "nll": 1.0946042720437617, "brier": 0.56942959827625, "expected_brier": 0.5818906574662812, "multiclass_ece": 0.12254775101750316, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 42, "coverage": 0.3925233644859813, "accuracy": 0.75, "expected_accuracy": 0.7301587301587302, "expected_risk": 0.26984126984126977 }, { "threshold": 0.7, "selected": 23, "coverage": 0.21495327102803738, "accuracy": 0.7727272727272727, "expected_accuracy": 0.753623188405797, "expected_risk": 0.246376811594203 }, { "threshold": 0.8, "selected": 16, "coverage": 0.14953271028037382, "accuracy": 0.8666666666666667, "expected_accuracy": 0.8333333333333333, "expected_risk": 0.16666666666666674 }, { "threshold": 0.9, "selected": 5, "coverage": 0.04672897196261682, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 0.04672897196261682, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "ordinal_mae": 0.709066227740182 } }, "by_source": { "customer-control-v1": { "count": 81, "hard_count": 77, "accuracy": 0.8311688311688312, "expected_accuracy": 0.8106995884773661, "nll": 0.37617606707960116, "brier": 0.19115138188851535, "expected_brier": 0.2199579662506552, "multiclass_ece": 0.08401884498760603, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 78, "coverage": 0.9629629629629629, "accuracy": 0.8513513513513513, "expected_accuracy": 0.8290598290598289, "expected_risk": 0.1709401709401711 }, { "threshold": 0.7, "selected": 66, "coverage": 0.8148148148148148, "accuracy": 0.9230769230769231, "expected_accuracy": 0.914141414141414, "expected_risk": 0.08585858585858597 }, { "threshold": 0.8, "selected": 58, "coverage": 0.7160493827160493, "accuracy": 0.9649122807017544, "expected_accuracy": 0.9540229885057471, "expected_risk": 0.04597701149425293 }, { "threshold": 0.9, "selected": 45, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 35, "coverage": 0.43209876543209874, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.2345679012345679, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 81, "hard_count": 81, "accuracy": 0.5679012345679012, "expected_accuracy": 0.5679012345679012, "nll": 1.1029633710109417, "brier": 0.5819492003591643, "expected_brier": 0.5819492003591644, "multiclass_ece": 0.17142472812699902, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 0.3333333333333333, "accuracy": 0.7037037037037037, "expected_accuracy": 0.7037037037037037, "expected_risk": 0.2962962962962963 }, { "threshold": 0.7, "selected": 11, "coverage": 0.13580246913580246, "accuracy": 0.7272727272727273, "expected_accuracy": 0.7272727272727273, "expected_risk": 0.2727272727272727 }, { "threshold": 0.8, "selected": 2, "coverage": 0.024691358024691357, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1": { "count": 80, "hard_count": 80, "accuracy": 0.5625, "expected_accuracy": 0.5625, "nll": 1.0210882008033129, "brier": 0.5763802717504679, "expected_brier": 0.576380271750468, "multiclass_ece": 0.16487920806934858, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 46, "coverage": 0.575, "accuracy": 0.6086956521739131, "expected_accuracy": 0.6086956521739131, "expected_risk": 0.3913043478260869 }, { "threshold": 0.7, "selected": 17, "coverage": 0.2125, "accuracy": 0.7647058823529411, "expected_accuracy": 0.7647058823529411, "expected_risk": 0.23529411764705888 }, { "threshold": 0.8, "selected": 12, "coverage": 0.15, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 7, "coverage": 0.0875, "accuracy": 0.7142857142857143, "expected_accuracy": 0.7142857142857143, "expected_risk": 0.2857142857142857 }, { "threshold": 0.95, "selected": 2, "coverage": 0.025, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1": { "count": 54, "hard_count": 48, "accuracy": 0.5625, "expected_accuracy": 0.5185185185185185, "nll": 0.8516621529728791, "brier": 0.4622240424540098, "expected_brier": 0.5177795980095653, "multiclass_ece": 0.16563956030459756, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 0.5, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.7, "selected": 24, "coverage": 0.4444444444444444, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1": { "count": 81, "hard_count": 81, "accuracy": 0.49382716049382713, "expected_accuracy": 0.49382716049382713, "nll": 1.0093397938793658, "brier": 0.6052507522871464, "expected_brier": 0.6052507522871465, "multiclass_ece": 0.1621857211656454, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 60, "coverage": 0.7407407407407407, "accuracy": 0.55, "expected_accuracy": 0.55, "expected_risk": 0.44999999999999996 }, { "threshold": 0.7, "selected": 3, "coverage": 0.037037037037037035, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1": { "count": 27, "hard_count": 9, "accuracy": 0.2222222222222222, "expected_accuracy": 0.10823045267489712, "nll": 2.738924995140249, "brier": 0.4673862052988379, "expected_brier": 0.9755137773152989, "multiclass_ece": 0.1530230740329492, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.037037037037037035, "accuracy": null, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.3333333333333333, "optimal_set_probability_mass": 0.32625293617620443 }, "workflow-controls-v1/agent_trace_observability": { "count": 27, "hard_count": 27, "accuracy": 0.7777777777777778, "expected_accuracy": 0.7777777777777778, "nll": 0.388957212437285, "brier": 0.2508960948567942, "expected_brier": 0.25089609485679426, "multiclass_ece": 0.19159223936241232, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.7777777777777778, "expected_accuracy": 0.7777777777777778, "expected_risk": 0.2222222222222222 }, { "threshold": 0.7, "selected": 23, "coverage": 0.8518518518518519, "accuracy": 0.8695652173913043, "expected_accuracy": 0.8695652173913043, "expected_risk": 0.13043478260869568 }, { "threshold": 0.8, "selected": 14, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 0.2222222222222222, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.3631713431145638, "brier": 0.20852010973925925, "expected_brier": 0.2085201097392593, "multiclass_ece": 0.2357326544751406, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 11, "coverage": 0.4074074074074074, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 8, "coverage": 0.2962962962962963, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 0.14814814814814814, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.4061707977741641, "brier": 0.24213746735591324, "expected_brier": 0.24213746735591318, "multiclass_ece": 0.27410299766318114, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 0.18518518518518517, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.30035909652504195, "brier": 0.17265204061879386, "expected_brier": 0.17265204061879388, "multiclass_ece": 0.1922011689286163, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 15, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 14, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 11, "coverage": 0.4074074074074074, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.25, "nll": 1.4306531274705012, "brier": 0.9441287792762594, "expected_brier": 0.9441287792762594, "multiclass_ece": 0.5652198076395148, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.875, "accuracy": 0.2857142857142857, "expected_accuracy": 0.2857142857142857, "expected_risk": 0.7142857142857143 }, { "threshold": 0.7, "selected": 5, "coverage": 0.625, "accuracy": 0.2, "expected_accuracy": 0.2, "expected_risk": 0.8 }, { "threshold": 0.8, "selected": 3, "coverage": 0.375, "accuracy": 0.3333333333333333, "expected_accuracy": 0.3333333333333333, "expected_risk": 0.6666666666666667 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/category": { "count": 27, "hard_count": 25, "accuracy": 0.96, "expected_accuracy": 0.9259259259259259, "nll": 0.22153212991484542, "brier": 0.06450893136998503, "expected_brier": 0.1015459684070221, "multiclass_ece": 0.07459580347922881, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 0.96, "expected_risk": 0.040000000000000036 }, { "threshold": 0.7, "selected": 22, "coverage": 0.8148148148148148, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/churn_likelihood_level": { "count": 9, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "nll": 0.5517852694894145, "brier": 0.09326869793984813, "expected_brier": 0.2414168460879963, "multiclass_ece": 0.12347739641893694, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "expected_risk": 0.14814814814814825 }, { "threshold": 0.7, "selected": 8, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 0.9166666666666666, "expected_risk": 0.08333333333333337 }, { "threshold": 0.8, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 0.9047619047619048, "expected_risk": 0.09523809523809523 }, { "threshold": 0.9, "selected": 5, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/frustration": { "count": 10, "hard_count": 10, "accuracy": 0.7, "expected_accuracy": 0.7, "nll": 0.4696252730381919, "brier": 0.30577686230772766, "expected_brier": 0.3057768623077276, "multiclass_ece": 0.27200167761723587, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 0.7, "expected_accuracy": 0.7, "expected_risk": 0.30000000000000004 }, { "threshold": 0.7, "selected": 8, "coverage": 0.8, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.8, "selected": 5, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "customer-control-v1/has_reproducible_steps": { "count": 15, "hard_count": 15, "accuracy": 0.8, "expected_accuracy": 0.8, "nll": 0.1645454154891476, "brier": 0.1238452485074723, "expected_brier": 0.1238452485074723, "multiclass_ece": 0.11304965697603218, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 15, "coverage": 1.0, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.7, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 0.8, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 11, "coverage": 0.7333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 12, "hard_count": 12, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.07609729315491606, "brier": 0.03613540996850612, "expected_brier": 0.036135409968506116, "multiclass_ece": 0.0643388677743417, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 11, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 8, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.15713615438043194, "brier": 0.042289476868721615, "expected_brier": 0.04228947686872164, "multiclass_ece": 0.14541230496199697, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5910297096151154, "brier": 0.8093270451223136, "expected_brier": 0.8093270451223136, "multiclass_ece": 0.24640841004907088, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.25343912616602565, "brier": 0.10023824982844642, "expected_brier": 0.10023824982844642, "multiclass_ece": 0.22387301068736087, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4487216461309, "brier": 0.26144578994054285, "expected_brier": 0.2614457899405429, "multiclass_ece": 0.3615562127391416, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0230259637516665, "brier": 0.820466557201427, "expected_brier": 0.8204665572014269, "multiclass_ece": 0.6404945578228698, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.565964282191571, "brier": 0.7875792300886548, "expected_brier": 0.7875792300886548, "multiclass_ece": 0.25136792181479806, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y11": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.7191318754837774, "brier": 0.7400750800417625, "expected_brier": 0.7400750800417626, "multiclass_ece": 0.8207783328802276, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y4_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.36504935826390833, "brier": 0.18707329058365685, "expected_brier": 0.18707329058365696, "multiclass_ece": 0.3058376126179192, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8243393056118659, "brier": 0.47472006558526025, "expected_brier": 0.47472006558526036, "multiclass_ece": 0.5614753724652484, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9913812064330799, "brier": 0.600795137013345, "expected_brier": 0.6007951370133449, "multiclass_ece": 0.3755175016316305, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3836030558432264, "brier": 0.7127816238074719, "expected_brier": 0.712781623807472, "multiclass_ece": 0.2682574562954671, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.35810807502636177, "brier": 0.18120497780760692, "expected_brier": 0.18120497780760703, "multiclass_ece": 0.30100247325197094, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.645533413132792, "brier": 0.7818469255853528, "expected_brier": 0.7818469255853528, "multiclass_ece": 0.25356264503848874, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.9865953872639242, "brier": 0.47634345846055165, "expected_brier": 0.4763434584605517, "multiclass_ece": 0.6271560782524179, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y10": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4574784643847551, "brier": 0.6567819768847069, "expected_brier": 0.656781976884707, "multiclass_ece": 0.7671773943158309, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x11_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5307532491275229, "brier": 0.7820603279299418, "expected_brier": 0.7820603279299418, "multiclass_ece": 0.26038006060335983, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1177709179634183, "brier": 0.5117823290560667, "expected_brier": 0.5117823290560666, "multiclass_ece": 0.6729920897346061, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y10_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.2530513905704652, "brier": 1.0206435894841714, "expected_brier": 1.0206435894841714, "multiclass_ece": 0.7143681087101283, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.7249436784766013, "brier": 0.4095600230560429, "expected_brier": 0.4095600230560428, "multiclass_ece": 0.5156481523789138, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.0959062065145955, "brier": 0.683499169000035, "expected_brier": 0.6834991690000352, "multiclass_ece": 0.42941609774308936, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7283705114122874, "brier": 0.8012773670223967, "expected_brier": 0.8012773670223967, "multiclass_ece": 0.2688935532550974, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4823850651597383, "brier": 0.7497196562338574, "expected_brier": 0.7497196562338573, "multiclass_ece": 0.7729045953876632, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y7_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9327719481296413, "brier": 0.7357778011733447, "expected_brier": 0.7357778011733447, "multiclass_ece": 0.606538457632055, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y8_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.427413401281249, "brier": 0.7234559819063838, "expected_brier": 0.7234559819063837, "multiclass_ece": 0.7600712803665796, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.2044340834703493, "brier": 0.9803874234105567, "expected_brier": 0.9803874234105567, "multiclass_ece": 0.7001383518314637, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2633071719505589, "brier": 0.6059312219104475, "expected_brier": 0.6059312219104473, "multiclass_ece": 0.7172825166321335, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y10_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.6960838283158604, "brier": 0.8425079538441353, "expected_brier": 0.8425079538441353, "multiclass_ece": 0.24348493560775208, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.4753046369921228, "brier": 1.1897794800933106, "expected_brier": 1.1897794800933106, "multiclass_ece": 0.7712909568033682, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.697848376258325, "brier": 0.8382815587509945, "expected_brier": 0.8382815587509946, "multiclass_ece": 0.2333404403937181, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1017902949653484, "brier": 0.8917115366240813, "expected_brier": 0.8917115366240813, "multiclass_ece": 0.6677243205934921, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8436743879194105, "brier": 0.49975551746309665, "expected_brier": 0.49975551746309677, "multiclass_ece": 0.5698728377798119, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y8": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 1.486662214338097, "brier": 0.6788896488434198, "expected_brier": 0.6788896488434198, "multiclass_ece": 0.24399391488145072, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3525870454065916, "brier": 0.6932950174575082, "expected_brier": 0.6932950174575083, "multiclass_ece": 0.7414295389051204, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2412278593079246, "brier": 0.09188105454504325, "expected_brier": 0.09188105454504325, "multiclass_ece": 0.21433741454193578, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2477025471658996, "brier": 0.09627965484932766, "expected_brier": 0.09627965484932766, "multiclass_ece": 0.21940790191937898, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y6": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5149163327737933, "brier": 0.7005570473847036, "expected_brier": 0.7005570473847036, "multiclass_ece": 0.26521283314354305, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.27894450502016876, "brier": 0.11850475396698759, "expected_brier": 0.11850475396698767, "multiclass_ece": 0.24341811145330539, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0182982750598324, "brier": 0.4618087452562559, "expected_brier": 0.4618087452562558, "multiclass_ece": 0.638790904010005, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y10_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3970074411319915, "brier": 0.6727556135719555, "expected_brier": 0.6727556135719555, "multiclass_ece": 0.7526639748365689, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4199564699148846, "brier": 0.7225058947373295, "expected_brier": 0.7225058947373295, "multiclass_ece": 0.758275461042223, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2308923906302514, "brier": 0.587359849387248, "expected_brier": 0.5873598493872478, "multiclass_ece": 0.7079681451261575, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.1661179111105673, "brier": 0.7492271835106256, "expected_brier": 0.7492271835106257, "multiclass_ece": 0.48265418715902153, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.8415457330933317, "brier": 0.5008634001527822, "expected_brier": 0.5008634001527823, "multiclass_ece": 0.5689562703383642, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9846011493052711, "brier": 0.6045344393980296, "expected_brier": 0.6045344393980296, "multiclass_ece": 0.4024998806711955, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y6_red": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.7841767210379892, "brier": 0.5907945653172187, "expected_brier": 0.5907945653172186, "multiclass_ece": 0.5435046298410063, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.6689491828317677, "brier": 0.8253370607640771, "expected_brier": 0.8253370607640771, "multiclass_ece": 0.22819228143592582, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.124102153661779, "brier": 0.5099533982693013, "expected_brier": 0.5099533982693014, "multiclass_ece": 0.6750559137177368, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.01499528610279, "brier": 0.4760091421223584, "expected_brier": 0.47600914212235845, "multiclass_ece": 0.6375958618356803, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5730871431643838, "brier": 0.7875203141976369, "expected_brier": 0.787520314197637, "multiclass_ece": 0.22061976340907188, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y7": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 1.445893578431114, "brier": 0.6248482219232895, "expected_brier": 0.6248482219232896, "multiclass_ece": 0.41652814782900965, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.5956938446762811, "brier": 0.4028787850622717, "expected_brier": 0.4028787850622717, "multiclass_ece": 0.4488200001460896, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 2.0140933087147967, "brier": 0.8892572307274957, "expected_brier": 0.8892572307274957, "multiclass_ece": 0.3238117907451292, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4095123178697741, "brier": 0.22582697299200508, "expected_brier": 0.22582697299200505, "multiclass_ece": 0.3360260205638881, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4269291130797204, "brier": 0.7269085717291017, "expected_brier": 0.7269085717291017, "multiclass_ece": 0.7599550575780621, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.7704534766338766, "brier": 0.8186165110992346, "expected_brier": 0.8186165110992345, "multiclass_ece": 0.27791505751362017, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.8111967596832792, "brier": 0.8064558799275031, "expected_brier": 0.806455879927503, "multiclass_ece": 0.2862891704549594, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.43190182202236344, "brier": 0.24601869168638574, "expected_brier": 0.2460186916863858, "multiclass_ece": 0.35072688212224745, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3772697643246494, "brier": 0.7007705925699378, "expected_brier": 0.7007705925699379, "multiclass_ece": 0.7477336396888326, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.3357727874640468, "brier": 0.6898828549021601, "expected_brier": 0.6898828549021601, "multiclass_ece": 0.26421009183161637, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y8_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9975624066365107, "brier": 0.7968842558788729, "expected_brier": 0.7968842558788729, "multiclass_ece": 0.6312227245112746, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_lightness": { "count": 2, "hard_count": 2, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.5363246144166043, "brier": 0.7742559156219049, "expected_brier": 0.774255915621905, "multiclass_ece": 0.25457934613709515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4458018978925449, "brier": 0.25875291976730397, "expected_brier": 0.25875291976730397, "multiclass_ece": 0.35968939362129093, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y5_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.46210514787849066, "brier": 0.2738649789448326, "expected_brier": 0.27386497894483264, "multiclass_ece": 0.37004390208786886, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.7385200127778827, "brier": 0.4098827069310851, "expected_brier": 0.4098827069310851, "multiclass_ece": 0.5221794391964899, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.35317001891007604, "brier": 0.17706277854398778, "expected_brier": 0.17706277854398778, "multiclass_ece": 0.29754224787749706, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.443878035181647, "brier": 0.25698189277418837, "expected_brier": 0.2569818927741885, "multiclass_ece": 0.35845633818792244, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6716819198973604, "brier": 0.47853808970685857, "expected_brier": 0.4785380897068585, "multiclass_ece": 0.48915135168312607, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y9_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.9953427556282445, "brier": 0.6209797851910026, "expected_brier": 0.6209797851910026, "multiclass_ece": 0.42670809239519264, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.20999210431792104, "brier": 0.07175180668132852, "expected_brier": 0.07175180668132852, "multiclass_ece": 0.1894093538890418, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2968561588650543, "brier": 0.1319429199889635, "expected_brier": 0.13194291998896346, "multiclass_ece": 0.2568490996567474, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x8_y3": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1235191203378168, "brier": 0.49732094071469746, "expected_brier": 0.49732094071469746, "multiclass_ece": 0.6748664052475015, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.132460349338523, "brier": 0.5149381970578693, "expected_brier": 0.5149381970578693, "multiclass_ece": 0.677760541326198, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4196096487035295, "brier": 0.682927474095206, "expected_brier": 0.682927474095206, "multiclass_ece": 0.7581916113052142, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4620346702120521, "brier": 0.6780891028287305, "expected_brier": 0.6780891028287304, "multiclass_ece": 0.7682357691109705, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y2": { "count": 2, "hard_count": 2, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.583569077860854, "brier": 0.7329595774237219, "expected_brier": 0.732959577423722, "multiclass_ece": 0.27720924030890115, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4976229153041272, "brier": 0.30736853502969114, "expected_brier": 0.3073685350296911, "multiclass_ece": 0.3920258505696347, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.6066994065362894, "brier": 0.7637890699698683, "expected_brier": 0.7637890699698682, "multiclass_ece": 0.2911428758778796, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/decision": { "count": 27, "hard_count": 27, "accuracy": 0.6296296296296297, "expected_accuracy": 0.6296296296296297, "nll": 1.1324383403263523, "brier": 0.606943890857316, "expected_brier": 0.6069438908573161, "multiclass_ece": 0.23895323173030028, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 0.6153846153846154, "expected_accuracy": 0.6153846153846154, "expected_risk": 0.3846153846153846 }, { "threshold": 0.7, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/ordinal": { "count": 26, "hard_count": 26, "accuracy": 0.5384615384615384, "expected_accuracy": 0.5384615384615384, "nll": 1.188862904325455, "brier": 0.5965219638483286, "expected_brier": 0.5965219638483286, "multiclass_ece": 0.22762264569514531, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 0.23076923076923078, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.07692307692307693, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.038461538461538464, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/verification": { "count": 27, "hard_count": 27, "accuracy": 0.5185185185185185, "expected_accuracy": 0.5185185185185185, "nll": 0.7481772356663597, "brier": 0.5264209491419763, "expected_brier": 0.5264209491419765, "multiclass_ece": 0.21553453813873602, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.5185185185185185, "expected_accuracy": 0.5185185185185185, "expected_risk": 0.4814814814814815 }, { "threshold": 0.7, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 0.6923076923076923, "expected_accuracy": 0.6923076923076923, "expected_risk": 0.3076923076923077 }, { "threshold": 0.8, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.9, "selected": 7, "coverage": 0.25925925925925924, "accuracy": 0.7142857142857143, "expected_accuracy": 0.7142857142857143, "expected_risk": 0.2857142857142857 }, { "threshold": 0.95, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/choice": { "count": 27, "hard_count": 21, "accuracy": 0.19047619047619047, "expected_accuracy": 0.18518518518518517, "nll": 1.2370104406569735, "brier": 0.6386965042271409, "expected_brier": 0.7498076153382521, "multiclass_ece": 0.22290907805845328, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "snake-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "nll": 0.4663138652887848, "brier": 0.2857515806808786, "expected_brier": 0.2857515806808786, "multiclass_ece": 0.10837004255074184, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8518518518518519, "expected_accuracy": 0.8518518518518519, "expected_risk": 0.14814814814814814 }, { "threshold": 0.7, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/choice": { "count": 27, "hard_count": 27, "accuracy": 0.25925925925925924, "expected_accuracy": 0.25925925925925924, "nll": 1.2630464207152188, "brier": 0.7906640098972033, "expected_brier": 0.7906640098972033, "multiclass_ece": 0.3401503428284294, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 23, "coverage": 0.8518518518518519, "accuracy": 0.30434782608695654, "expected_accuracy": 0.30434782608695654, "expected_risk": 0.6956521739130435 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.6428980582937186, "brier": 0.4519432145875845, "expected_brier": 0.45194321458758435, "multiclass_ece": 0.1349773202911661, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.7, "selected": 3, "coverage": 0.1111111111111111, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "expected_risk": 0.33333333333333337 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/score": { "count": 27, "hard_count": 27, "accuracy": 0.5555555555555556, "expected_accuracy": 0.5555555555555556, "nll": 1.1220749026291605, "brier": 0.5731450323766519, "expected_brier": 0.5731450323766522, "multiclass_ece": 0.1912630152118416, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "wikispeedia-v1/choice": { "count": 27, "hard_count": 9, "accuracy": 0.2222222222222222, "expected_accuracy": 0.10823045267489712, "nll": 2.738924995140249, "brier": 0.4673862052988379, "expected_brier": 0.9755137773152989, "multiclass_ece": 0.1530230740329492, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 0.037037037037037035, "accuracy": null, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 4, "hard_count": 4, "accuracy": 0.25, "expected_accuracy": 0.25, "nll": 0.7081144699864139, "brier": 0.5462386303842099, "expected_brier": 0.5462386303842099, "multiclass_ece": 0.4571411822743693, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.25, "expected_accuracy": 0.25, "expected_risk": 0.75 }, { "threshold": 0.7, "selected": 2, "coverage": 0.5, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.8, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0936122640225923, "brier": 0.01900383984150196, "expected_brier": 0.019003839841501963, "multiclass_ece": 0.08843766385250489, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.11431065790216396, "brier": 0.02793396103366963, "expected_brier": 0.02793396103366963, "multiclass_ece": 0.10655575401279982, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3423698538839707, "brier": 0.1681008018652152, "expected_brier": 0.16810080186521525, "multiclass_ece": 0.2899144717543566, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 6, "hard_count": 6, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.3509974504730673, "brier": 0.2127029782600611, "expected_brier": 0.2127029782600611, "multiclass_ece": 0.16149393253793184, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 5, "coverage": 0.8333333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.16666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 0.6, "expected_accuracy": 0.6, "nll": 0.6736162228873016, "brier": 0.4839439920500343, "expected_brier": 0.4839439920500343, "multiclass_ece": 0.32485277651518485, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 0.6, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.35421876158742643, "brier": 0.18510348226816667, "expected_brier": 0.1851034822681666, "multiclass_ece": 0.29384495381019865, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.16371940672593366, "brier": 0.06297075425426164, "expected_brier": 0.06297075425426163, "multiclass_ece": 0.14409147643722822, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.21905295763025637, "brier": 0.1180862310967798, "expected_brier": 0.11808623109677978, "multiclass_ece": 0.17810711819082, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4344006681594581, "brier": 0.24891448598621632, "expected_brier": 0.24891448598621632, "multiclass_ece": 0.3518293312064708, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.24636392927232506, "brier": 0.11726897678535918, "expected_brier": 0.11726897678535919, "multiclass_ece": 0.20774241648083522, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.601508043807606, "brier": 0.4113880907338614, "expected_brier": 0.41138809073386134, "multiclass_ece": 0.42174521680269017, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.18910158890048573, "brier": 0.05937291418424165, "expected_brier": 0.05937291418424162, "multiclass_ece": 0.17229758295495856, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.43459404867091406, "brier": 0.2569216082273687, "expected_brier": 0.25692160822736865, "multiclass_ece": 0.3455945022663303, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.42089468138520025, "brier": 0.2379579476409155, "expected_brier": 0.2379579476409155, "multiclass_ece": 0.34205228669236365, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.24574963810877684, "brier": 0.09549234981721137, "expected_brier": 0.09549234981721133, "multiclass_ece": 0.21764107460726403, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.34112948478359295, "brier": 0.17251129300444962, "expected_brier": 0.17251129300444967, "multiclass_ece": 0.2858394315635961, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4681667740923459, "brier": 0.27981116532916345, "expected_brier": 0.27981116532916345, "multiclass_ece": 0.3735777000818641, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6176792882727312, "brier": 0.42474076506553554, "expected_brier": 0.42474076506553554, "multiclass_ece": 0.4606277652251616, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.44580500314264765, "brier": 0.2592302458049517, "expected_brier": 0.2592302458049516, "multiclass_ece": 0.3592732816095793, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2604590426396778, "brier": 0.10515906987451025, "expected_brier": 0.10515906987451018, "multiclass_ece": 0.22930227852608687, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4254178870878693, "brier": 0.24177323876261844, "expected_brier": 0.24177323876261844, "multiclass_ece": 0.3451518950687882, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.965514938678196, "brier": 0.7668493121536542, "expected_brier": 0.7668493121536542, "multiclass_ece": 0.6192129327435169, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6166021063113581, "brier": 0.4236333120244868, "expected_brier": 0.42363331202448673, "multiclass_ece": 0.46016179593879386, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.08127714949906181, "brier": 0.01295256014381893, "expected_brier": 0.012952560143818903, "multiclass_ece": 0.07783185578035268, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/AUTO CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1146938971118772, "brier": 0.024049386643663, "expected_brier": 0.024049386643663, "multiclass_ece": 0.10818253497088752, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.30015560666284985, "brier": 0.18418040523748289, "expected_brier": 0.1841804052374829, "multiclass_ece": 0.2347349742306879, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.20644463115561024, "brier": 0.11342220684487878, "expected_brier": 0.11342220684487875, "multiclass_ece": 0.16519661174496097, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4058245455850529, "brier": 0.2561668126289453, "expected_brier": 0.2561668126289453, "multiclass_ece": 0.3134955888140369, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.3333333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4168960162165471, "brier": 0.23244000484476063, "expected_brier": 0.23244000484476057, "multiclass_ece": 0.34091054900425166, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 2, "hard_count": 2, "accuracy": 0.5, "expected_accuracy": 0.5, "nll": 0.7371509327176, "brier": 0.5424894274217507, "expected_brier": 0.5424894274217507, "multiclass_ece": 0.5095711770060263, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 0.5, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.4388816946186441, "brier": 0.25239510229341966, "expected_brier": 0.25239510229341966, "multiclass_ece": 0.35524294665300515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.2102200992386903, "brier": 0.07189187840256644, "expected_brier": 0.07189187840256639, "multiclass_ece": 0.18959414337284586, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.22095680616874316, "brier": 0.10915184580623619, "expected_brier": 0.10915184580623627, "multiclass_ece": 0.18584948957890268, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3441322239826263, "brier": 0.1695538851982418, "expected_brier": 0.16955388519824188, "multiclass_ece": 0.2911648031598615, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.06813210681385704, "brier": 0.008735986509574615, "expected_brier": 0.008735986509574634, "multiclass_ece": 0.0658456269432861, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.09532285043196342, "brier": 0.01653311449589432, "expected_brier": 0.01653311449589423, "multiclass_ece": 0.090920609588515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.29234979896396834, "brier": 0.1638920257320871, "expected_brier": 0.16389202573208717, "multiclass_ece": 0.2361636344502624, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.48970918316019707, "brier": 0.3037771524304745, "expected_brier": 0.30377715243047454, "multiclass_ece": 0.38291449248985543, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } } }, "trained_test": { "count": 512, "hard_count": 470, "accuracy": 0.951063829787234, "expected_accuracy": 0.9037861173115078, "nll": 0.2730973695486436, "brier": 0.07925829944327434, "expected_brier": 0.13055030713176624, "multiclass_ece": 0.03443108287253511, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 472, "coverage": 0.921875, "accuracy": 0.9712389380530974, "expected_accuracy": 0.9486228813559321, "expected_risk": 0.05137711864406791 }, { "threshold": 0.7, "selected": 447, "coverage": 0.873046875, "accuracy": 0.9773755656108597, "expected_accuracy": 0.9705443698732289, "expected_risk": 0.029455630126771104 }, { "threshold": 0.8, "selected": 442, "coverage": 0.86328125, "accuracy": 0.9817767653758542, "expected_accuracy": 0.9775641025641025, "expected_risk": 0.022435897435897467 }, { "threshold": 0.9, "selected": 437, "coverage": 0.853515625, "accuracy": 0.9839449541284404, "expected_accuracy": 0.9824561403508771, "expected_risk": 0.01754385964912286 }, { "threshold": 0.95, "selected": 428, "coverage": 0.8359375, "accuracy": 0.9906542056074766, "expected_accuracy": 0.9906542056074766, "expected_risk": 0.009345794392523366 }, { "threshold": 0.99, "selected": 415, "coverage": 0.810546875, "accuracy": 0.9951807228915662, "expected_accuracy": 0.9951807228915662, "expected_risk": 0.004819277108433773 } ], "mean_latency_seconds": 0.054687873664079234, "by_kind": { "choice": { "count": 197, "hard_count": 158, "accuracy": 0.8924050632911392, "expected_accuracy": 0.7880126500684876, "nll": 0.6012325504991404, "brier": 0.15791620215322152, "expected_brier": 0.2836091358410793, "multiclass_ece": 0.06548353491264955, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 162, "coverage": 0.8223350253807107, "accuracy": 0.9379310344827586, "expected_accuracy": 0.8842592592592593, "expected_risk": 0.1157407407407407 }, { "threshold": 0.7, "selected": 143, "coverage": 0.7258883248730964, "accuracy": 0.9420289855072463, "expected_accuracy": 0.9219114219114217, "expected_risk": 0.07808857808857828 }, { "threshold": 0.8, "selected": 138, "coverage": 0.700507614213198, "accuracy": 0.9555555555555556, "expected_accuracy": 0.9426328502415457, "expected_risk": 0.05736714975845425 }, { "threshold": 0.9, "selected": 133, "coverage": 0.6751269035532995, "accuracy": 0.9621212121212122, "expected_accuracy": 0.9573934837092731, "expected_risk": 0.04260651629072687 }, { "threshold": 0.95, "selected": 125, "coverage": 0.6345177664974619, "accuracy": 0.984, "expected_accuracy": 0.984, "expected_risk": 0.016000000000000014 }, { "threshold": 0.99, "selected": 118, "coverage": 0.5989847715736041, "accuracy": 0.9915254237288136, "expected_accuracy": 0.9915254237288136, "expected_risk": 0.008474576271186418 } ] }, "noul": { "count": 219, "hard_count": 219, "accuracy": 0.9954337899543378, "expected_accuracy": 0.9954337899543378, "nll": 0.025832979378208012, "brier": 0.010554095080809707, "expected_brier": 0.01055409508080971, "multiclass_ece": 0.005581697774105317, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 219, "coverage": 1.0, "accuracy": 0.9954337899543378, "expected_accuracy": 0.9954337899543378, "expected_risk": 0.004566210045662156 }, { "threshold": 0.7, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.8, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.9, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.95, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.99, "selected": 213, "coverage": 0.9726027397260274, "accuracy": 0.9953051643192489, "expected_accuracy": 0.9953051643192489, "expected_risk": 0.0046948356807511304 } ] }, "score": { "count": 96, "hard_count": 93, "accuracy": 0.946236559139785, "expected_accuracy": 0.9322916666666666, "nll": 0.16380852371611723, "brier": 0.07457719445910939, "expected_brier": 0.09020219445910936, "multiclass_ece": 0.05110353597331116, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 91, "coverage": 0.9479166666666666, "accuracy": 0.9659090909090909, "expected_accuracy": 0.9505494505494505, "expected_risk": 0.0494505494505495 }, { "threshold": 0.7, "selected": 86, "coverage": 0.8958333333333334, "accuracy": 0.9883720930232558, "expected_accuracy": 0.9883720930232558, "expected_risk": 0.011627906976744207 }, { "threshold": 0.8, "selected": 86, "coverage": 0.8958333333333334, "accuracy": 0.9883720930232558, "expected_accuracy": 0.9883720930232558, "expected_risk": 0.011627906976744207 }, { "threshold": 0.9, "selected": 86, "coverage": 0.8958333333333334, "accuracy": 0.9883720930232558, "expected_accuracy": 0.9883720930232558, "expected_risk": 0.011627906976744207 }, { "threshold": 0.95, "selected": 85, "coverage": 0.8854166666666666, "accuracy": 0.9882352941176471, "expected_accuracy": 0.9882352941176471, "expected_risk": 0.0117647058823529 }, { "threshold": 0.99, "selected": 84, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "ordinal_mae": 0.08260350804361621 } }, "by_source": { "customer-control-v1": { "count": 72, "hard_count": 68, "accuracy": 1.0, "expected_accuracy": 0.9722222222222222, "nll": 0.03871482321851678, "brier": 0.00015547581179074397, "expected_brier": 0.027933253589568528, "multiclass_ece": 0.0017043011441966183, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 72, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9722222222222222, "expected_risk": 0.02777777777777779 }, { "threshold": 0.7, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 74, "hard_count": 74, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.492570700436071e-06, "brier": 8.717862937126001e-10, "expected_brier": 8.717863038072366e-10, "multiclass_ece": 5.492351438708799e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1": { "count": 72, "hard_count": 72, "accuracy": 0.8611111111111112, "expected_accuracy": 0.8611111111111112, "nll": 0.4948289808244745, "brier": 0.2535605842911844, "expected_brier": 0.2535605842911844, "multiclass_ece": 0.14944485316983305, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 65, "coverage": 0.9027777777777778, "accuracy": 0.8769230769230769, "expected_accuracy": 0.8769230769230769, "expected_risk": 0.12307692307692308 }, { "threshold": 0.7, "selected": 59, "coverage": 0.8194444444444444, "accuracy": 0.8983050847457628, "expected_accuracy": 0.8983050847457628, "expected_risk": 0.10169491525423724 }, { "threshold": 0.8, "selected": 58, "coverage": 0.8055555555555556, "accuracy": 0.9137931034482759, "expected_accuracy": 0.9137931034482759, "expected_risk": 0.08620689655172409 }, { "threshold": 0.9, "selected": 56, "coverage": 0.7777777777777778, "accuracy": 0.9285714285714286, "expected_accuracy": 0.9285714285714286, "expected_risk": 0.0714285714285714 }, { "threshold": 0.95, "selected": 52, "coverage": 0.7222222222222222, "accuracy": 0.9615384615384616, "expected_accuracy": 0.9615384615384616, "expected_risk": 0.038461538461538436 }, { "threshold": 0.99, "selected": 46, "coverage": 0.6388888888888888, "accuracy": 0.9782608695652174, "expected_accuracy": 0.9782608695652174, "expected_risk": 0.021739130434782594 } ] }, "snake-v1": { "count": 48, "hard_count": 43, "accuracy": 1.0, "expected_accuracy": 0.9409722222222222, "nll": 0.14248087323622258, "brier": 0.024920451580762486, "expected_brier": 0.08394822935854025, "multiclass_ece": 0.024534679009574956, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 48, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9409722222222222, "expected_risk": 0.05902777777777779 }, { "threshold": 0.7, "selected": 43, "coverage": 0.8958333333333334, "accuracy": 1.0, "expected_accuracy": 0.9844961240310078, "expected_risk": 0.015503875968992165 }, { "threshold": 0.8, "selected": 42, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 0.9841269841269842, "expected_risk": 0.015873015873015817 }, { "threshold": 0.9, "selected": 42, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 0.9841269841269842, "expected_risk": 0.015873015873015817 }, { "threshold": 0.95, "selected": 39, "coverage": 0.8125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 35, "coverage": 0.7291666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "tic_tac_toe-v1": { "count": 24, "hard_count": 7, "accuracy": 0.5714285714285714, "expected_accuracy": 0.45734126984126994, "nll": 1.227646432320823, "brier": 0.2654261316416398, "expected_brier": 0.6830848618003701, "multiclass_ece": 0.17273821487896293, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 0.75, "expected_accuracy": 0.5064102564102564, "expected_risk": 0.4935897435897436 }, { "threshold": 0.7, "selected": 6, "coverage": 0.25, "accuracy": 0.5, "expected_accuracy": 0.4166666666666667, "expected_risk": 0.5833333333333333 }, { "threshold": 0.8, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 0.5833333333333334, "expected_risk": 0.41666666666666663 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.33316206807741666, "brier": 0.08311508703088777, "expected_brier": 0.08311508703088778, "multiclass_ece": 0.040854452908501315, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 } ] }, "trex_runner-v1": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 1.3470814357076584, "brier": 0.48953235157002783, "expected_brier": 0.48953235157002783, "multiclass_ece": 0.24548685847570906, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.99, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1": { "count": 73, "hard_count": 73, "accuracy": 0.9726027397260274, "expected_accuracy": 0.9726027397260274, "nll": 0.058789442953600474, "brier": 0.035552196502514705, "expected_brier": 0.0355521965025147, "multiclass_ece": 0.01963309836546598, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 73, "coverage": 1.0, "accuracy": 0.9726027397260274, "expected_accuracy": 0.9726027397260274, "expected_risk": 0.0273972602739726 }, { "threshold": 0.7, "selected": 72, "coverage": 0.9863013698630136, "accuracy": 0.9861111111111112, "expected_accuracy": 0.9861111111111112, "expected_risk": 0.01388888888888884 }, { "threshold": 0.8, "selected": 72, "coverage": 0.9863013698630136, "accuracy": 0.9861111111111112, "expected_accuracy": 0.9861111111111112, "expected_risk": 0.01388888888888884 }, { "threshold": 0.9, "selected": 72, "coverage": 0.9863013698630136, "accuracy": 0.9861111111111112, "expected_accuracy": 0.9861111111111112, "expected_risk": 0.01388888888888884 }, { "threshold": 0.95, "selected": 70, "coverage": 0.958904109589041, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 68, "coverage": 0.9315068493150684, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.2748181216931217, "nll": 1.9763174425179686, "brier": 0.3415867593975574, "expected_brier": 0.8167686377044356, "multiclass_ece": 0.13119423742430122, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.75, "optimal_set_probability_mass": 0.5171147619421269 }, "workflow-controls-v1/agent_trace_observability": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.6604432368658567e-06, "brier": 2.798853479508269e-10, "expected_brier": 2.7988532787700155e-10, "multiclass_ece": 2.660373262871237e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.636907953462622e-06, "brier": 3.4046354342452066e-11, "expected_brier": 3.404634907333559e-11, "multiclass_ece": 1.6368994419183973e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.743358791332781e-06, "brier": 5.50056836723532e-10, "expected_brier": 5.500568334149136e-10, "multiclass_ece": 3.7432212696882536e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents": { "count": 25, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.610670700938362e-06, "brier": 7.0882192081723e-11, "expected_brier": 7.088219788897731e-11, "multiclass_ece": 1.6106529800197223e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.053922938815684e-08, "brier": 9.054050043282134e-15, "expected_brier": 9.048317650695026e-15, "multiclass_ece": 3.053922648099672e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/category": { "count": 24, "hard_count": 23, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "nll": 0.02905251163462311, "brier": 0.0001670041031618374, "expected_brier": 0.02100033743649519, "multiclass_ece": 0.0018635744862966024, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "expected_risk": 0.02083333333333337 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 7, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.0770544960672033e-05, "brier": 1.6709423105377666e-09, "expected_brier": 1.6709422734351165e-09, "multiclass_ece": 2.07700196961369e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/frustration": { "count": 9, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 0.8333333333333334, "nll": 0.23221795626627115, "brier": 0.0007984603674699621, "expected_brier": 0.16746512703413663, "multiclass_ece": 0.008637612613822405, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.7, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/has_reproducible_steps": { "count": 14, "hard_count": 14, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.35004177892205e-06, "brier": 3.5254637350465703e-10, "expected_brier": 3.5254634132643655e-10, "multiclass_ece": 6.3499536397314064e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0847392347894324e-06, "brier": 3.3501976573924298e-12, "expected_brier": 3.3502089991088724e-12, "multiclass_ece": 1.084738397416629e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.918173926186648e-08, "brier": 6.083117148818351e-15, "expected_brier": 5.995204332975845e-15, "multiclass_ece": 5.9181737510627386e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.9159311933116574e-09, "brier": 5.731077754809096e-18, "expected_brier": 0.0, "multiclass_ece": 1.915931191476261e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.468634172032918e-07, "brier": 1.1644347218865593e-13, "expected_brier": 1.163513729807164e-13, "multiclass_ece": 2.4686338673252095e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.877234566591509e-06, "brier": 1.5001527871990365e-11, "expected_brier": 1.500155555334004e-11, "multiclass_ece": 2.8772304273561033e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.251424308759302e-05, "brier": 1.013759459374483e-09, "expected_brier": 1.0137595207737604e-09, "multiclass_ece": 2.2513989643924148e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.6550284237722198e-05, "brier": 5.478147501451263e-10, "expected_brier": 5.478146825055319e-10, "multiclass_ece": 1.6550147282523575e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.575127533611962e-06, "brier": 1.1918690816359916e-11, "expected_brier": 1.1918688258560906e-11, "multiclass_ece": 2.575124217973901e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.107898384454718e-07, "brier": 2.45487738901879e-14, "expected_brier": 2.4646951146678475e-14, "multiclass_ece": 1.1078983230827788e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.8021391062357634e-09, "brier": 5.4080045415545295e-18, "expected_brier": 0.0, "multiclass_ece": 1.8021391046119106e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.547727947003108e-09, "brier": 4.704536372260626e-18, "expected_brier": 0.0, "multiclass_ece": 1.5477279458053772e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.821503450511011e-08, "brier": 9.306581231856558e-15, "expected_brier": 9.325873406851315e-15, "multiclass_ece": 6.82150321784647e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.9238853215855406e-06, "brier": 2.698652521564532e-11, "expected_brier": 2.698663514877353e-11, "multiclass_ece": 3.923877623157601e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.848446236662008e-07, "brier": 5.305770300167661e-13, "expected_brier": 5.306866057708248e-13, "multiclass_ece": 5.848444526446173e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.350177593033553e-07, "brier": 1.6678896388577508e-12, "expected_brier": 1.6678880498943727e-12, "multiclass_ece": 7.350172770426155e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.7113176471422084e-07, "brier": 1.4702482783272726e-13, "expected_brier": 1.4699352846037073e-13, "multiclass_ece": 2.711317279580072e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y2": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.963335420343508e-07, "brier": 2.103880628878185e-13, "expected_brier": 2.1042427060062133e-13, "multiclass_ece": 2.963334824931015e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.163791947906164e-11, "brier": 1.0463844339149544e-20, "expected_brier": 0.0, "multiclass_ece": 9.163791947486288e-11, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.058277167004193e-09, "brier": 2.7474878262046373e-17, "expected_brier": 0.0, "multiclass_ece": 4.0582771587693856e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.7917235689582944e-06, "brier": 1.5095986147039672e-11, "expected_brier": 1.509592451043318e-11, "multiclass_ece": 2.791719672101678e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.137812682002121e-06, "brier": 1.224744556464724e-10, "expected_brier": 1.2247447500612907e-10, "multiclass_ece": 8.137779570094317e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.9157662141698817e-09, "brier": 7.213601517196387e-18, "expected_brier": 0.0, "multiclass_ece": 1.915766212334802e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.001865419740981e-07, "brier": 2.0074684384756815e-14, "expected_brier": 1.9984014443252818e-14, "multiclass_ece": 1.0018653695542667e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.34925864876067e-10, "brier": 1.3767621719257769e-19, "expected_brier": 0.0, "multiclass_ece": 3.349258648199793e-10, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y1": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.361596755321624e-08, "brier": 8.24708147008113e-15, "expected_brier": 8.326672684688674e-15, "multiclass_ece": 6.36159653844004e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.2372220502969308e-07, "brier": 8.16835886127461e-14, "expected_brier": 8.171241461241152e-14, "multiclass_ece": 2.2372218000388244e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y3_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.475843684885198e-06, "brier": 4.5778920395831675e-11, "expected_brier": 4.5778936197393705e-11, "multiclass_ece": 5.475828692480533e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.171862716643285e-08, "brier": 9.166211175247871e-15, "expected_brier": 9.103828801926284e-15, "multiclass_ece": 7.171862459465217e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y4_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.917345970865876e-06, "brier": 3.069107868360856e-11, "expected_brier": 3.069100529273783e-11, "multiclass_ece": 3.917338298076167e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0767878946471381e-07, "brier": 2.3189440905638434e-14, "expected_brier": 2.3092638912203256e-14, "multiclass_ece": 1.0767878366735317e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y5_green": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.325241288651713e-06, "brier": 1.3865839529649377e-10, "expected_brier": 1.3865830705839244e-10, "multiclass_ece": 8.325206623860382e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.014998957018308e-06, "brier": 1.2847938679617248e-10, "expected_brier": 1.2847944930172162e-10, "multiclass_ece": 8.014966836999982e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4149240324015672e-08, "brier": 3.6628939779232616e-16, "expected_brier": 4.440892098500626e-16, "multiclass_ece": 1.4149240223915172e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.049786607653533e-05, "brier": 1.860182938115799e-09, "expected_brier": 1.8601828966069434e-09, "multiclass_ece": 3.049740102134546e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y4": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.338471044763321e-08, "brier": 8.079214925118787e-15, "expected_brier": 7.993605777301127e-15, "multiclass_ece": 7.338470775497541e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y5_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.102990106900271e-06, "brier": 5.2080750293742255e-11, "expected_brier": 5.208078412977102e-11, "multiclass_ece": 5.102977086668403e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.519246214406518e-09, "brier": 1.296684201964795e-16, "expected_brier": 2.220446049250313e-16, "multiclass_ece": 9.519246169098494e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.692714672313729e-09, "brier": 2.6899054551034192e-17, "expected_brier": 0.0, "multiclass_ece": 4.6927146613029436e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.50918409277063e-07, "brier": 1.8084899226862858e-12, "expected_brier": 1.80855330711438e-12, "multiclass_ece": 9.509179571542958e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.762538334419296e-07, "brier": 1.310973260814594e-13, "expected_brier": 1.3100631690576847e-13, "multiclass_ece": 2.7625379528384286e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.2210204791524685e-07, "brier": 3.3015747299698087e-13, "expected_brier": 3.3018032752352156e-13, "multiclass_ece": 4.2210195883018997e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5292191561013552e-07, "brier": 3.635255361056961e-14, "expected_brier": 3.6415315207705135e-14, "multiclass_ece": 1.5292190391757998e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.2930900283155734e-09, "brier": 1.7440697622255248e-17, "expected_brier": 0.0, "multiclass_ece": 3.2930900228933524e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.706344870526846e-08, "brier": 1.0325083958453296e-14, "expected_brier": 1.0436096431476471e-14, "multiclass_ece": 7.706344573588098e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.711847209525222e-08, "brier": 1.7889402505270774e-14, "expected_brier": 1.7985612998927536e-14, "multiclass_ece": 9.711846737925356e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.263868296645768e-06, "brier": 6.594122369610846e-12, "expected_brier": 6.594058632458655e-12, "multiclass_ece": 2.2638657340978696e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.293359478412316e-05, "brier": 7.92077620891923e-09, "expected_brier": 7.920776257108741e-09, "multiclass_ece": 6.293161450698914e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.520629421652325e-07, "brier": 1.4520212776770578e-12, "expected_brier": 1.4519496716047797e-12, "multiclass_ece": 8.52062579159707e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.987234987419246e-10, "brier": 3.147097840253959e-19, "expected_brier": 0.0, "multiclass_ece": 3.987234986624344e-10, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y6_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0970187271903372e-07, "brier": 2.4068999111172743e-14, "expected_brier": 2.398081733190338e-14, "multiclass_ece": 1.097018667017835e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.583228669946892e-07, "brier": 3.402854954661498e-13, "expected_brier": 3.40394379350073e-13, "multiclass_ece": 4.5832276196478006e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.336994550275667e-05, "brier": 3.351365019151282e-10, "expected_brier": 3.351365851500532e-10, "multiclass_ece": 1.3369856125433621e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.562757312265137e-06, "brier": 6.188819356126043e-11, "expected_brier": 6.188827228470473e-11, "multiclass_ece": 5.562741840159369e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.00437417574132e-09, "brier": 1.2392255572903363e-16, "expected_brier": 2.220446049250313e-16, "multiclass_ece": 9.004374135201942e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y1_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.283413484917888e-07, "brier": 1.7236337184363801e-12, "expected_brier": 1.723732268033018e-12, "multiclass_ece": 9.283409175830926e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.49164147691232e-08, "brier": 1.1224937570541336e-14, "expected_brier": 1.1324274851176597e-14, "multiclass_ece": 7.491641196288867e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y2_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001610817153087074, "brier": 5.156143906603846e-08, "expected_brier": 5.1561439118330554e-08, "multiclass_ece": 0.0001610687423457824, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.323114316810423e-08, "brier": 8.731202673936114e-16, "expected_brier": 8.881784197001252e-16, "multiclass_ece": 2.3231142898261226e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y4": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.138259084137785e-07, "brier": 4.0636483608249255e-13, "expected_brier": 4.063416270128073e-13, "multiclass_ece": 4.1382578386262026e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2510767460782994e-07, "brier": 2.1803469597364626e-14, "expected_brier": 2.1760371282653068e-14, "multiclass_ece": 1.2510766678186513e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.725756177920632e-08, "brier": 1.0846974943851679e-14, "expected_brier": 1.0880185641326534e-14, "multiclass_ece": 7.725755879484097e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.658955515121477e-07, "brier": 4.8580536050285065e-14, "expected_brier": 4.8627768478581856e-14, "multiclass_ece": 1.6589553775148147e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5254635782114307e-08, "brier": 4.452062549667058e-16, "expected_brier": 4.440892098500626e-16, "multiclass_ece": 1.525463566576235e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1314815819887755e-05, "brier": 2.5604721693693994e-10, "expected_brier": 2.5604718345562105e-10, "multiclass_ece": 1.1314751807600665e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0266708295807647e-05, "brier": 2.108084341466069e-10, "expected_brier": 2.1080848178200995e-10, "multiclass_ece": 1.0266655593338392e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.493314551456668e-07, "brier": 1.2187164774004804e-13, "expected_brier": 1.219024881038422e-13, "multiclass_ece": 2.4933142406258213e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4819969972470856e-07, "brier": 3.37985676670731e-14, "expected_brier": 3.375077994860476e-14, "multiclass_ece": 1.481996887431336e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.931293187496968e-08, "brier": 8.952789210262808e-15, "expected_brier": 8.992806499463768e-15, "multiclass_ece": 6.931292939604816e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/decision": { "count": 24, "hard_count": 24, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.6807798248011029, "brier": 0.3663666987708292, "expected_brier": 0.3663666987708292, "multiclass_ece": 0.24965200522827982, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 22, "coverage": 0.9166666666666666, "accuracy": 0.8181818181818182, "expected_accuracy": 0.8181818181818182, "expected_risk": 0.18181818181818177 }, { "threshold": 0.7, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 0.7894736842105263, "expected_accuracy": 0.7894736842105263, "expected_risk": 0.21052631578947367 }, { "threshold": 0.8, "selected": 18, "coverage": 0.75, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 0.875, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.95, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/ordinal": { "count": 24, "hard_count": 24, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "nll": 0.5681429431143931, "brier": 0.29800935445254456, "expected_brier": 0.29800935445254456, "multiclass_ece": 0.2012443727995605, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 0.8421052631578947, "expected_accuracy": 0.8421052631578947, "expected_risk": 0.1578947368421053 }, { "threshold": 0.7, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.9411764705882353, "expected_accuracy": 0.9411764705882353, "expected_risk": 0.05882352941176472 }, { "threshold": 0.8, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.9411764705882353, "expected_accuracy": 0.9411764705882353, "expected_risk": 0.05882352941176472 }, { "threshold": 0.9, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.9411764705882353, "expected_accuracy": 0.9411764705882353, "expected_risk": 0.05882352941176472 }, { "threshold": 0.95, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 0.9375, "expected_accuracy": 0.9375, "expected_risk": 0.0625 }, { "threshold": 0.99, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/verification": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.23556417455792777, "brier": 0.09630569965017922, "expected_brier": 0.09630569965017921, "multiclass_ece": 0.05109464981561951, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.8, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.9, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.95, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.99, "selected": 18, "coverage": 0.75, "accuracy": 0.9444444444444444, "expected_accuracy": 0.9444444444444444, "expected_risk": 0.05555555555555558 } ] }, "snake-v1/choice": { "count": 24, "hard_count": 19, "accuracy": 1.0, "expected_accuracy": 0.8819444444444445, "nll": 0.2848515676056521, "brier": 0.0498404916940409, "expected_brier": 0.1678960472495965, "multiclass_ece": 0.04895928216988906, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.8819444444444445, "expected_risk": 0.11805555555555547 }, { "threshold": 0.7, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 1.0, "expected_accuracy": 0.9649122807017543, "expected_risk": 0.03508771929824572 }, { "threshold": 0.8, "selected": 18, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.9, "selected": 18, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.95, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 11, "coverage": 0.4583333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/noul": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00011017886679311858, "brier": 4.114674840568323e-07, "expected_brier": 4.1146748404793004e-07, "multiclass_ece": 0.00011007584926107465, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "tic_tac_toe-v1/choice": { "count": 24, "hard_count": 7, "accuracy": 0.5714285714285714, "expected_accuracy": 0.45734126984126994, "nll": 1.227646432320823, "brier": 0.2654261316416398, "expected_brier": 0.6830848618003701, "multiclass_ece": 0.17273821487896293, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 0.75, "expected_accuracy": 0.5064102564102564, "expected_risk": 0.4935897435897436 }, { "threshold": 0.7, "selected": 6, "coverage": 0.25, "accuracy": 0.5, "expected_accuracy": 0.4166666666666667, "expected_risk": 0.5833333333333333 }, { "threshold": 0.8, "selected": 3, "coverage": 0.125, "accuracy": 1.0, "expected_accuracy": 0.5833333333333334, "expected_risk": 0.41666666666666663 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.33316206807741666, "brier": 0.08311508703088777, "expected_brier": 0.08311508703088778, "multiclass_ece": 0.040854452908501315, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 } ] }, "trex_runner-v1/choice": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 1.3470814357076584, "brier": 0.48953235157002783, "expected_brier": 0.48953235157002783, "multiclass_ece": 0.24548685847570906, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.99, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.9166666666666666, "expected_accuracy": 0.9166666666666666, "nll": 0.1787856855634765, "brier": 0.10813792593815595, "expected_brier": 0.10813792593815592, "multiclass_ece": 0.059749543000504834, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9166666666666666, "expected_accuracy": 0.9166666666666666, "expected_risk": 0.08333333333333337 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.8, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.9, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.95, "selected": 21, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/noul": { "count": 25, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.8314118577052413e-05, "brier": 4.641253859624695e-09, "expected_brier": 4.641253852355476e-09, "multiclass_ece": 2.831295818808144e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/score": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.709546873797099e-06, "brier": 2.5568683485176984e-10, "expected_brier": 2.556868420692619e-10, "multiclass_ece": 2.7094740996010578e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1/choice": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.2748181216931217, "nll": 1.9763174425179686, "brier": 0.3415867593975574, "expected_brier": 0.8167686377044356, "multiclass_ece": 0.13119423742430122, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.08333333333333333, "accuracy": null, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0699141160613043e-08, "brier": 2.760351117372856e-16, "expected_brier": 2.9605947323337506e-16, "multiclass_ece": 1.06991411286117e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3433393605095495e-08, "brier": 5.337028042345648e-16, "expected_brier": 5.181040781584064e-16, "multiclass_ece": 1.3433393508677227e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.886238424836902e-08, "brier": 6.389012273926509e-15, "expected_brier": 6.394884621840902e-15, "multiclass_ece": 4.8862382584502484e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.8611691873714713e-08, "brier": 1.763834409705725e-15, "expected_brier": 1.7763568394002505e-15, "multiclass_ece": 2.861169146051168e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.3834137725480524e-08, "brier": 1.1361322176851874e-15, "expected_brier": 1.1102230246251565e-15, "multiclass_ece": 2.3834137441447467e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0818684233542534e-06, "brier": 4.5836882397257415e-12, "expected_brier": 4.583622370546436e-12, "multiclass_ece": 1.0818672773638127e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.9328768379083334e-05, "brier": 2.231429114625414e-09, "expected_brier": 2.23142904154372e-09, "multiclass_ece": 1.9328210500250503e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.4978211949723125e-09, "brier": 9.982895981825486e-17, "expected_brier": 1.1102230246251565e-16, "multiclass_ece": 6.497821170015072e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.174203457582625e-06, "brier": 2.5509191734471437e-10, "expected_brier": 2.5509194756523357e-10, "multiclass_ece": 9.1741396840798e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.08047934386313e-07, "brier": 8.008302629434366e-13, "expected_brier": 8.008408750962796e-13, "multiclass_ece": 5.080477342526516e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2661598823057273e-08, "brier": 3.6246749191804233e-16, "expected_brier": 3.7007434154171886e-16, "multiclass_ece": 1.2661598732440401e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.918770773000505e-07, "brier": 3.071351671042779e-13, "expected_brier": 3.070876886113183e-13, "multiclass_ece": 3.9187700051623864e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0127929478325632e-08, "brier": 2.2911033410908517e-16, "expected_brier": 2.220446049250313e-16, "multiclass_ece": 1.0127929310025744e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.0227550645238102e-06, "brier": 1.1722906037461334e-11, "expected_brier": 1.1722900428168259e-11, "multiclass_ece": 2.0227521336790844e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.349550289013446e-07, "brier": 4.4670553750578847e-13, "expected_brier": 4.4666492726719296e-13, "multiclass_ece": 3.349549171360877e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.8477997144049938e-08, "brier": 1.6219925963426019e-15, "expected_brier": 1.5543122344752192e-15, "multiclass_ece": 2.847799673855178e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.806600007714239e-08, "brier": 2.8980406132627432e-15, "expected_brier": 2.886579864025407e-15, "multiclass_ece": 3.806599935263222e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.893762757717962e-08, "brier": 1.3115816247660096e-14, "expected_brier": 1.3156142841808105e-14, "multiclass_ece": 7.893762432598095e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.580466077549156e-07, "brier": 1.750666323312231e-12, "expected_brier": 1.750655176380178e-12, "multiclass_ece": 8.58046170115756e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.683350385275331e-08, "brier": 6.460093947472967e-15, "expected_brier": 6.439293542825908e-15, "multiclass_ece": 5.683350223772976e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1137157775913341e-07, "brier": 6.358974043360258e-14, "expected_brier": 6.357877187686729e-14, "multiclass_ece": 1.113715618987025e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.1821234878807347e-06, "brier": 1.8592093130297297e-11, "expected_brier": 1.859212783728026e-11, "multiclass_ece": 2.182118839844094e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.0312978468423667e-05, "brier": 3.289230705035709e-09, "expected_brier": 3.2892307211263017e-09, "multiclass_ece": 2.0312156116197855e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.917333996539348e-09, "brier": 4.837985742175725e-17, "expected_brier": 0.0, "multiclass_ece": 4.917333984444383e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.593149428902215e-09, "brier": 5.014746202488685e-17, "expected_brier": 0.0, "multiclass_ece": 4.59314941636535e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.0338936296951154e-07, "brier": 2.0601049441290714e-13, "expected_brier": 2.0605739337042905e-13, "multiclass_ece": 2.033893114668691e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.7276452246949293e-08, "brier": 1.488009651120715e-15, "expected_brier": 1.5543122344752192e-15, "multiclass_ece": 2.7276451874946872e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5603515060769607e-08, "brier": 5.447659387711167e-16, "expected_brier": 5.551115123125783e-16, "multiclass_ece": 1.560351492457812e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.8133491355893436e-07, "brier": 2.0416580871220932e-13, "expected_brier": 2.041700142285663e-13, "multiclass_ece": 2.813348625174683e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0024754991556228e-05, "brier": 5.508157820228131e-10, "expected_brier": 5.508157633708303e-10, "multiclass_ece": 1.0024617285016765e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.033597850330522e-07, "brier": 1.0962813433196218e-13, "expected_brier": 1.0961601996465713e-13, "multiclass_ece": 2.0335975762601302e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/KILL PROCESS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.30777713661013e-07, "brier": 3.420561632950097e-14, "expected_brier": 3.419486915845482e-14, "multiclass_ece": 1.3077770510960818e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.483988562277004e-08, "brier": 9.508613536176174e-15, "expected_brier": 9.547918011776346e-15, "multiclass_ece": 5.4839883301127657e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.840550504746181e-09, "brier": 6.822405975820593e-17, "expected_brier": 0.0, "multiclass_ece": 5.840550487690166e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.171703589492515e-08, "brier": 8.957360067412488e-15, "expected_brier": 8.992806499463768e-15, "multiclass_ece": 5.1717033655585e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3297408674962752e-07, "brier": 4.266413783371902e-14, "expected_brier": 4.263256414560601e-14, "multiclass_ece": 1.329740760835918e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.5651221365207874e-08, "brier": 1.3162344658891465e-15, "expected_brier": 1.3322676295501878e-15, "multiclass_ece": 2.5651221036149252e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.684119071865498e-06, "brier": 1.1809046439725262e-10, "expected_brier": 1.1809042632648925e-10, "multiclass_ece": 7.684089549098161e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } } }, "trained_ood": { "count": 512, "hard_count": 484, "accuracy": 0.9235537190082644, "expected_accuracy": 0.8863878038194444, "nll": 0.41376637414783385, "brier": 0.11775550082256626, "expected_brier": 0.15496795741978844, "multiclass_ece": 0.0485584951471779, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 473, "coverage": 0.923828125, "accuracy": 0.944206008583691, "expected_accuracy": 0.9369978858350951, "expected_risk": 0.06300211416490487 }, { "threshold": 0.7, "selected": 458, "coverage": 0.89453125, "accuracy": 0.9519650655021834, "expected_accuracy": 0.9519650655021834, "expected_risk": 0.048034934497816595 }, { "threshold": 0.8, "selected": 453, "coverage": 0.884765625, "accuracy": 0.9558498896247241, "expected_accuracy": 0.9558498896247241, "expected_risk": 0.04415011037527594 }, { "threshold": 0.9, "selected": 448, "coverage": 0.875, "accuracy": 0.9598214285714286, "expected_accuracy": 0.9598214285714286, "expected_risk": 0.0401785714285714 }, { "threshold": 0.95, "selected": 435, "coverage": 0.849609375, "accuracy": 0.9701149425287356, "expected_accuracy": 0.9701149425287356, "expected_risk": 0.029885057471264354 }, { "threshold": 0.99, "selected": 409, "coverage": 0.798828125, "accuracy": 0.9779951100244498, "expected_accuracy": 0.9779951100244498, "expected_risk": 0.022004889975550168 } ], "mean_latency_seconds": 0.053288100301870145, "by_kind": { "choice": { "count": 162, "hard_count": 136, "accuracy": 0.8823529411764706, "expected_accuracy": 0.7787894375857338, "nll": 0.739576306233558, "brier": 0.17904855906453362, "expected_brier": 0.2884278457586351, "multiclass_ece": 0.0765935544552265, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 132, "coverage": 0.8148148148148148, "accuracy": 0.912, "expected_accuracy": 0.8878787878787879, "expected_risk": 0.11212121212121207 }, { "threshold": 0.7, "selected": 121, "coverage": 0.7469135802469136, "accuracy": 0.9256198347107438, "expected_accuracy": 0.9256198347107438, "expected_risk": 0.07438016528925617 }, { "threshold": 0.8, "selected": 119, "coverage": 0.7345679012345679, "accuracy": 0.9327731092436975, "expected_accuracy": 0.9327731092436975, "expected_risk": 0.0672268907563025 }, { "threshold": 0.9, "selected": 116, "coverage": 0.7160493827160493, "accuracy": 0.9396551724137931, "expected_accuracy": 0.9396551724137931, "expected_risk": 0.06034482758620685 }, { "threshold": 0.95, "selected": 110, "coverage": 0.6790123456790124, "accuracy": 0.9545454545454546, "expected_accuracy": 0.9545454545454546, "expected_risk": 0.045454545454545414 }, { "threshold": 0.99, "selected": 97, "coverage": 0.5987654320987654, "accuracy": 0.9587628865979382, "expected_accuracy": 0.9587628865979382, "expected_risk": 0.04123711340206182 } ] }, "noul": { "count": 243, "hard_count": 243, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.17358515833051552, "brier": 0.06325290305419451, "expected_brier": 0.06325290305419454, "multiclass_ece": 0.030571138213166996, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 243, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 239, "coverage": 0.9835390946502057, "accuracy": 0.9707112970711297, "expected_accuracy": 0.9707112970711297, "expected_risk": 0.029288702928870314 }, { "threshold": 0.8, "selected": 237, "coverage": 0.9753086419753086, "accuracy": 0.9704641350210971, "expected_accuracy": 0.9704641350210971, "expected_risk": 0.029535864978902926 }, { "threshold": 0.9, "selected": 235, "coverage": 0.9670781893004116, "accuracy": 0.9744680851063829, "expected_accuracy": 0.9744680851063829, "expected_risk": 0.025531914893617058 }, { "threshold": 0.95, "selected": 231, "coverage": 0.9506172839506173, "accuracy": 0.9826839826839827, "expected_accuracy": 0.9826839826839827, "expected_risk": 0.017316017316017285 }, { "threshold": 0.99, "selected": 225, "coverage": 0.9259259259259259, "accuracy": 0.9866666666666667, "expected_accuracy": 0.9866666666666667, "expected_risk": 0.013333333333333308 } ] }, "score": { "count": 107, "hard_count": 105, "accuracy": 0.8857142857142857, "expected_accuracy": 0.8753894080996886, "nll": 0.4659423222386861, "brier": 0.1487335926217778, "expected_brier": 0.16119465181180892, "multiclass_ece": 0.06717181070319621, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 98, "coverage": 0.9158878504672897, "accuracy": 0.9387755102040817, "expected_accuracy": 0.9387755102040817, "expected_risk": 0.061224489795918324 }, { "threshold": 0.7, "selected": 98, "coverage": 0.9158878504672897, "accuracy": 0.9387755102040817, "expected_accuracy": 0.9387755102040817, "expected_risk": 0.061224489795918324 }, { "threshold": 0.8, "selected": 97, "coverage": 0.9065420560747663, "accuracy": 0.9484536082474226, "expected_accuracy": 0.9484536082474226, "expected_risk": 0.05154639175257736 }, { "threshold": 0.9, "selected": 97, "coverage": 0.9065420560747663, "accuracy": 0.9484536082474226, "expected_accuracy": 0.9484536082474226, "expected_risk": 0.05154639175257736 }, { "threshold": 0.95, "selected": 94, "coverage": 0.8785046728971962, "accuracy": 0.9574468085106383, "expected_accuracy": 0.9574468085106383, "expected_risk": 0.04255319148936165 }, { "threshold": 0.99, "selected": 87, "coverage": 0.8130841121495327, "accuracy": 0.9770114942528736, "expected_accuracy": 0.9770114942528736, "expected_risk": 0.02298850574712641 } ], "ordinal_mae": 0.1588138994218155 } }, "by_source": { "customer-control-v1": { "count": 81, "hard_count": 77, "accuracy": 1.0, "expected_accuracy": 0.97119341563786, "nll": 0.046679777977167713, "brier": 0.0005666128432830484, "expected_brier": 0.029373197205422995, "multiclass_ece": 0.0038442056969190635, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 79, "coverage": 0.9753086419753086, "accuracy": 1.0, "expected_accuracy": 0.9873417721518988, "expected_risk": 0.012658227848101222 }, { "threshold": 0.7, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 76, "coverage": 0.9382716049382716, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 74, "coverage": 0.9135802469135802, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 81, "hard_count": 81, "accuracy": 0.9135802469135802, "expected_accuracy": 0.9135802469135802, "nll": 0.5376333937954092, "brier": 0.14443370842818698, "expected_brier": 0.14443370842818695, "multiclass_ece": 0.07799424748036997, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.925, "expected_accuracy": 0.925, "expected_risk": 0.07499999999999996 }, { "threshold": 0.7, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.925, "expected_accuracy": 0.925, "expected_risk": 0.07499999999999996 }, { "threshold": 0.8, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.925, "expected_accuracy": 0.925, "expected_risk": 0.07499999999999996 }, { "threshold": 0.9, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 0.935064935064935, "expected_accuracy": 0.935064935064935, "expected_risk": 0.06493506493506496 }, { "threshold": 0.95, "selected": 75, "coverage": 0.9259259259259259, "accuracy": 0.96, "expected_accuracy": 0.96, "expected_risk": 0.040000000000000036 }, { "threshold": 0.99, "selected": 69, "coverage": 0.8518518518518519, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 } ] }, "reasoning-control-v1": { "count": 80, "hard_count": 80, "accuracy": 0.7, "expected_accuracy": 0.7, "nll": 1.0326296702752802, "brier": 0.4280734912958815, "expected_brier": 0.42807349129588135, "multiclass_ece": 0.1750081381365492, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 71, "coverage": 0.8875, "accuracy": 0.7746478873239436, "expected_accuracy": 0.7746478873239436, "expected_risk": 0.22535211267605637 }, { "threshold": 0.7, "selected": 64, "coverage": 0.8, "accuracy": 0.796875, "expected_accuracy": 0.796875, "expected_risk": 0.203125 }, { "threshold": 0.8, "selected": 60, "coverage": 0.75, "accuracy": 0.8166666666666667, "expected_accuracy": 0.8166666666666667, "expected_risk": 0.18333333333333335 }, { "threshold": 0.9, "selected": 58, "coverage": 0.725, "accuracy": 0.8275862068965517, "expected_accuracy": 0.8275862068965517, "expected_risk": 0.1724137931034483 }, { "threshold": 0.95, "selected": 52, "coverage": 0.65, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8461538461538461, "expected_risk": 0.15384615384615385 }, { "threshold": 0.99, "selected": 42, "coverage": 0.525, "accuracy": 0.9047619047619048, "expected_accuracy": 0.9047619047619048, "expected_risk": 0.09523809523809523 } ] }, "snake-v1": { "count": 54, "hard_count": 48, "accuracy": 1.0, "expected_accuracy": 0.9444444444444444, "nll": 0.08315215143011277, "brier": 0.000919640877546131, "expected_brier": 0.05647519643310167, "multiclass_ece": 0.007763070010934502, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 52, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 0.9615384615384616, "expected_risk": 0.038461538461538436 }, { "threshold": 0.7, "selected": 48, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 48, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 48, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 46, "coverage": 0.8518518518518519, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 40, "coverage": 0.7407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1": { "count": 81, "hard_count": 81, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.11687943881078713, "brier": 0.057356696025997016, "expected_brier": 0.05735669602599702, "multiclass_ece": 0.030034055597346756, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 81, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.975, "expected_accuracy": 0.975, "expected_risk": 0.025000000000000022 }, { "threshold": 0.8, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.975, "expected_accuracy": 0.975, "expected_risk": 0.025000000000000022 }, { "threshold": 0.9, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.975, "expected_accuracy": 0.975, "expected_risk": 0.025000000000000022 }, { "threshold": 0.95, "selected": 78, "coverage": 0.9629629629629629, "accuracy": 0.9871794871794872, "expected_accuracy": 0.9871794871794872, "expected_risk": 0.012820512820512775 }, { "threshold": 0.99, "selected": 76, "coverage": 0.9382716049382716, "accuracy": 0.9868421052631579, "expected_accuracy": 0.9868421052631579, "expected_risk": 0.013157894736842146 } ] }, "wikispeedia-v1": { "count": 27, "hard_count": 9, "accuracy": 0.7777777777777778, "expected_accuracy": 0.3394032921810699, "nll": 2.0190641052354965, "brier": 0.28164310911904916, "expected_brier": 0.7897706811355097, "multiclass_ece": 0.19853193997242854, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.7037037037037037, "optimal_set_probability_mass": 0.49622877371794544 }, "workflow-controls-v1/agent_trace_observability": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2164406804409496e-06, "brier": 5.764009407233811e-11, "expected_brier": 5.7640098455530514e-11, "multiclass_ece": 1.2164262701975094e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.49764458804024797, "brier": 0.07407385781090632, "expected_brier": 0.07407385781090632, "multiclass_ece": 0.037033341358026814, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 } ] }, "workflow-controls-v1/invoice_processing": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.508209599959256e-07, "brier": 9.343289251395451e-12, "expected_brier": 9.343291572526545e-12, "multiclass_ece": 5.508186244237887e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.656548406219587e-07, "brier": 3.1023699958946197e-12, "expected_brier": 3.102382548389768e-12, "multiclass_ece": 3.6565406518278635e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.011311440237569373, "brier": 0.0008766577107120299, "expected_brier": 0.0008766577107120754, "multiclass_ece": 0.011058442812166502, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 7, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/category": { "count": 27, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "nll": 0.05311457504118664, "brier": 0.00038443677761936315, "expected_brier": 0.0374214738146564, "multiclass_ece": 0.004577130578462343, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 9, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "nll": 0.24990372389089155, "brier": 0.003160869777844638, "expected_brier": 0.1513090179259928, "multiclass_ece": 0.010222319394735693, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/frustration": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006803449676447941, "brier": 5.462525862661702e-06, "expected_brier": 5.462525862665046e-06, "multiclass_ece": 0.0006789748433131848, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/has_reproducible_steps": { "count": 15, "hard_count": 15, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3901148579390944e-05, "brier": 1.617201463019869e-09, "expected_brier": 1.6172014986418995e-09, "multiclass_ece": 1.3900744261145329e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 12, "hard_count": 12, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.76238512356898e-05, "brier": 9.008938128220348e-09, "expected_brier": 9.008938132530156e-09, "multiclass_ece": 2.7621598745882398e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.5078929024952865e-07, "brier": 4.064217852071974e-13, "expected_brier": 4.063416270128073e-13, "multiclass_ece": 4.507891886440518e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.119205879524776e-07, "brier": 1.1016014461479925e-12, "expected_brier": 1.1015632850330803e-12, "multiclass_ece": 9.119201721530246e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.876822751185171e-06, "brier": 7.0449140565283296e-12, "expected_brier": 7.044809180456468e-12, "multiclass_ece": 1.876820989954453e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.04895474864288717, "brier": 0.004565050660514162, "expected_brier": 0.004565050660514158, "multiclass_ece": 0.0477757818382607, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00011618005081336594, "brier": 2.6992472275400434e-08, "expected_brier": 2.69924722573478e-08, "multiclass_ece": 0.00011617330217261745, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.216218820783946e-07, "brier": 1.9437843453480744e-14, "expected_brier": 1.9539925233402755e-14, "multiclass_ece": 1.2162187468245378e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y11": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00020446700701429735, "brier": 8.277669000875963e-08, "expected_brier": 8.277668994693954e-08, "multiclass_ece": 0.0002044461050604296, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y4_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5934060441578229e-07, "brier": 5.0778848360240746e-14, "expected_brier": 5.084821452783217e-14, "multiclass_ece": 1.5934059172106885e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.148363407806514e-07, "brier": 6.272814353827526e-13, "expected_brier": 6.272760089132134e-13, "multiclass_ece": 6.148361517688272e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 15.94126858470617, "brier": 1.999999480025334, "expected_brier": 1.999999480025334, "multiclass_ece": 0.999999859355119, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 } ] }, "painting-geometry-v1/x10_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.031037139344275093, "brier": 0.0014373915850006706, "expected_brier": 0.0014373915850007801, "multiclass_ece": 0.03056043194244329, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003557140558177481, "brier": 2.5297497891859905e-07, "expected_brier": 2.52974978964815e-07, "multiclass_ece": 0.000355650797073892, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.826165125938474e-05, "brier": 4.86857951281636e-10, "expected_brier": 4.868578873384877e-10, "multiclass_ece": 1.826148451644638e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0040366509315366e-07, "brier": 1.2959404965616878e-14, "expected_brier": 1.2878587085651816e-14, "multiclass_ece": 1.0040366005270585e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y10": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.641137065845264e-05, "brier": 6.114853423110933e-09, "expected_brier": 6.114853512428908e-09, "multiclass_ece": 5.640977956700155e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.813544905350692e-05, "brier": 6.692558726676578e-09, "expected_brier": 6.692558729426423e-09, "multiclass_ece": 5.813375922103514e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.4189747585583734e-06, "brier": 1.7013022340128387e-11, "expected_brier": 1.70130576293559e-11, "multiclass_ece": 3.4189689138708346e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y10_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001681910817183755, "brier": 5.6566965213387893e-08, "expected_brier": 5.656696511202597e-08, "multiclass_ece": 0.00016817693839132897, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.9190039317198767e-09, "brier": 1.3526203302659686e-17, "expected_brier": 0.0, "multiclass_ece": 2.9190039274595847e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1557361244221853e-06, "brier": 2.4365618773943592e-12, "expected_brier": 2.4364954498423685e-12, "multiclass_ece": 1.155735456559448e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003016720776327575, "brier": 1.3698945550432177e-07, "expected_brier": 1.369894555569573e-07, "multiclass_ece": 0.0003016265791868644, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.4811358026451375e-06, "brier": 4.153394573105652e-11, "expected_brier": 4.1533887440436956e-11, "multiclass_ece": 5.481120781247739e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y7_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.823119840924391e-05, "brier": 1.2239283284087886e-08, "expected_brier": 1.2239283364579023e-08, "multiclass_ece": 7.822813842883747e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y8_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.793942765781961e-07, "brier": 1.914976540768052e-13, "expected_brier": 1.91402449445377e-13, "multiclass_ece": 3.793942046081966e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001635128318968077, "brier": 5.346414971967472e-08, "expected_brier": 5.346414977402958e-08, "multiclass_ece": 0.00016349946440230578, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001681850701522221, "brier": 4.220017486278243e-08, "expected_brier": 4.220017491007866e-08, "multiclass_ece": 0.00016817092783616427, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y10_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.4398913527067756e-05, "brier": 8.821074417627216e-10, "expected_brier": 8.8210749815687e-10, "multiclass_ece": 2.4398615875997898e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00019390187548005118, "brier": 7.518129565729668e-08, "expected_brier": 7.518129563344189e-08, "multiclass_ece": 0.0001938830777263867, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.670058950315769e-06, "brier": 3.011792680072616e-11, "expected_brier": 3.011790816742632e-11, "multiclass_ece": 4.6700480456074445e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00021729605857176282, "brier": 9.44146363557026e-08, "expected_brier": 9.441463633130809e-08, "multiclass_ece": 0.00021727245149316676, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.9220236125944077e-06, "brier": 1.3350284039341279e-11, "expected_brier": 1.3350209826512582e-11, "multiclass_ece": 2.9220193434875696e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y8": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00013465733115758914, "brier": 3.141453410165696e-08, "expected_brier": 3.141453408517236e-08, "multiclass_ece": 0.00013464717488820277, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.971261732260981e-07, "brier": 3.611249379760809e-13, "expected_brier": 3.610445276081009e-13, "multiclass_ece": 4.971260496589025e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.42459748147439e-07, "brier": 5.885248374840961e-13, "expected_brier": 5.88418203051333e-13, "multiclass_ece": 5.424596010161764e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2405920323639147e-06, "brier": 3.0781333629302457e-12, "expected_brier": 3.078204358075709e-12, "multiclass_ece": 1.2405912628299376e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0012423293325627273, "brier": 3.0356156703317427e-06, "expected_brier": 3.0356156703881254e-06, "multiclass_ece": 0.0012415579609430516, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.826839449237871e-08, "brier": 1.5582617561643157e-14, "expected_brier": 1.554312234475219e-14, "multiclass_ece": 8.826839059672409e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.944754698456472e-07, "brier": 1.0187930997541521e-13, "expected_brier": 1.0191847366058937e-13, "multiclass_ece": 2.9447542648775027e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y10_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5480313152291246e-05, "brier": 3.7950267675028094e-10, "expected_brier": 3.795026515263089e-10, "multiclass_ece": 1.548019333286188e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1140023682377553e-06, "brier": 1.8847854498533333e-12, "expected_brier": 1.8847146066036657e-12, "multiclass_ece": 1.1140017477373476e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00018993032108852135, "brier": 5.3371000351036045e-08, "expected_brier": 5.337100028590669e-08, "multiclass_ece": 0.00018991228546694217, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.6446634575905706e-06, "brier": 1.391799723575796e-11, "expected_brier": 1.3917977881305887e-11, "multiclass_ece": 2.6446599604712517e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.236223547511031e-08, "brier": 4.370790940230439e-15, "expected_brier": 4.440892098500626e-15, "multiclass_ece": 5.236223410420848e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.780013053475859e-07, "brier": 8.593266330540402e-13, "expected_brier": 8.593126210598712e-13, "multiclass_ece": 6.780010755047527e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.19588964651407537, "brier": 0.06329471965948424, "expected_brier": 0.06329471965948419, "multiclass_ece": 0.17789704840087184, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.659227461913563e-06, "brier": 1.1820182842626425e-11, "expected_brier": 1.1820100453974192e-11, "multiclass_ece": 2.65922392617135e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 5.571527887372376, "brier": 1.8203271592961097, "expected_brier": 1.8203271592961094, "multiclass_ece": 0.9054994543024808, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002897143128577451, "brier": 1.5325922889922706e-05, "expected_brier": 1.53259228898861e-05, "multiclass_ece": 0.00289295045932203, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.757495594458492e-06, "brier": 1.0652657947884425e-11, "expected_brier": 1.0652589921278377e-11, "multiclass_ece": 2.7574917925710096e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.02183483371222088, "brier": 0.0010997614006628801, "expected_brier": 0.0010997614006628131, "multiclass_ece": 0.02147014521705959, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y8": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 2.1056759339112125, "brier": 1.5425992532432373, "expected_brier": 1.542599253243237, "multiclass_ece": 0.878236657525532, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1595939993998328, "brier": 0.03278073187361775, "expected_brier": 0.03278073187361774, "multiclass_ece": 0.14751016990233945, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.6847857302260876e-05, "brier": 5.676910269132238e-10, "expected_brier": 5.676910053153961e-10, "multiclass_ece": 1.6847715377910077e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.4289035642674465e-08, "brier": 2.655705230900874e-15, "expected_brier": 2.6645352591003757e-15, "multiclass_ece": 4.428903466191514e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023444637824068546, "brier": 8.230206122896778e-08, "expected_brier": 8.23020611662173e-08, "multiclass_ece": 0.00023441889783615277, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y3": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0022454868592847736, "brier": 9.961528364191603e-06, "expected_brier": 9.96152836418851e-06, "multiclass_ece": 0.0022429676396448217, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.6771299549445847e-05, "brier": 1.433366585622654e-09, "expected_brier": 1.433366536573999e-09, "multiclass_ece": 2.6770941201403886e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.8156361577447567, "brier": 0.5572236872023694, "expected_brier": 0.5572236872023693, "multiclass_ece": 0.4934599344978995, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.02142133015117876, "brier": 0.0007139845493668291, "expected_brier": 0.0007139845493668862, "multiclass_ece": 0.021193523002092096, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y8_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 2.504751358457233, "brier": 1.6865648090896266, "expected_brier": 1.6865648090896266, "multiclass_ece": 0.9183040915431082, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004315352939658608, "brier": 3.1650635811990783e-07, "expected_brier": 3.1650635812230377e-07, "multiclass_ece": 0.00043144219038304676, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.886593310524489e-05, "brier": 7.118334343544417e-10, "expected_brier": 7.118334810485294e-10, "multiclass_ece": 1.8865755144648055e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 6.439297646819743, "brier": 1.9936149909303706, "expected_brier": 1.9936149909303706, "multiclass_ece": 0.9984024716842328, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 } ] }, "painting-geometry-v1/x7_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.722215932677745e-05, "brier": 7.769511776390303e-09, "expected_brier": 7.769511700672638e-09, "multiclass_ece": 6.721989996805178e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y7": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.02816927163153563, "brier": 0.0015430364654405982, "expected_brier": 0.001543036465440517, "multiclass_ece": 0.027776217034007677, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.660193922719021e-07, "brier": 8.871630708842889e-13, "expected_brier": 8.870681966755001e-13, "multiclass_ece": 6.660191704810359e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.012980201438938152, "brier": 0.0003326302391979389, "expected_brier": 0.00033263023919793966, "multiclass_ece": 0.012896321940730648, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y9_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.8388495294146934e-05, "brier": 5.155440075495728e-10, "expected_brier": 5.155440518933574e-10, "multiclass_ece": 1.8388326226803642e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3233205074248314e-05, "brier": 3.5023079837324907e-10, "expected_brier": 3.5023073330364696e-10, "multiclass_ece": 1.3233117515776271e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4092782822315451e-06, "brier": 3.972124955769804e-12, "expected_brier": 3.972155937503885e-12, "multiclass_ece": 1.4092772891993732e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y3": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 9.613794632069327, "brier": 1.9918744978339407, "expected_brier": 1.9918744978339404, "multiclass_ece": 0.9959882644853626, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 } ] }, "painting-geometry-v1/x9_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.928253750035621e-06, "brier": 2.1954996758449223e-11, "expected_brier": 2.195510440117232e-11, "multiclass_ece": 3.928246034456961e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.765219843672118e-05, "brier": 6.641188447112913e-09, "expected_brier": 6.641188488032412e-09, "multiclass_ece": 6.764991007834897e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002405999874521805, "brier": 8.624484018150356e-08, "expected_brier": 8.624484015840039e-08, "multiclass_ece": 0.00024057104559638276, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.148026756164107e-05, "brier": 2.214736015899997e-09, "expected_brier": 2.2147360612123634e-09, "multiclass_ece": 3.147953121152014e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0038346214899421614, "brier": 2.9296124763458054e-05, "expected_brier": 2.9296124763433795e-05, "multiclass_ece": 0.0038272787175392953, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00014688587206009184, "brier": 2.9318007836440417e-08, "expected_brier": 2.9318007799261636e-08, "multiclass_ece": 0.00014687508485855538, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/decision": { "count": 27, "hard_count": 27, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 1.2790675627106833, "brier": 0.47591032807402706, "expected_brier": 0.4759103280740272, "multiclass_ece": 0.2271682186834142, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 0.7619047619047619, "expected_accuracy": 0.7619047619047619, "expected_risk": 0.23809523809523814 }, { "threshold": 0.8, "selected": 20, "coverage": 0.7407407407407407, "accuracy": 0.8, "expected_accuracy": 0.8, "expected_risk": 0.19999999999999996 }, { "threshold": 0.9, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.95, "selected": 17, "coverage": 0.6296296296296297, "accuracy": 0.8235294117647058, "expected_accuracy": 0.8235294117647058, "expected_risk": 0.17647058823529416 }, { "threshold": 0.99, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8461538461538461, "expected_risk": 0.15384615384615385 } ] }, "reasoning-control-v1/ordinal": { "count": 26, "hard_count": 26, "accuracy": 0.6153846153846154, "expected_accuracy": 0.6153846153846154, "nll": 1.1807295128579929, "brier": 0.5122924424290822, "expected_brier": 0.5122924424290822, "multiclass_ece": 0.22657255184936617, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 20, "coverage": 0.7692307692307693, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 20, "coverage": 0.7692307692307693, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 19, "coverage": 0.7307692307692307, "accuracy": 0.7894736842105263, "expected_accuracy": 0.7894736842105263, "expected_risk": 0.21052631578947367 }, { "threshold": 0.9, "selected": 19, "coverage": 0.7307692307692307, "accuracy": 0.7894736842105263, "expected_accuracy": 0.7894736842105263, "expected_risk": 0.21052631578947367 }, { "threshold": 0.95, "selected": 17, "coverage": 0.6538461538461539, "accuracy": 0.8235294117647058, "expected_accuracy": 0.8235294117647058, "expected_risk": 0.17647058823529416 }, { "threshold": 0.99, "selected": 13, "coverage": 0.5, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 } ] }, "reasoning-control-v1/verification": { "count": 27, "hard_count": 27, "accuracy": 0.8148148148148148, "expected_accuracy": 0.8148148148148148, "nll": 0.643577114612078, "brier": 0.29913692379687556, "expected_brier": 0.2991369237968755, "multiclass_ece": 0.13962427800727661, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8148148148148148, "expected_accuracy": 0.8148148148148148, "expected_risk": 0.18518518518518523 }, { "threshold": 0.7, "selected": 23, "coverage": 0.8518518518518519, "accuracy": 0.8695652173913043, "expected_accuracy": 0.8695652173913043, "expected_risk": 0.13043478260869568 }, { "threshold": 0.8, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 0.8571428571428571, "expected_accuracy": 0.8571428571428571, "expected_risk": 0.1428571428571429 }, { "threshold": 0.9, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 0.8571428571428571, "expected_accuracy": 0.8571428571428571, "expected_risk": 0.1428571428571429 }, { "threshold": 0.95, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "expected_risk": 0.11111111111111116 }, { "threshold": 0.99, "selected": 16, "coverage": 0.5925925925925926, "accuracy": 0.9375, "expected_accuracy": 0.9375, "expected_risk": 0.0625 } ] }, "snake-v1/choice": { "count": 27, "hard_count": 21, "accuracy": 1.0, "expected_accuracy": 0.8888888888888888, "nll": 0.16536586185727659, "brier": 0.001794152343911241, "expected_brier": 0.1129052634550223, "multiclass_ece": 0.014599170525165881, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 0.92, "expected_risk": 0.07999999999999996 }, { "threshold": 0.7, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 19, "coverage": 0.7037037037037037, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 14, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009384410029490114, "brier": 4.5129411181020995e-05, "expected_brier": 4.512941118101282e-05, "multiclass_ece": 0.0009269694967033859, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/choice": { "count": 27, "hard_count": 27, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "nll": 0.35058813421823526, "brier": 0.1720700381098475, "expected_brier": 0.17207003810984756, "multiclass_ece": 0.09015233650817549, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "expected_risk": 0.11111111111111116 }, { "threshold": 0.7, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 }, { "threshold": 0.8, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 }, { "threshold": 0.9, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 }, { "threshold": 0.95, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.99, "selected": 22, "coverage": 0.8148148148148148, "accuracy": 0.9545454545454546, "expected_accuracy": 0.9545454545454546, "expected_risk": 0.045454545454545414 } ] }, "vizdoom-basic-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.9755389280047854e-05, "brier": 4.996570752946496e-08, "expected_brier": 4.99657075220878e-08, "multiclass_ece": 4.974289191724042e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/score": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.268248461427553e-07, "brier": 2.435966227193354e-12, "expected_brier": 2.4359691218899536e-12, "multiclass_ece": 4.268242171882001e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1/choice": { "count": 27, "hard_count": 9, "accuracy": 0.7777777777777778, "expected_accuracy": 0.3394032921810699, "nll": 2.0190641052354965, "brier": 0.28164310911904916, "expected_brier": 0.7897706811355097, "multiclass_ece": 0.19853193997242854, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 0.6, "expected_risk": 0.4 }, { "threshold": 0.7, "selected": 1, "coverage": 0.037037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.9107376454684505e-08, "brier": 1.8918642929989888e-15, "expected_brier": 1.942890293094024e-15, "multiclass_ece": 2.9107375953962844e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.107152798548123e-09, "brier": 1.384761967000171e-16, "expected_brier": 1.1102230246251565e-16, "multiclass_ece": 8.10715272692164e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.304861568634711e-08, "brier": 2.256100641968908e-15, "expected_brier": 2.3314683517128287e-15, "multiclass_ece": 3.3048615177833085e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.7235728083588695e-08, "brier": 2.7729987886964004e-15, "expected_brier": 2.6645352591003757e-15, "multiclass_ece": 3.723572739033898e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.8085894735260475e-06, "brier": 2.57662138832615e-10, "expected_brier": 2.576621138672408e-10, "multiclass_ece": 4.808525056798629e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.3908449133814395e-07, "brier": 5.038079089735359e-13, "expected_brier": 5.03819208574896e-13, "multiclass_ece": 3.3908436538609974e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.761778973307758e-07, "brier": 2.591661299382156e-12, "expected_brier": 2.5917046286849654e-12, "multiclass_ece": 6.761772494516075e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.659654373391006e-06, "brier": 1.7316737828143852e-10, "expected_brier": 1.731674248439674e-10, "multiclass_ece": 4.659611080981563e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0614547801700459e-06, "brier": 6.585187729422625e-12, "expected_brier": 6.5851768482616535e-12, "multiclass_ece": 1.0614531338326572e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.20297159970749e-06, "brier": 6.680056353258489e-10, "expected_brier": 6.680056507946119e-10, "multiclass_ece": 8.20280459379319e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.469936859705904e-06, "brier": 4.297646973134232e-10, "expected_brier": 4.297647423120073e-10, "multiclass_ece": 8.469829416786942e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 3.359076621955294, "brier": 0.49999853887870044, "expected_brier": 0.4999985388787004, "multiclass_ece": 0.24999940114682317, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.561813386822088e-09, "brier": 1.82856549551928e-16, "expected_brier": 2.220446049250313e-16, "multiclass_ece": 9.56181334110795e-09, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5790655815860897e-06, "brier": 8.927943034970154e-12, "expected_brier": 8.92789545995735e-12, "multiclass_ece": 1.579063349558633e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.0987141369727974e-06, "brier": 2.784421021529479e-12, "expected_brier": 2.7843838346086613e-12, "multiclass_ece": 1.0987134408946275e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.9192828521744066e-08, "brier": 2.3558197906914927e-15, "expected_brier": 2.3684757858670005e-15, "multiclass_ece": 2.9192828006803968e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5849521324425603e-07, "brier": 7.798812510782433e-14, "expected_brier": 7.80486786311485e-14, "multiclass_ece": 1.584951937472212e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.16640043152062e-08, "brier": 3.826916778327341e-15, "expected_brier": 3.848773152033876e-15, "multiclass_ece": 4.1664003358476975e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.104731278364772e-07, "brier": 1.1431865882773799e-12, "expected_brier": 1.143233655890678e-12, "multiclass_ece": 7.104728420396592e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.5224989014785135e-07, "brier": 5.634342184139471e-14, "expected_brier": 5.631051180898794e-14, "multiclass_ece": 1.5224987599538052e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.1948813787631974e-08, "brier": 2.855482991862668e-16, "expected_brier": 2.220446049250313e-16, "multiclass_ece": 1.19488137162449e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.842403749351865e-08, "brier": 7.908967963411054e-16, "expected_brier": 8.141635513917814e-16, "multiclass_ece": 1.8424037258787013e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.6247324417895256e-07, "brier": 5.279510158003547e-14, "expected_brier": 5.284661597215745e-14, "multiclass_ece": 1.6247323098017574e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.577356073442529e-06, "brier": 1.2416115237139536e-10, "expected_brier": 1.241611258251396e-10, "multiclass_ece": 5.577325032923852e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.6382460827435e-08, "brier": 1.3125926640703647e-15, "expected_brier": 1.27675647831893e-15, "multiclass_ece": 1.6382460499286822e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/AUTO CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.631019704433829e-08, "brier": 1.4254608353903336e-15, "expected_brier": 1.4802973661668755e-15, "multiclass_ece": 2.6310196687973075e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.2199658681269646e-08, "brier": 2.5210260968470616e-15, "expected_brier": 2.55351295663786e-15, "multiclass_ece": 3.21996580510131e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.761744096014677e-08, "brier": 1.9683686540073033e-14, "expected_brier": 1.965094753586527e-14, "multiclass_ece": 6.76174360947357e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.70887953289038e-08, "brier": 4.135540685826251e-15, "expected_brier": 4.144832625267251e-15, "multiclass_ece": 3.708879425801115e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2834637177348595e-07, "brier": 3.294557805383443e-14, "expected_brier": 3.2862601528904634e-14, "multiclass_ece": 1.2834636353709072e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.04833589332437e-06, "brier": 4.134911835103844e-11, "expected_brier": 4.134914632913933e-11, "multiclass_ece": 4.048325556005317e-06, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.892303090073466e-07, "brier": 7.161620611514804e-14, "expected_brier": 7.172040739078511e-14, "multiclass_ece": 1.8923029110329281e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.654495132372643e-08, "brier": 5.474708193573685e-16, "expected_brier": 4.440892098500626e-16, "multiclass_ece": 1.6544951186858725e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.9292123573100545e-08, "brier": 2.272587514937402e-15, "expected_brier": 2.220446049250313e-15, "multiclass_ece": 2.92921229494425e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.803696791843797e-08, "brier": 1.5721431010329512e-15, "expected_brier": 1.5543122344752192e-15, "multiclass_ece": 2.803696752540219e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.3819704143372842e-08, "brier": 3.8232921562724e-16, "expected_brier": 4.440892098500626e-16, "multiclass_ece": 1.3819704047790538e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.4362499071053218e-08, "brier": 4.1256275438478635e-16, "expected_brier": 4.440892098500626e-16, "multiclass_ece": 1.436249896791253e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.6965348344357244e-08, "brier": 5.37611959154391e-15, "expected_brier": 5.440092820663267e-15, "multiclass_ece": 4.696534694481613e-08, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.471337003027899e-07, "brier": 4.210686753361163e-13, "expected_brier": 4.211075932403219e-13, "multiclass_ece": 3.471335949800647e-07, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } } }, "calibrated_test": { "count": 512, "hard_count": 470, "accuracy": 0.951063829787234, "expected_accuracy": 0.9037861173115078, "nll": 0.25439166797949847, "brier": 0.07411491388101164, "expected_brier": 0.12540692156950367, "multiclass_ece": 0.02946644382007577, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 460, "coverage": 0.8984375, "accuracy": 0.9754464285714286, "expected_accuracy": 0.9615942028985507, "expected_risk": 0.03840579710144931 }, { "threshold": 0.7, "selected": 437, "coverage": 0.853515625, "accuracy": 0.9862385321100917, "expected_accuracy": 0.9847444698703279, "expected_risk": 0.015255530129672068 }, { "threshold": 0.8, "selected": 428, "coverage": 0.8359375, "accuracy": 0.9906542056074766, "expected_accuracy": 0.9906542056074766, "expected_risk": 0.009345794392523366 }, { "threshold": 0.9, "selected": 417, "coverage": 0.814453125, "accuracy": 0.9952038369304557, "expected_accuracy": 0.9952038369304557, "expected_risk": 0.004796163069544335 }, { "threshold": 0.95, "selected": 403, "coverage": 0.787109375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 345, "coverage": 0.673828125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "mean_latency_seconds": 0.054687873664079234, "by_kind": { "choice": { "count": 197, "hard_count": 158, "accuracy": 0.8924050632911392, "expected_accuracy": 0.7880126500684876, "nll": 0.5654593450864857, "brier": 0.1474463797969612, "expected_brier": 0.27313931348481874, "multiclass_ece": 0.06644238960899163, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 153, "coverage": 0.7766497461928934, "accuracy": 0.9370629370629371, "expected_accuracy": 0.9041394335511983, "expected_risk": 0.0958605664488017 }, { "threshold": 0.7, "selected": 133, "coverage": 0.6751269035532995, "accuracy": 0.9696969696969697, "expected_accuracy": 0.9649122807017543, "expected_risk": 0.03508771929824572 }, { "threshold": 0.8, "selected": 125, "coverage": 0.6345177664974619, "accuracy": 0.984, "expected_accuracy": 0.984, "expected_risk": 0.016000000000000014 }, { "threshold": 0.9, "selected": 119, "coverage": 0.6040609137055838, "accuracy": 0.9915966386554622, "expected_accuracy": 0.9915966386554622, "expected_risk": 0.008403361344537785 }, { "threshold": 0.95, "selected": 109, "coverage": 0.5532994923857868, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 63, "coverage": 0.3197969543147208, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "noul": { "count": 219, "hard_count": 219, "accuracy": 0.9954337899543378, "expected_accuracy": 0.9954337899543378, "nll": 0.020836502060672318, "brier": 0.010779856491247792, "expected_brier": 0.010779856491247799, "multiclass_ece": 0.009265782003989597, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 219, "coverage": 1.0, "accuracy": 0.9954337899543378, "expected_accuracy": 0.9954337899543378, "expected_risk": 0.004566210045662156 }, { "threshold": 0.7, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.8, "selected": 218, "coverage": 0.9954337899543378, "accuracy": 0.9954128440366973, "expected_accuracy": 0.9954128440366973, "expected_risk": 0.004587155963302725 }, { "threshold": 0.9, "selected": 214, "coverage": 0.9771689497716894, "accuracy": 0.9953271028037384, "expected_accuracy": 0.9953271028037384, "expected_risk": 0.004672897196261627 }, { "threshold": 0.95, "selected": 211, "coverage": 0.9634703196347032, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 203, "coverage": 0.9269406392694064, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "score": { "count": 96, "hard_count": 93, "accuracy": 0.946236559139785, "expected_accuracy": 0.9322916666666666, "nll": 0.14885259450185698, "brier": 0.06811573453638868, "expected_brier": 0.0837407345363887, "multiclass_ece": 0.03765362558052509, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 88, "coverage": 0.9166666666666666, "accuracy": 0.9883720930232558, "expected_accuracy": 0.9772727272727273, "expected_risk": 0.022727272727272707 }, { "threshold": 0.7, "selected": 86, "coverage": 0.8958333333333334, "accuracy": 0.9883720930232558, "expected_accuracy": 0.9883720930232558, "expected_risk": 0.011627906976744207 }, { "threshold": 0.8, "selected": 85, "coverage": 0.8854166666666666, "accuracy": 0.9882352941176471, "expected_accuracy": 0.9882352941176471, "expected_risk": 0.0117647058823529 }, { "threshold": 0.9, "selected": 84, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 83, "coverage": 0.8645833333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 79, "coverage": 0.8229166666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ], "ordinal_mae": 0.07979127946568991 } }, "by_source": { "customer-control-v1": { "count": 72, "hard_count": 68, "accuracy": 1.0, "expected_accuracy": 0.9722222222222222, "nll": 0.04075872989412672, "brier": 7.873111478653201e-05, "expected_brier": 0.02785650889256431, "multiclass_ece": 0.0019168309138481998, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 71, "coverage": 0.9861111111111112, "accuracy": 1.0, "expected_accuracy": 0.9788732394366197, "expected_risk": 0.021126760563380254 }, { "threshold": 0.7, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 68, "coverage": 0.9444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 66, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 74, "hard_count": 74, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0010655767573010857, "brier": 6.86719761811915e-06, "expected_brier": 6.867197618101053e-06, "multiclass_ece": 0.001063696733741854, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 74, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 73, "coverage": 0.9864864864864865, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1": { "count": 72, "hard_count": 72, "accuracy": 0.8611111111111112, "expected_accuracy": 0.8611111111111112, "nll": 0.4078731506046455, "brier": 0.22903773861889276, "expected_brier": 0.22903773861889276, "multiclass_ece": 0.10352094477721414, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 62, "coverage": 0.8611111111111112, "accuracy": 0.9032258064516129, "expected_accuracy": 0.9032258064516129, "expected_risk": 0.09677419354838712 }, { "threshold": 0.7, "selected": 55, "coverage": 0.7638888888888888, "accuracy": 0.9454545454545454, "expected_accuracy": 0.9454545454545454, "expected_risk": 0.054545454545454564 }, { "threshold": 0.8, "selected": 52, "coverage": 0.7222222222222222, "accuracy": 0.9615384615384616, "expected_accuracy": 0.9615384615384616, "expected_risk": 0.038461538461538436 }, { "threshold": 0.9, "selected": 47, "coverage": 0.6527777777777778, "accuracy": 0.9787234042553191, "expected_accuracy": 0.9787234042553191, "expected_risk": 0.021276595744680882 }, { "threshold": 0.95, "selected": 43, "coverage": 0.5972222222222222, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 32, "coverage": 0.4444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1": { "count": 48, "hard_count": 43, "accuracy": 1.0, "expected_accuracy": 0.9409722222222222, "nll": 0.15061360543500893, "brier": 0.027062705328359176, "expected_brier": 0.08609048310613694, "multiclass_ece": 0.05391567675702954, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 46, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 0.963768115942029, "expected_risk": 0.03623188405797095 }, { "threshold": 0.7, "selected": 42, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 0.9841269841269842, "expected_risk": 0.015873015873015817 }, { "threshold": 0.8, "selected": 39, "coverage": 0.8125, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 36, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 33, "coverage": 0.6875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 28, "coverage": 0.5833333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "tic_tac_toe-v1": { "count": 24, "hard_count": 7, "accuracy": 0.5714285714285714, "expected_accuracy": 0.45734126984126994, "nll": 1.1849234218485212, "brier": 0.23991144494057084, "expected_brier": 0.6575701750993009, "multiclass_ece": 0.12109375036088056, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 0.375, "accuracy": 0.6666666666666666, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.21459948988092156, "brier": 0.07806250145934908, "expected_brier": 0.07806250145934908, "multiclass_ece": 0.07399205733604802, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.95, "selected": 21, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.7668877262100418, "brier": 0.40590170973174894, "expected_brier": 0.40590170973174894, "multiclass_ece": 0.2272462205045263, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1": { "count": 73, "hard_count": 73, "accuracy": 0.9726027397260274, "expected_accuracy": 0.9726027397260274, "nll": 0.057337284791964614, "brier": 0.029548781575421092, "expected_brier": 0.029548781575421103, "multiclass_ece": 0.03069570110491719, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 73, "coverage": 1.0, "accuracy": 0.9726027397260274, "expected_accuracy": 0.9726027397260274, "expected_risk": 0.0273972602739726 }, { "threshold": 0.7, "selected": 72, "coverage": 0.9863013698630136, "accuracy": 0.9861111111111112, "expected_accuracy": 0.9861111111111112, "expected_risk": 0.01388888888888884 }, { "threshold": 0.8, "selected": 70, "coverage": 0.958904109589041, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 68, "coverage": 0.9315068493150684, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 64, "coverage": 0.8767123287671232, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 49, "coverage": 0.6712328767123288, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.2748181216931217, "nll": 2.0728827643161636, "brier": 0.36410946204707334, "expected_brier": 0.8392913403539518, "multiclass_ece": 0.11058666428069612, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.75, "optimal_set_probability_mass": 0.4380175730014135 }, "workflow-controls-v1/agent_trace_observability": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00044206319110479315, "brier": 3.0438318724083507e-06, "expected_brier": 3.043831872393913e-06, "multiclass_ece": 0.00044129947251514423, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0005452763618080866, "brier": 1.7151649719625157e-06, "expected_brier": 1.715164971965999e-06, "multiclass_ece": 0.0005448469244521507, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0005503521038192851, "brier": 4.3470174075025245e-06, "expected_brier": 4.347017407489255e-06, "multiclass_ece": 0.00054926065265537, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents": { "count": 25, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000436773966449034, "brier": 1.7526746171215538e-06, "expected_brier": 1.7526746171103725e-06, "multiclass_ece": 0.0004363348110274501, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.502059416206712e-05, "brier": 3.574327712074298e-08, "expected_brier": 3.5743277088107206e-08, "multiclass_ece": 8.500901337449296e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/category": { "count": 24, "hard_count": 23, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "nll": 0.029463046115659017, "brier": 4.818004953681634e-05, "expected_brier": 0.02088151338287016, "multiclass_ece": 0.0010718929231993228, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9791666666666666, "expected_risk": 0.02083333333333337 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 7, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0036833994939611783, "brier": 3.304033550411628e-05, "expected_brier": 3.304033550412454e-05, "multiclass_ece": 0.003672561210451976, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 7, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/frustration": { "count": 9, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 0.8333333333333334, "nll": 0.2414806601241707, "brier": 0.0004637444413206932, "expected_brier": 0.16713041110798738, "multiclass_ece": 0.006466630484817326, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 0.875, "expected_risk": 0.125 }, { "threshold": 0.7, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 0.4444444444444444, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/has_reproducible_steps": { "count": 14, "hard_count": 14, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0014986500497953382, "brier": 6.903891399796249e-06, "expected_brier": 6.903891399789863e-06, "multiclass_ece": 0.0014969199049065399, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 14, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006744442575611427, "brier": 1.0396329517545642e-06, "expected_brier": 1.0396329517936565e-06, "multiclass_ece": 0.0006741842028095135, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00022671381994201014, "brier": 7.471266364421162e-08, "expected_brier": 7.471266361136486e-08, "multiclass_ece": 0.00022668812230597357, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.46486324125133e-05, "brier": 1.8224727035546962e-09, "expected_brier": 1.8224726172633154e-09, "multiclass_ece": 3.464803215558199e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004914572507299034, "brier": 3.4790509229769373e-07, "expected_brier": 3.479050922905458e-07, "multiclass_ece": 0.0004913365053964558, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.001519717159773939, "brier": 3.714752898883982e-06, "expected_brier": 3.7147528988423772e-06, "multiclass_ece": 0.001518562974403559, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y2_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.003541409253987781, "brier": 2.4994512505271618e-05, "expected_brier": 2.4994512505260857e-05, "multiclass_ece": 0.003535145860164124, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0030118467630921185, "brier": 1.8087895471708318e-05, "expected_brier": 1.8087895471641247e-05, "multiclass_ece": 0.003007315702724611, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0014434899029626784, "brier": 3.33271007024985e-06, "expected_brier": 3.3327100703584023e-06, "multiclass_ece": 0.0014424485725229585, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00021537934972870343, "brier": 9.275654894099997e-08, "expected_brier": 9.27565488861859e-08, "multiclass_ece": 0.00021535615726164803, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.2514461710362374e-05, "brier": 1.6440011953636095e-09, "expected_brier": 1.6440011574303526e-09, "multiclass_ece": 3.2513933120981164e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.5205285248813577e-05, "brier": 1.1468345661511898e-09, "expected_brier": 1.1468346272636154e-09, "multiclass_ece": 2.520496759828017e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00016679445242167486, "brier": 5.563149904698514e-08, "expected_brier": 5.5631498963037984e-08, "multiclass_ece": 0.0001667805430003444, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0018479471226817436, "brier": 5.3526650860305705e-06, "expected_brier": 5.352665086055808e-06, "multiclass_ece": 0.001846240719673653, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009820593500033498, "brier": 1.2186815146907053e-06, "expected_brier": 1.2186815145920349e-06, "multiclass_ece": 0.0009815772875374495, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006714726281854714, "brier": 9.589826839585516e-07, "expected_brier": 9.589826839295768e-07, "multiclass_ece": 0.0006711634966861668, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00034520540649101626, "brier": 2.3825128787792087e-07, "expected_brier": 2.3825128780963212e-07, "multiclass_ece": 0.00034514582996025833, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y2": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0005288231085669136, "brier": 4.6647755469637755e-07, "expected_brier": 4.664775546577478e-07, "multiclass_ece": 0.0005286542256008886, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2009877504138737e-05, "brier": 1.6815419803515915e-10, "expected_brier": 1.6815415726512128e-10, "multiclass_ece": 1.2009805385848615e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.9829785242694544e-05, "brier": 3.864241236688151e-09, "expected_brier": 3.864241149287295e-09, "multiclass_ece": 4.982854375956691e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0013684200363669134, "brier": 3.249303333722558e-06, "expected_brier": 3.2493033337299693e-06, "multiclass_ece": 0.0013674841766007084, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002941934457380139, "brier": 1.2832002363335596e-05, "expected_brier": 1.2832002363349915e-05, "multiclass_ece": 0.002937611208814861, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.8144888123437538e-05, "brier": 1.4327832588865211e-09, "expected_brier": 1.4327832253968609e-09, "multiclass_ece": 2.8144492059789528e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00020425562165384215, "brier": 8.342367674915315e-08, "expected_brier": 8.342367685543195e-08, "multiclass_ece": 0.00020423476289455067, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 2.4879879292988437e-05, "brier": 7.121356847607825e-10, "expected_brier": 7.121356837558324e-10, "multiclass_ece": 2.4879569791358414e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y1": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002229587711017857, "brier": 7.687241658625656e-08, "expected_brier": 7.687241654341648e-08, "multiclass_ece": 0.00022293308733023487, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00042165934092064985, "brier": 2.7212004346647155e-07, "expected_brier": 2.721200433875026e-07, "multiclass_ece": 0.00042157045511437286, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y3_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.003063721709994343, "brier": 1.2038247238464736e-05, "expected_brier": 1.2038247238388422e-05, "multiclass_ece": 0.003059033303848935, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00021675297923647057, "brier": 7.571738955775824e-08, "expected_brier": 7.571738946943185e-08, "multiclass_ece": 0.00021672949000661745, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y4_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0014101574323492092, "brier": 3.971484258434139e-06, "expected_brier": 3.971484258347147e-06, "multiclass_ece": 0.001409163627552501, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002121698927324271, "brier": 9.001302702148424e-08, "expected_brier": 9.001302703204317e-08, "multiclass_ece": 0.00021214738629249386, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y5_green": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0020974998816344127, "brier": 8.781282618086615e-06, "expected_brier": 8.781282618053332e-06, "multiclass_ece": 0.0020953014890853217, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002056041272986084, "brier": 8.437249232500436e-06, "expected_brier": 8.437249232606803e-06, "multiclass_ece": 0.002053929067969529, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010163721823842407, "brier": 1.5530483908443e-08, "expected_brier": 1.553048400992907e-08, "multiclass_ece": 0.00010163205335134151, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.004154622031660122, "brier": 3.437869052858229e-05, "expected_brier": 3.4378690528580336e-05, "multiclass_ece": 0.004146003529218345, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y4": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00037330917595239924, "brier": 1.6772465774545885e-07, "expected_brier": 1.6772465771808243e-07, "multiclass_ece": 0.0003732395047518766, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y5_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0016209091247894076, "brier": 5.2461834500226934e-06, "expected_brier": 5.246183450102748e-06, "multiclass_ece": 0.0016195961610880882, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00011082719648383489, "brier": 1.558148269277857e-08, "expected_brier": 1.558148277069904e-08, "multiclass_ece": 0.00011082105537696396, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.716696388210564e-05, "brier": 1.0951655462755968e-08, "expected_brier": 1.0951655360358359e-08, "multiclass_ece": 9.716224332556589e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000668800959517633, "brier": 8.939913779741252e-07, "expected_brier": 8.93991378037029e-07, "multiclass_ece": 0.0006685773620060953, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00045296968557796046, "brier": 3.2365056030596597e-07, "expected_brier": 3.236505603521067e-07, "multiclass_ece": 0.0004528671102983495, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006663983305783751, "brier": 6.301476538967448e-07, "expected_brier": 6.30147653835067e-07, "multiclass_ece": 0.00066617633652577, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004105829784215159, "brier": 2.3094246933161923e-07, "expected_brier": 2.3094246937027663e-07, "multiclass_ece": 0.0004104987007651495, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.88255771081337e-05, "brier": 4.553720799661831e-09, "expected_brier": 4.553720733824207e-09, "multiclass_ece": 5.8823846917799294e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002589299356342629, "brier": 9.781705892410881e-08, "expected_brier": 9.781705889722048e-08, "multiclass_ece": 0.00025889641617160564, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002384698462877605, "brier": 9.677903117362632e-08, "expected_brier": 9.677903123339604e-08, "multiclass_ece": 0.00023844141461404345, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0023375758944589576, "brier": 6.482513472018536e-06, "expected_brier": 6.482513472017004e-06, "multiclass_ece": 0.0023348458915384285, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.006080957695036884, "brier": 7.350796022809365e-05, "expected_brier": 7.350796022809192e-05, "multiclass_ece": 0.0060625060918770846, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006312100272802717, "brier": 7.963494011318071e-07, "expected_brier": 7.963494010887473e-07, "multiclass_ece": 0.00063101085613948, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 1.2024895282333029e-05, "brier": 2.672332540278524e-10, "expected_brier": 2.672333465625343e-10, "multiclass_ece": 1.202482298356955e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y6_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00021426187509051607, "brier": 9.179663196009754e-08, "expected_brier": 9.179663185143738e-08, "multiclass_ece": 0.00021423892265426403, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007386400615240164, "brier": 7.440966200517219e-07, "expected_brier": 7.440966200977783e-07, "multiclass_ece": 0.0007383673341070374, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.003816768659478026, "brier": 2.160672513655824e-05, "expected_brier": 2.160672513662476e-05, "multiclass_ece": 0.0038094940560804424, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.001696262568190709, "brier": 5.744861716876216e-06, "expected_brier": 5.744861716783234e-06, "multiclass_ece": 0.0016948247279403583, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.902844809545942e-05, "brier": 9.418229599017259e-09, "expected_brier": 9.418229529600808e-09, "multiclass_ece": 7.902532542991558e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y1_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006603852592974687, "brier": 8.716416037197318e-07, "expected_brier": 8.716416037835018e-07, "multiclass_ece": 0.0006601672529441549, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00017524032531485352, "brier": 6.140758136261214e-08, "expected_brier": 6.140758146422343e-08, "multiclass_ece": 0.0001752249716259202, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y2_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.010664141964816628, "brier": 0.0002104409841864265, "expected_brier": 0.00021044098418632196, "multiclass_ece": 0.010607481593172041, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y2_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00015145201674621187, "brier": 3.1526934279202574e-08, "expected_brier": 3.152693417440844e-08, "multiclass_ece": 0.00015144054846849553, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y4": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007537847372593145, "brier": 8.152493111091989e-07, "expected_brier": 8.152493110591408e-07, "multiclass_ece": 0.000753467381418016, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y5": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000496698632791552, "brier": 2.9603839570563255e-07, "expected_brier": 2.960383957972823e-07, "multiclass_ece": 0.0004965752984464888, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00025171928958723935, "brier": 9.444568381819583e-08, "expected_brier": 9.444568371819173e-08, "multiclass_ece": 0.0002516876109449617, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003397295661852634, "brier": 1.848702369256243e-07, "expected_brier": 1.848702368967281e-07, "multiclass_ece": 0.0003396718646306862, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y0_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.874623874566823e-05, "brier": 1.3555999710554025e-08, "expected_brier": 1.3555999656134077e-08, "multiclass_ece": 8.874230091471258e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002465382062814064, "brier": 1.2126290766022168e-05, "expected_brier": 1.212629076596805e-05, "multiclass_ece": 0.002462345504394392, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y2_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0023423704179335522, "brier": 1.0947729671892242e-05, "expected_brier": 1.0947729671917372e-05, "multiclass_ece": 0.0023396292090727133, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00040080462971282853, "brier": 2.6899325190170587e-07, "expected_brier": 2.6899325189511103e-07, "multiclass_ece": 0.00040072431826732036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003712101456955502, "brier": 1.9844051962246008e-07, "expected_brier": 1.9844051957207398e-07, "multiclass_ece": 0.0003711412557338978, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00032617679751884433, "brier": 1.525667803789894e-07, "expected_brier": 1.525667803203845e-07, "multiclass_ece": 0.00032611406068405735, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/decision": { "count": 24, "hard_count": 24, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "nll": 0.5427320513146096, "brier": 0.31667575659491426, "expected_brier": 0.31667575659491426, "multiclass_ece": 0.19934858720390158, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 21, "coverage": 0.875, "accuracy": 0.8095238095238095, "expected_accuracy": 0.8095238095238095, "expected_risk": 0.19047619047619047 }, { "threshold": 0.7, "selected": 15, "coverage": 0.625, "accuracy": 0.9333333333333333, "expected_accuracy": 0.9333333333333333, "expected_risk": 0.06666666666666665 }, { "threshold": 0.8, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 13, "coverage": 0.5416666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 10, "coverage": 0.4166666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/ordinal": { "count": 24, "hard_count": 24, "accuracy": 0.7916666666666666, "expected_accuracy": 0.7916666666666666, "nll": 0.502571560450988, "brier": 0.2722735668179664, "expected_brier": 0.2722735668179664, "multiclass_ece": 0.14591112557612815, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.9411764705882353, "expected_accuracy": 0.9411764705882353, "expected_risk": 0.05882352941176472 }, { "threshold": 0.7, "selected": 17, "coverage": 0.7083333333333334, "accuracy": 0.9411764705882353, "expected_accuracy": 0.9411764705882353, "expected_risk": 0.05882352941176472 }, { "threshold": 0.8, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 0.9375, "expected_accuracy": 0.9375, "expected_risk": 0.0625 }, { "threshold": 0.9, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 14, "coverage": 0.5833333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 12, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/verification": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.178315840048339, "brier": 0.09816389244379749, "expected_brier": 0.09816389244379746, "multiclass_ece": 0.07278448228378913, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.8, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.9, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 0.9473684210526315, "expected_accuracy": 0.9473684210526315, "expected_risk": 0.052631578947368474 }, { "threshold": 0.95, "selected": 16, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 10, "coverage": 0.4166666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/choice": { "count": 24, "hard_count": 19, "accuracy": 1.0, "expected_accuracy": 0.8819444444444445, "nll": 0.29712601929559984, "brier": 0.053982200603160245, "expected_brier": 0.17203775615871583, "multiclass_ece": 0.10376679563424629, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 22, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 0.9242424242424242, "expected_risk": 0.0757575757575758 }, { "threshold": 0.7, "selected": 18, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.8, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 12, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.375, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 0.25, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/noul": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.004101191574417952, "brier": 0.00014321005355808977, "expected_brier": 0.0001432100535580886, "multiclass_ece": 0.004064557879812784, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 22, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "tic_tac_toe-v1/choice": { "count": 24, "hard_count": 7, "accuracy": 0.5714285714285714, "expected_accuracy": 0.45734126984126994, "nll": 1.1849234218485212, "brier": 0.23991144494057084, "expected_brier": 0.6575701750993009, "multiclass_ece": 0.12109375036088056, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 9, "coverage": 0.375, "accuracy": 0.6666666666666666, "expected_accuracy": 0.5, "expected_risk": 0.5 }, { "threshold": 0.7, "selected": 1, "coverage": 0.041666666666666664, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "tile_platformer-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "nll": 0.21459948988092156, "brier": 0.07806250145934908, "expected_brier": 0.07806250145934908, "multiclass_ece": 0.07399205733604802, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.95, "selected": 21, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "trex_runner-v1/choice": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.7668877262100418, "brier": 0.40590170973174894, "expected_brier": 0.40590170973174894, "multiclass_ece": 0.2272462205045263, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/choice": { "count": 24, "hard_count": 24, "accuracy": 0.9166666666666666, "expected_accuracy": 0.9166666666666666, "nll": 0.17065624958851966, "brier": 0.0898379608773434, "expected_brier": 0.08983796087734341, "multiclass_ece": 0.08963187467063914, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 0.9166666666666666, "expected_accuracy": 0.9166666666666666, "expected_risk": 0.08333333333333337 }, { "threshold": 0.7, "selected": 23, "coverage": 0.9583333333333334, "accuracy": 0.9565217391304348, "expected_accuracy": 0.9565217391304348, "expected_risk": 0.04347826086956519 }, { "threshold": 0.8, "selected": 21, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 19, "coverage": 0.7916666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 25, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0029201255732878508, "brier": 3.3208274537195004e-05, "expected_brier": 3.320827453721087e-05, "multiclass_ece": 0.002911785681931036, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/score": { "count": 24, "hard_count": 24, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007028608481978504, "brier": 4.991128586173711e-06, "expected_brier": 4.991128586179228e-06, "multiclass_ece": 0.0007011061048053749, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 24, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1/choice": { "count": 24, "hard_count": 8, "accuracy": 0.25, "expected_accuracy": 0.2748181216931217, "nll": 2.0728827643161636, "brier": 0.36410946204707334, "expected_brier": 0.8392913403539518, "multiclass_ece": 0.11058666428069612, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.134363138088053e-05, "brier": 7.934939226222013e-09, "expected_brier": 7.934939224204149e-09, "multiclass_ece": 6.134164755596434e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.683141086259371e-05, "brier": 1.0160824924532853e-08, "expected_brier": 1.0160824996200782e-08, "multiclass_ece": 6.682887051271773e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001339040446799648, "brier": 3.950864679749943e-08, "expected_brier": 3.950864675417165e-08, "multiclass_ece": 0.00013389416646578667, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010449617085868287, "brier": 2.2308989347535604e-08, "expected_brier": 2.230898932253922e-08, "multiclass_ece": 0.00010449059320605247, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.581808163421519e-05, "brier": 1.8360450202767066e-08, "expected_brier": 1.8360450271259765e-08, "multiclass_ece": 9.581349122844696e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0005872859989919082, "brier": 1.0559807631020422e-06, "expected_brier": 1.0559807630539098e-06, "multiclass_ece": 0.00058702182654784, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002035079492118767, "brier": 2.247087806275251e-05, "expected_brier": 2.2470878062730293e-05, "multiclass_ece": 0.002029439985679282, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.713146307819914e-05, "brier": 4.68969330616843e-09, "expected_brier": 4.689693411386031e-09, "multiclass_ece": 4.7130290614205705e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0018375887314990742, "brier": 9.933717026489394e-06, "expected_brier": 9.93371702643723e-06, "multiclass_ece": 0.0018351007547604918, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004140012892537333, "brier": 4.717664994087792e-07, "expected_brier": 4.717664994213777e-07, "multiclass_ece": 0.0004138833002601909, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.761972783620936e-05, "brier": 9.457872920911922e-09, "expected_brier": 9.457872929274913e-09, "multiclass_ece": 6.761736325389123e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00041917276621391815, "brier": 3.51264349703224e-07, "expected_brier": 3.5126434960730535e-07, "multiclass_ece": 0.0004190849255838458, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.020155140571572e-05, "brier": 7.4715301053143e-09, "expected_brier": 7.471530061309295e-09, "multiclass_ece": 6.019968344350701e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009507565410855474, "brier": 2.00072611219172e-06, "expected_brier": 2.000726112227902e-06, "multiclass_ece": 0.0009502559701646263, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003263873853394745, "brier": 3.0663447104894676e-07, "expected_brier": 3.066344710589419e-07, "multiclass_ece": 0.0003263106988454112, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001052449190455895, "brier": 2.2150654623729562e-08, "expected_brier": 2.2150654643837697e-08, "multiclass_ece": 0.0001052393809933827, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00012264102405465987, "brier": 3.007795258293642e-08, "expected_brier": 3.007795257303769e-08, "multiclass_ece": 0.0001226335039516968, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00017879204481086638, "brier": 6.499232825324499e-08, "expected_brier": 6.499232824141998e-08, "multiclass_ece": 0.00017877579473202232, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006182613257252711, "brier": 8.065218210501825e-07, "expected_brier": 8.065218210506764e-07, "multiclass_ece": 0.0006180596032241414, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00015149313738515414, "brier": 4.5893388377326044e-08, "expected_brier": 4.58933884228685e-08, "multiclass_ece": 0.00015148166287926035, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00016905661500543913, "brier": 9.735301622943813e-08, "expected_brier": 9.735301624106778e-08, "multiclass_ece": 0.00016903227107730365, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0008144009523417361, "brier": 2.2178545733161367e-06, "expected_brier": 2.217854573216549e-06, "multiclass_ece": 0.000813845944963143, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0018357649718195898, "brier": 2.4000752315097694e-05, "expected_brier": 2.4000752315067597e-05, "multiclass_ece": 0.001829736973597118, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.1694282847471326e-05, "brier": 3.4770713054573314e-09, "expected_brier": 3.477071297552925e-09, "multiclass_ece": 4.1693413555421444e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 3.921886317989902e-05, "brier": 3.2537944702967945e-09, "expected_brier": 3.2537944560928622e-09, "multiclass_ece": 3.921804970774545e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023195833888703672, "brier": 1.8342147642972625e-07, "expected_brier": 1.8342147637776898e-07, "multiclass_ece": 0.00023191246906921018, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010288028477962328, "brier": 2.1166528280887476e-08, "expected_brier": 2.1166528307858812e-08, "multiclass_ece": 0.00010287499278460732, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/DISABLE ACCOUNT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.546350328857911e-05, "brier": 1.1787575687915564e-08, "expected_brier": 1.178757569331168e-08, "multiclass_ece": 7.546055623630643e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00033795426321249527, "brier": 2.498482765071669e-07, "expected_brier": 2.4984827651231e-07, "multiclass_ece": 0.00033789178464571634, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0016558152472601947, "brier": 1.1187481206192702e-05, "expected_brier": 1.1187481206246611e-05, "multiclass_ece": 0.0016530110984596114, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00028276411504809623, "brier": 1.7767887593126247e-07, "expected_brier": 1.7767887590854534e-07, "multiclass_ece": 0.000282719685306206, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/KILL PROCESS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023505746250042795, "brier": 1.10478050120608e-07, "expected_brier": 1.1047805004160693e-07, "multiclass_ece": 0.00023502983865952842, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001350706710307835, "brier": 4.501749840364122e-08, "expected_brier": 4.501749839569413e-08, "multiclass_ece": 0.00013505941525837617, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.5655170836877416e-05, "brier": 4.168598926577302e-09, "expected_brier": 4.168599021525665e-09, "multiclass_ece": 4.565412865542573e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00012793675873279568, "brier": 4.246664275590985e-08, "expected_brier": 4.2466642757865714e-08, "multiclass_ece": 0.00012792614075129993, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REMOVE FORWARDING RULES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023059938816193335, "brier": 1.131178233353197e-07, "expected_brier": 1.1311782333045528e-07, "multiclass_ece": 0.00023057110383617463, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 9.959949221521067e-05, "brier": 1.9839248325204797e-08, "expected_brier": 1.983924824422445e-08, "multiclass_ece": 9.959453207375457e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0020108949862393433, "brier": 8.071153445019375e-06, "expected_brier": 8.07115344492182e-06, "multiclass_ece": 0.0020088744914776857, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } } }, "calibrated_ood": { "count": 512, "hard_count": 484, "accuracy": 0.9235537190082644, "expected_accuracy": 0.8863878038194444, "nll": 0.3274741434510443, "brier": 0.11207364503013588, "expected_brier": 0.14928610162735823, "multiclass_ece": 0.028300110745501435, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 469, "coverage": 0.916015625, "accuracy": 0.9460043196544277, "expected_accuracy": 0.9402985074626866, "expected_risk": 0.05970149253731338 }, { "threshold": 0.7, "selected": 448, "coverage": 0.875, "accuracy": 0.9598214285714286, "expected_accuracy": 0.9598214285714286, "expected_risk": 0.0401785714285714 }, { "threshold": 0.8, "selected": 437, "coverage": 0.853515625, "accuracy": 0.9679633867276888, "expected_accuracy": 0.9679633867276888, "expected_risk": 0.03203661327231122 }, { "threshold": 0.9, "selected": 410, "coverage": 0.80078125, "accuracy": 0.9780487804878049, "expected_accuracy": 0.9780487804878049, "expected_risk": 0.02195121951219514 }, { "threshold": 0.95, "selected": 392, "coverage": 0.765625, "accuracy": 0.9821428571428571, "expected_accuracy": 0.9821428571428571, "expected_risk": 0.017857142857142905 }, { "threshold": 0.99, "selected": 328, "coverage": 0.640625, "accuracy": 0.9908536585365854, "expected_accuracy": 0.9908536585365854, "expected_risk": 0.009146341463414642 } ], "mean_latency_seconds": 0.053288100301870145, "by_kind": { "choice": { "count": 162, "hard_count": 136, "accuracy": 0.8823529411764706, "expected_accuracy": 0.7787894375857338, "nll": 0.6548541859616599, "brier": 0.17740883137839214, "expected_brier": 0.2867881180724936, "multiclass_ece": 0.06383224061265874, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 128, "coverage": 0.7901234567901234, "accuracy": 0.9180327868852459, "expected_accuracy": 0.8984375, "expected_risk": 0.1015625 }, { "threshold": 0.7, "selected": 115, "coverage": 0.7098765432098766, "accuracy": 0.9478260869565217, "expected_accuracy": 0.9478260869565217, "expected_risk": 0.05217391304347829 }, { "threshold": 0.8, "selected": 111, "coverage": 0.6851851851851852, "accuracy": 0.954954954954955, "expected_accuracy": 0.954954954954955, "expected_risk": 0.04504504504504503 }, { "threshold": 0.9, "selected": 97, "coverage": 0.5987654320987654, "accuracy": 0.9587628865979382, "expected_accuracy": 0.9587628865979382, "expected_risk": 0.04123711340206182 }, { "threshold": 0.95, "selected": 83, "coverage": 0.5123456790123457, "accuracy": 0.9759036144578314, "expected_accuracy": 0.9759036144578314, "expected_risk": 0.02409638554216864 }, { "threshold": 0.99, "selected": 41, "coverage": 0.25308641975308643, "accuracy": 0.975609756097561, "expected_accuracy": 0.975609756097561, "expected_risk": 0.024390243902439046 } ] }, "noul": { "count": 243, "hard_count": 243, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.10974939814038452, "brier": 0.05812875048533286, "expected_brier": 0.05812875048533285, "multiclass_ece": 0.023946645010765814, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 243, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 236, "coverage": 0.9711934156378601, "accuracy": 0.9703389830508474, "expected_accuracy": 0.9703389830508474, "expected_risk": 0.029661016949152574 }, { "threshold": 0.8, "selected": 233, "coverage": 0.9588477366255144, "accuracy": 0.9785407725321889, "expected_accuracy": 0.9785407725321889, "expected_risk": 0.02145922746781115 }, { "threshold": 0.9, "selected": 226, "coverage": 0.9300411522633745, "accuracy": 0.9867256637168141, "expected_accuracy": 0.9867256637168141, "expected_risk": 0.013274336283185861 }, { "threshold": 0.95, "selected": 224, "coverage": 0.9218106995884774, "accuracy": 0.9866071428571429, "expected_accuracy": 0.9866071428571429, "expected_risk": 0.013392857142857095 }, { "threshold": 0.99, "selected": 211, "coverage": 0.8683127572016461, "accuracy": 0.995260663507109, "expected_accuracy": 0.995260663507109, "expected_risk": 0.004739336492891044 } ] }, "score": { "count": 107, "hard_count": 105, "accuracy": 0.8857142857142857, "expected_accuracy": 0.8753894080996886, "nll": 0.3262736408694633, "brier": 0.13566531966536616, "expected_brier": 0.14812637885539728, "multiclass_ece": 0.051506733181640355, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 98, "coverage": 0.9158878504672897, "accuracy": 0.9387755102040817, "expected_accuracy": 0.9387755102040817, "expected_risk": 0.061224489795918324 }, { "threshold": 0.7, "selected": 97, "coverage": 0.9065420560747663, "accuracy": 0.9484536082474226, "expected_accuracy": 0.9484536082474226, "expected_risk": 0.05154639175257736 }, { "threshold": 0.8, "selected": 93, "coverage": 0.8691588785046729, "accuracy": 0.956989247311828, "expected_accuracy": 0.956989247311828, "expected_risk": 0.043010752688172005 }, { "threshold": 0.9, "selected": 87, "coverage": 0.8130841121495327, "accuracy": 0.9770114942528736, "expected_accuracy": 0.9770114942528736, "expected_risk": 0.02298850574712641 }, { "threshold": 0.95, "selected": 85, "coverage": 0.794392523364486, "accuracy": 0.9764705882352941, "expected_accuracy": 0.9764705882352941, "expected_risk": 0.02352941176470591 }, { "threshold": 0.99, "selected": 76, "coverage": 0.7102803738317757, "accuracy": 0.9868421052631579, "expected_accuracy": 0.9868421052631579, "expected_risk": 0.013157894736842146 } ], "ordinal_mae": 0.16061127324321497 } }, "by_source": { "customer-control-v1": { "count": 81, "hard_count": 77, "accuracy": 1.0, "expected_accuracy": 0.97119341563786, "nll": 0.05752075183048635, "brier": 0.0026166689133205433, "expected_brier": 0.03142325327546045, "multiclass_ece": 0.011954242228997934, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 79, "coverage": 0.9753086419753086, "accuracy": 1.0, "expected_accuracy": 0.9873417721518988, "expected_risk": 0.012658227848101222 }, { "threshold": 0.7, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 76, "coverage": 0.9382716049382716, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 74, "coverage": 0.9135802469135802, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 72, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 66, "coverage": 0.8148148148148148, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1": { "count": 81, "hard_count": 81, "accuracy": 0.9135802469135802, "expected_accuracy": 0.9135802469135802, "nll": 0.3310712700069499, "brier": 0.13379486342020344, "expected_brier": 0.13379486342020344, "multiclass_ece": 0.06536289347636093, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.925, "expected_accuracy": 0.925, "expected_risk": 0.07499999999999996 }, { "threshold": 0.7, "selected": 77, "coverage": 0.9506172839506173, "accuracy": 0.935064935064935, "expected_accuracy": 0.935064935064935, "expected_risk": 0.06493506493506496 }, { "threshold": 0.8, "selected": 73, "coverage": 0.9012345679012346, "accuracy": 0.958904109589041, "expected_accuracy": 0.958904109589041, "expected_risk": 0.04109589041095896 }, { "threshold": 0.9, "selected": 70, "coverage": 0.8641975308641975, "accuracy": 0.9571428571428572, "expected_accuracy": 0.9571428571428572, "expected_risk": 0.042857142857142816 }, { "threshold": 0.95, "selected": 65, "coverage": 0.8024691358024691, "accuracy": 0.9692307692307692, "expected_accuracy": 0.9692307692307692, "expected_risk": 0.03076923076923077 }, { "threshold": 0.99, "selected": 45, "coverage": 0.5555555555555556, "accuracy": 0.9777777777777777, "expected_accuracy": 0.9777777777777777, "expected_risk": 0.022222222222222254 } ] }, "reasoning-control-v1": { "count": 80, "hard_count": 80, "accuracy": 0.7, "expected_accuracy": 0.7, "nll": 0.7237110706906145, "brier": 0.38911651895787797, "expected_brier": 0.38911651895787797, "multiclass_ece": 0.13289847513450215, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 69, "coverage": 0.8625, "accuracy": 0.782608695652174, "expected_accuracy": 0.782608695652174, "expected_risk": 0.21739130434782605 }, { "threshold": 0.7, "selected": 58, "coverage": 0.725, "accuracy": 0.8275862068965517, "expected_accuracy": 0.8275862068965517, "expected_risk": 0.1724137931034483 }, { "threshold": 0.8, "selected": 55, "coverage": 0.6875, "accuracy": 0.8363636363636363, "expected_accuracy": 0.8363636363636363, "expected_risk": 0.1636363636363637 }, { "threshold": 0.9, "selected": 42, "coverage": 0.525, "accuracy": 0.9047619047619048, "expected_accuracy": 0.9047619047619048, "expected_risk": 0.09523809523809523 }, { "threshold": 0.95, "selected": 40, "coverage": 0.5, "accuracy": 0.9, "expected_accuracy": 0.9, "expected_risk": 0.09999999999999998 }, { "threshold": 0.99, "selected": 29, "coverage": 0.3625, "accuracy": 0.9655172413793104, "expected_accuracy": 0.9655172413793104, "expected_risk": 0.03448275862068961 } ] }, "snake-v1": { "count": 54, "hard_count": 48, "accuracy": 1.0, "expected_accuracy": 0.9444444444444444, "nll": 0.11428921672858919, "brier": 0.0070235237181837185, "expected_brier": 0.06257907927373928, "multiclass_ece": 0.0333918416747302, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 52, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 0.9615384615384616, "expected_risk": 0.038461538461538436 }, { "threshold": 0.7, "selected": 48, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 47, "coverage": 0.8703703703703703, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 41, "coverage": 0.7592592592592593, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 36, "coverage": 0.6666666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 28, "coverage": 0.5185185185185185, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1": { "count": 81, "hard_count": 81, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.08872748524690638, "brier": 0.04853337901580716, "expected_brier": 0.048533379015807156, "multiclass_ece": 0.021508724448020638, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 81, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 80, "coverage": 0.9876543209876543, "accuracy": 0.975, "expected_accuracy": 0.975, "expected_risk": 0.025000000000000022 }, { "threshold": 0.8, "selected": 78, "coverage": 0.9629629629629629, "accuracy": 0.9871794871794872, "expected_accuracy": 0.9871794871794872, "expected_risk": 0.012820512820512775 }, { "threshold": 0.9, "selected": 75, "coverage": 0.9259259259259259, "accuracy": 0.9866666666666667, "expected_accuracy": 0.9866666666666667, "expected_risk": 0.013333333333333308 }, { "threshold": 0.95, "selected": 71, "coverage": 0.8765432098765432, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 52, "coverage": 0.6419753086419753, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1": { "count": 27, "hard_count": 9, "accuracy": 0.7777777777777778, "expected_accuracy": 0.3394032921810699, "nll": 2.1410487638121265, "brier": 0.32947255656184854, "expected_brier": 0.8376001285783096, "multiclass_ece": 0.23049063930445224, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ], "optimal_action_hit": 0.7037037037037037, "optimal_set_probability_mass": 0.4188949329606126 }, "workflow-controls-v1/agent_trace_observability": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00033549396903294343, "brier": 1.3288180881908662e-06, "expected_brier": 1.3288180881980364e-06, "multiclass_ece": 0.0003351609819247606, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service": { "count": 27, "hard_count": 27, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "nll": 0.26312139319474426, "brier": 0.07395395075650218, "expected_brier": 0.07395395075650218, "multiclass_ece": 0.036259317079984886, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 0.9629629629629629, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 } ] }, "workflow-controls-v1/invoice_processing": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00027584851310097407, "brier": 5.651231280426225e-07, "expected_brier": 5.651231280629972e-07, "multiclass_ece": 0.00027570704257610885, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023250104840684022, "brier": 3.631375298814665e-07, "expected_brier": 3.631375298898779e-07, "multiclass_ece": 0.00023241017921299195, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "by_question": { "customer-control-v1/bug_severity": { "count": 8, "hard_count": 8, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0596270372994904, "brier": 0.016077183000560602, "expected_brier": 0.016077183000560602, "multiclass_ece": 0.053891848116715296, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 8, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 7, "coverage": 0.875, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 0.625, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/category": { "count": 27, "hard_count": 25, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "nll": 0.06629311590797901, "brier": 0.002386847370949229, "expected_brier": 0.039423884407986255, "multiclass_ece": 0.011429094801366006, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 0.9629629629629629, "expected_risk": 0.03703703703703709 }, { "threshold": 0.7, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 23, "coverage": 0.8518518518518519, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 22, "coverage": 0.8148148148148148, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/churn_likelihood_level": { "count": 9, "hard_count": 7, "accuracy": 1.0, "expected_accuracy": 0.8518518518518517, "nll": 0.246437888698758, "brier": 0.0010209429346550104, "expected_brier": 0.1491690910828032, "multiclass_ece": 0.006309306236315957, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 7, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/frustration": { "count": 10, "hard_count": 10, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0120343418729683, "brier": 0.0009056885858861934, "expected_brier": 0.0009056885858861774, "multiclass_ece": 0.011798498182724515, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 10, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 9, "coverage": 0.9, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 0.6, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/has_reproducible_steps": { "count": 15, "hard_count": 15, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0019110672352558352, "brier": 1.5979716770424227e-05, "expected_brier": 1.5979716770416394e-05, "multiclass_ece": 0.0019070571026006179, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 15, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "customer-control-v1/refund_requested": { "count": 12, "hard_count": 12, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0021083370675581002, "brier": 3.3564244711390324e-05, "expected_brier": 3.356424471138523e-05, "multiclass_ece": 0.0020998901237594403, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 12, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 11, "coverage": 0.9166666666666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004512847900242236, "brier": 4.0713215631337377e-07, "expected_brier": 4.071321562104657e-07, "multiclass_ece": 0.0004511829763595987, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0011646329848062264, "brier": 1.755993494064031e-06, "expected_brier": 1.7559934941679956e-06, "multiclass_ece": 0.0011639550630138107, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y3_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009569691617571543, "brier": 1.829828165624737e-06, "expected_brier": 1.8298281656559112e-06, "multiclass_ece": 0.0009565114127977337, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x0_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.18772461628203724, "brier": 0.05858948819743733, "expected_brier": 0.05858948819743737, "multiclass_ece": 0.1711570743461066, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x0_y8_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.008391410333324587, "brier": 0.00013965552360146826, "expected_brier": 0.0001396555236015118, "multiclass_ece": 0.008356300724646881, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004135449945617499, "brier": 2.1883774941812448e-07, "expected_brier": 2.1883774947717427e-07, "multiclass_ece": 0.0004134594966166416, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y11": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.013280535499054464, "brier": 0.0002991648034055647, "expected_brier": 0.0002991648034056382, "multiclass_ece": 0.01319273828192824, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y4_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00026085310927372176, "brier": 1.3605319547896966e-07, "expected_brier": 1.360531953764621e-07, "multiclass_ece": 0.0002608190900594831, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007048178842140534, "brier": 7.708315703799688e-07, "expected_brier": 7.708315703514046e-07, "multiclass_ece": 0.0007045695584340095, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x10_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 8.404114498445805, "brier": 1.9989238086574577, "expected_brier": 1.998923808657458, "multiclass_ece": 0.9996857698158839, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 } ] }, "painting-geometry-v1/x10_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.22900987924445249, "brier": 0.05773931366943554, "expected_brier": 0.05773931366943552, "multiclass_ece": 0.20467932370109687, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.015086741783439207, "brier": 0.000448411827202865, "expected_brier": 0.00044841182720278816, "multiclass_ece": 0.014973507057514324, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x10_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.006384606601236843, "brier": 4.988918618220202e-05, "expected_brier": 4.988918618220595e-05, "multiclass_ece": 0.006364268307531451, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004709489257930665, "brier": 2.6017713194408716e-07, "expected_brier": 2.6017713183890123e-07, "multiclass_ece": 0.0004708380467545181, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y10": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.008833033267761832, "brier": 0.00010950716510219036, "expected_brier": 0.00010950716510227387, "multiclass_ece": 0.008794136638730521, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x11_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.006404672173680857, "brier": 7.444760404614192e-05, "expected_brier": 7.444760404617945e-05, "multiclass_ece": 0.00638420597725764, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0027962083855187173, "brier": 9.41624854692937e-06, "expected_brier": 9.416248546889605e-06, "multiclass_ece": 0.0027923026361289516, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y10_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.010189466090209971, "brier": 0.00020554711286775474, "expected_brier": 0.0002055471128676878, "multiclass_ece": 0.010137729352960534, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y3_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 4.294280968988404e-05, "brier": 2.8150694570287523e-09, "expected_brier": 2.815069510830881e-09, "multiclass_ece": 4.294188766063023e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009192974560972416, "brier": 1.3840135173508133e-06, "expected_brier": 1.3840135173470713e-06, "multiclass_ece": 0.0009188750316453387, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y6_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.02050923890220168, "brier": 0.0005914898800113549, "expected_brier": 0.0005914898800114532, "multiclass_ece": 0.020300354916502505, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x1_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0029285870261912443, "brier": 1.1214949719288416e-05, "expected_brier": 1.1214949719340339e-05, "multiclass_ece": 0.002924302898373754, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y7_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.006817577305805964, "brier": 9.232748041629999e-05, "expected_brier": 9.2327480416321e-05, "multiclass_ece": 0.006794390348526469, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y8_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007494116660273965, "brier": 7.202323739749606e-07, "expected_brier": 7.202323739630145e-07, "multiclass_ece": 0.000749130927238828, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y9_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.010039798363485853, "brier": 0.00019958293085715355, "expected_brier": 0.00019958293085720058, "multiclass_ece": 0.00998956782992022, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x1_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.015711053352060717, "brier": 0.0003405801553428148, "expected_brier": 0.00034058015534288444, "multiclass_ece": 0.015588278568005931, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y10_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00610427972874662, "brier": 4.992302925858561e-05, "expected_brier": 4.99230292585473e-05, "multiclass_ece": 0.006085686465307605, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.010978699821910399, "brier": 0.00023843400336349938, "expected_brier": 0.00023843400336343024, "multiclass_ece": 0.010918653840183312, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002676509163232523, "brier": 9.395795789981184e-06, "expected_brier": 9.395795790068107e-06, "multiclass_ece": 0.0026729305060636532, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.011654288230521134, "brier": 0.000268500456016739, "expected_brier": 0.00026850045601678474, "multiclass_ece": 0.011586640065539666, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x2_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0016489536001657432, "brier": 4.1230169336038325e-06, "expected_brier": 4.123016933643697e-06, "multiclass_ece": 0.0016475948231340354, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x2_y8": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.01616585261066225, "brier": 0.00034323801394726223, "expected_brier": 0.0003432380139472002, "multiclass_ece": 0.016031281367488104, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0008268943441901229, "brier": 8.969146636358422e-07, "expected_brier": 8.969146636506053e-07, "multiclass_ece": 0.0008265525612745073, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y11_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004975332858050713, "brier": 4.94832494288423e-07, "expected_brier": 4.948324943132576e-07, "multiclass_ece": 0.0004974095366437892, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y1_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007694067422099408, "brier": 1.1830629214674472e-06, "expected_brier": 1.1830629214060195e-06, "multiclass_ece": 0.0007691108247409595, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y6": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.03600798668977202, "brier": 0.002051253676232974, "expected_brier": 0.0020512536762329336, "multiclass_ece": 0.03536741076887684, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x3_y7_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001910645846738137, "brier": 7.299740270434799e-08, "expected_brier": 7.299740278909894e-08, "multiclass_ece": 0.00019104633299849016, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x3_y7_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0009135139501852554, "brier": 9.510145350506098e-07, "expected_brier": 9.510145351221411e-07, "multiclass_ece": 0.0009130968233433956, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y10_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.005431950058352641, "brier": 3.734667209142724e-05, "expected_brier": 3.73466720913207e-05, "multiclass_ece": 0.0054172236939918905, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y11_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0012362481337569251, "brier": 2.0288266858102676e-06, "expected_brier": 2.0288266857626525e-06, "multiclass_ece": 0.0012354842938304689, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.017363271709081887, "brier": 0.00040655124081317745, "expected_brier": 0.00040655124081312444, "multiclass_ece": 0.017213398788601064, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x4_y1_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0012140652159571333, "brier": 2.7829475693172428e-06, "expected_brier": 2.78294756927977e-06, "multiclass_ece": 0.0012133285369384517, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y4_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00019632890525985485, "brier": 5.8917755822545804e-08, "expected_brier": 5.8917755785259374e-08, "multiclass_ece": 0.00019630963400152357, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y5_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0006764614307482312, "brier": 7.660833973770006e-07, "expected_brier": 7.660833973854864e-07, "multiclass_ece": 0.0006762326822973286, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x4_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.3689596749479083, "brier": 0.19040214107485892, "expected_brier": 0.19040214107485887, "multiclass_ece": 0.3085467072218232, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y1_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0017349524864745298, "brier": 4.281845011473484e-06, "expected_brier": 4.281845011444574e-06, "multiclass_ece": 0.0017334483264171974, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y2": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 3.245374266466996, "brier": 1.4526027035826286, "expected_brier": 1.4526027035826283, "multiclass_ece": 0.6972933969964465, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.062395410382802864, "brier": 0.0054888403529878204, "expected_brier": 0.0054888403529878005, "multiclass_ece": 0.0604886791956013, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y4_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002003101965885552, "brier": 5.300109766875741e-06, "expected_brier": 5.300109766848138e-06, "multiclass_ece": 0.0020010970960190333, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x5_y7": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.1919166714207906, "brier": 0.04638565273723018, "expected_brier": 0.046385652737230165, "multiclass_ece": 0.1704074289225055, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y8": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.343847953670822, "brier": 1.092714908793866, "expected_brier": 1.092714908793866, "multiclass_ece": 0.7391599653640157, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.6104294317873927, "brier": 0.25260285585725256, "expected_brier": 0.2526028558572526, "multiclass_ece": 0.45688241296744114, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x5_y9_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.003040231423321492, "brier": 1.8429911995706672e-05, "expected_brier": 1.8429911995632153e-05, "multiclass_ece": 0.0030356145996903727, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y0_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00024118937448701843, "brier": 7.471606501090774e-08, "expected_brier": 7.471606511266771e-08, "multiclass_ece": 0.00024116029066811873, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.017791355953146203, "brier": 0.00044919573574440924, "expected_brier": 0.0004491957357444232, "multiclass_ece": 0.01763402420987592, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.048300393526118174, "brier": 0.003683715153952731, "expected_brier": 0.0036837151539528268, "multiclass_ece": 0.04715248514234438, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y3_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0038792962763136538, "brier": 2.9981384385134036e-05, "expected_brier": 2.998138438514708e-05, "multiclass_ece": 0.0038717815269675437, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x6_y6_lightness": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 0.947009649415259, "brier": 0.5625773674219119, "expected_brier": 0.5625773674219119, "multiclass_ece": 0.41091134406193003, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y7_lightness": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.18706022209083661, "brier": 0.04075196652620534, "expected_brier": 0.040751966526205274, "multiclass_ece": 0.17060621294659084, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x6_y8_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 1.521811835443849, "brier": 1.2220598274581123, "expected_brier": 1.2220598274581125, "multiclass_ece": 0.7816840242252979, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y1_lightness": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.023321328317832596, "brier": 0.0008222040274293966, "expected_brier": 0.0008222040274294962, "multiclass_ece": 0.023036895073463737, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.003226771726564605, "brier": 2.0757043201006084e-05, "expected_brier": 2.0757043200925196e-05, "multiclass_ece": 0.0032215712937172514, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y5_blue": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 3.4268085525497622, "brier": 1.8721493924892352, "expected_brier": 1.8721493924892352, "multiclass_ece": 0.9675095328959905, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y6_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00816714868785301, "brier": 0.0001044594987842963, "expected_brier": 0.00010445949878423733, "multiclass_ece": 0.00813388813855176, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y7": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.14276648280286747, "brier": 0.03540114870599062, "expected_brier": 0.03540114870599065, "multiclass_ece": 0.1330435054897281, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y7_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0005543589954309227, "brier": 6.14287177341183e-07, "expected_brier": 6.142871773473502e-07, "multiclass_ece": 0.0005542053668727842, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x7_y8": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.09675675174877924, "brier": 0.017010239303482236, "expected_brier": 0.017010239303482333, "multiclass_ece": 0.0922232056032597, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x7_y9_saturation": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.004387935755037739, "brier": 2.8889321537436685e-05, "expected_brier": 2.8889321537439727e-05, "multiclass_ece": 0.004378322830380976, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y0_green": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002677289954428167, "brier": 1.4297441878793418e-05, "expected_brier": 1.4297441878863282e-05, "multiclass_ece": 0.002673709209954689, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y11_blue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0008228729165802322, "brier": 1.353125841250448e-06, "expected_brier": 1.353125841152547e-06, "multiclass_ece": 0.00082253444950664, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x8_y3": { "count": 1, "hard_count": 1, "accuracy": 0.0, "expected_accuracy": 0.0, "nll": 5.132145061187633, "brier": 1.866897738617619, "expected_brier": 1.8668977386176193, "multiclass_ece": 0.9360045818710508, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 0.0, "expected_accuracy": 0.0, "expected_risk": 1.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y0": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0029964269348101654, "brier": 1.0824682028985366e-05, "expected_brier": 1.0824682028953703e-05, "multiclass_ece": 0.002991942128205749, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y0_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.012669421555383725, "brier": 0.00019535930411328152, "expected_brier": 0.00019535930411329616, "multiclass_ece": 0.012589502300390687, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y1_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.018627261394164426, "brier": 0.00048213683015420637, "expected_brier": 0.0004821368301541895, "multiclass_ece": 0.018454846161547356, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y2": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.008008674185033315, "brier": 9.348099814094432e-05, "expected_brier": 9.348099814099342e-05, "multiclass_ece": 0.007970105323355181, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 0.5, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "painting-geometry-v1/x9_y6_red": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0519183717241388, "brier": 0.00511943239507501, "expected_brier": 0.005119432395074952, "multiclass_ece": 0.05059363791562632, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "painting-geometry-v1/x9_y9_hue": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0168000619161805, "brier": 0.0003604577181360302, "expected_brier": 0.0003604577181359758, "multiclass_ece": 0.016659727848654438, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "reasoning-control-v1/decision": { "count": 27, "hard_count": 27, "accuracy": 0.6666666666666666, "expected_accuracy": 0.6666666666666666, "nll": 0.877162899978905, "brier": 0.4378886390077441, "expected_brier": 0.4378886390077441, "multiclass_ece": 0.22621336215197788, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 22, "coverage": 0.8148148148148148, "accuracy": 0.7727272727272727, "expected_accuracy": 0.7727272727272727, "expected_risk": 0.2272727272727273 }, { "threshold": 0.7, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.8, "selected": 18, "coverage": 0.6666666666666666, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.9, "selected": 13, "coverage": 0.48148148148148145, "accuracy": 0.8461538461538461, "expected_accuracy": 0.8461538461538461, "expected_risk": 0.15384615384615385 }, { "threshold": 0.95, "selected": 12, "coverage": 0.4444444444444444, "accuracy": 0.8333333333333334, "expected_accuracy": 0.8333333333333334, "expected_risk": 0.16666666666666663 }, { "threshold": 0.99, "selected": 9, "coverage": 0.3333333333333333, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "expected_risk": 0.11111111111111116 } ] }, "reasoning-control-v1/ordinal": { "count": 26, "hard_count": 26, "accuracy": 0.6153846153846154, "expected_accuracy": 0.6153846153846154, "nll": 0.8548674417121757, "brier": 0.4502834847454645, "expected_brier": 0.45028348474546437, "multiclass_ece": 0.19542367248217338, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 20, "coverage": 0.7692307692307693, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 19, "coverage": 0.7307692307692307, "accuracy": 0.7894736842105263, "expected_accuracy": 0.7894736842105263, "expected_risk": 0.21052631578947367 }, { "threshold": 0.8, "selected": 17, "coverage": 0.6538461538461539, "accuracy": 0.8235294117647058, "expected_accuracy": 0.8235294117647058, "expected_risk": 0.17647058823529416 }, { "threshold": 0.9, "selected": 13, "coverage": 0.5, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 }, { "threshold": 0.95, "selected": 12, "coverage": 0.46153846153846156, "accuracy": 0.9166666666666666, "expected_accuracy": 0.9166666666666666, "expected_risk": 0.08333333333333337 }, { "threshold": 0.99, "selected": 9, "coverage": 0.34615384615384615, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "reasoning-control-v1/verification": { "count": 27, "hard_count": 27, "accuracy": 0.8148148148148148, "expected_accuracy": 0.8148148148148148, "nll": 0.44396051375193163, "brier": 0.2814428762977431, "expected_brier": 0.2814428762977432, "multiclass_ece": 0.12549231713920872, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8148148148148148, "expected_accuracy": 0.8148148148148148, "expected_risk": 0.18518518518518523 }, { "threshold": 0.7, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 0.8571428571428571, "expected_accuracy": 0.8571428571428571, "expected_risk": 0.1428571428571429 }, { "threshold": 0.8, "selected": 20, "coverage": 0.7407407407407407, "accuracy": 0.85, "expected_accuracy": 0.85, "expected_risk": 0.15000000000000002 }, { "threshold": 0.9, "selected": 16, "coverage": 0.5925925925925926, "accuracy": 0.9375, "expected_accuracy": 0.9375, "expected_risk": 0.0625 }, { "threshold": 0.95, "selected": 16, "coverage": 0.5925925925925926, "accuracy": 0.9375, "expected_accuracy": 0.9375, "expected_risk": 0.0625 }, { "threshold": 0.99, "selected": 11, "coverage": 0.4074074074074074, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/choice": { "count": 27, "hard_count": 21, "accuracy": 1.0, "expected_accuracy": 0.8888888888888888, "nll": 0.22140723206135995, "brier": 0.012859679986324733, "expected_brier": 0.12397079109743589, "multiclass_ece": 0.0599365212831876, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 0.92, "expected_risk": 0.07999999999999996 }, { "threshold": 0.7, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 20, "coverage": 0.7407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 15, "coverage": 0.5555555555555556, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 10, "coverage": 0.37037037037037035, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 0.07407407407407407, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "snake-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.007171201395818442, "brier": 0.0011873674500427075, "expected_brier": 0.001187367450042714, "multiclass_ece": 0.00684716206627305, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/choice": { "count": 27, "hard_count": 27, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "nll": 0.2629385175326002, "brier": 0.14553730306375842, "expected_brier": 0.14553730306375834, "multiclass_ece": 0.06775418608927014, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 0.8888888888888888, "expected_accuracy": 0.8888888888888888, "expected_risk": 0.11111111111111116 }, { "threshold": 0.7, "selected": 26, "coverage": 0.9629629629629629, "accuracy": 0.9230769230769231, "expected_accuracy": 0.9230769230769231, "expected_risk": 0.07692307692307687 }, { "threshold": 0.8, "selected": 24, "coverage": 0.8888888888888888, "accuracy": 0.9583333333333334, "expected_accuracy": 0.9583333333333334, "expected_risk": 0.04166666666666663 }, { "threshold": 0.9, "selected": 21, "coverage": 0.7777777777777778, "accuracy": 0.9523809523809523, "expected_accuracy": 0.9523809523809523, "expected_risk": 0.04761904761904767 }, { "threshold": 0.95, "selected": 17, "coverage": 0.6296296296296297, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "vizdoom-basic-v1/noul": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.002831105190248831, "brier": 6.226346861730999e-05, "expected_brier": 6.22634686173164e-05, "multiclass_ece": 0.002815362445335189, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 25, "coverage": 0.9259259259259259, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "vizdoom-basic-v1/score": { "count": 27, "hard_count": 27, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004128330178700702, "brier": 5.70515045773518e-07, "expected_brier": 5.705150457736686e-07, "multiclass_ece": 0.0004126502998715864, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 27, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "wikispeedia-v1/choice": { "count": 27, "hard_count": 9, "accuracy": 0.7777777777777778, "expected_accuracy": 0.3394032921810699, "nll": 2.1410487638121265, "brier": 0.32947255656184854, "expected_brier": 0.8376001285783096, "multiclass_ece": 0.23049063930445224, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "workflow-controls-v1/agent_trace_observability/AUTO-CLOSE": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001046157036634638, "brier": 2.275064777317268e-08, "expected_brier": 2.2750647810099167e-08, "multiclass_ece": 0.00010461001557693361, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/COUNT ONLY": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.3918536841240076e-05, "brier": 5.899118190121469e-09, "expected_brier": 5.899118193930519e-09, "multiclass_ece": 5.3917062007080574e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/FILE ISSUE ยท ROUTE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001133652134421077, "brier": 2.59384954929992e-08, "expected_brier": 2.5938495462618505e-08, "multiclass_ece": 0.00011335872831907956, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/HUMAN REVIEW": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00012122358853159514, "brier": 2.9386754285207804e-08, "expected_brier": 2.938675436325866e-08, "multiclass_ece": 0.00012121624124927788, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/NOT A BUG": { "count": 6, "hard_count": 6, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0008672678650548327, "brier": 5.3560427077316316e-06, "expected_brier": 5.356042707756655e-06, "multiclass_ece": 0.000865925411224803, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 6, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PAGE ON-CALL": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003148337515495486, "brier": 3.125111771595167e-07, "expected_brier": 3.1251117711406095e-07, "multiclass_ece": 0.00031475559355289564, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/agent_trace_observability/PRIORITY REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003968777215652089, "brier": 6.572050678204634e-07, "expected_brier": 6.572050678480915e-07, "multiclass_ece": 0.00039671331487263944, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CANCEL": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0008449689572096142, "brier": 5.120570996716288e-06, "expected_brier": 5.120570996774987e-06, "multiclass_ece": 0.0008436860808870295, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00047799134118430036, "brier": 1.0603532275861043e-06, "expected_brier": 1.0603532275924958e-06, "multiclass_ece": 0.00047772603296847294, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FLAG FOR REVIEW": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.001064394844110582, "brier": 9.371666571337994e-06, "expected_brier": 9.371666571311721e-06, "multiclass_ece": 0.0010620443675703583, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/FREEZE CARD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0012974797748565158, "brier": 9.457794227273432e-06, "expected_brier": 9.457794227267039e-06, "multiclass_ece": 0.0012951093767951487, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/HAND OFF": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 1.7712935257172324, "brier": 0.49916222233201657, "expected_brier": 0.49916222233201657, "multiclass_ece": 0.2495195151450289, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 } ] }, "workflow-controls-v1/customer_service/REFUND": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 5.9203524578481614e-05, "brier": 7.009699635873615e-09, "expected_brier": 7.009699709570327e-09, "multiclass_ece": 5.9201772084405135e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SAY": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007262192906153259, "brier": 1.5662082443036672e-06, "expected_brier": 1.5662082443560859e-06, "multiclass_ece": 0.0007258274327715716, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/customer_service/SET INTENT": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0007093471488072025, "brier": 1.0451009629668242e-06, "expected_brier": 1.0451009629375285e-06, "multiclass_ece": 0.0007090857398593942, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DISPUTE LINES": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010199092916059843, "brier": 2.296776495228978e-08, "expected_brier": 2.296776494918144e-08, "multiclass_ece": 0.00010198518674690386, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/DUPLICATE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023785268499747995, "brier": 1.3765332188923072e-07, "expected_brier": 1.376533218522269e-07, "multiclass_ece": 0.0002378182642638338, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/FRAUD REVIEW": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00012709394101049598, "brier": 3.317435719299215e-08, "expected_brier": 3.317435718723042e-08, "multiclass_ece": 0.00012708564667936648, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000564738766240431, "brier": 6.598955718016613e-07, "expected_brier": 6.598955718898717e-07, "multiclass_ece": 0.0005645737257551353, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/HOLD FOR DOCUMENTS": { "count": 5, "hard_count": 5, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00024735792661039445, "brier": 1.3049241020882002e-07, "expected_brier": 1.3049241021079183e-07, "multiclass_ece": 0.0002473252974573015, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 5, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/PAY": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.658372892040347e-05, "brier": 8.866195553157688e-09, "expected_brier": 8.86619555551249e-09, "multiclass_ece": 6.658151227312281e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/REQUEST CORRECTED INVOICE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 8.175479246145417e-05, "brier": 1.406550277000086e-08, "expected_brier": 1.4065502836520713e-08, "multiclass_ece": 8.17512758755079e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/ROUTE FOR APPROVAL": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000263543849295469, "brier": 1.388741175642974e-07, "expected_brier": 1.388741175389896e-07, "multiclass_ece": 0.0002635091246657728, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SCHEDULE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0012560004078333792, "brier": 5.970657823995638e-06, "expected_brier": 5.970657823928249e-06, "multiclass_ece": 0.0012545053100214698, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/invoice_processing/SHORT PAY": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 6.66377925444503e-05, "brier": 1.2797552767170783e-08, "expected_brier": 1.2797552850773997e-08, "multiclass_ece": 6.663459289679707e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/AUTO CLOSE": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0001005878679491487, "brier": 2.0393497376317226e-08, "expected_brier": 2.0393497409306367e-08, "multiclass_ece": 0.00010058276922730869, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK DESTINATION": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010904171260456801, "brier": 2.5373758975796112e-08, "expected_brier": 2.537375898281624e-08, "multiclass_ece": 0.00010903536864570462, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK EGRESS ASSET": { "count": 4, "hard_count": 4, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00014490730032584785, "brier": 5.6700568034538365e-08, "expected_brier": 5.670056796303058e-08, "multiclass_ece": 0.00014489312291465772, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 4, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/BLOCK SENDER": { "count": 3, "hard_count": 3, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00011287179928334396, "brier": 2.9687924733595102e-08, "expected_brier": 2.9687924711178464e-08, "multiclass_ece": 0.00011286437657709314, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE TIER2": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00023274374198972882, "brier": 1.0831408700004126e-07, "expected_brier": 1.0831408703282364e-07, "multiclass_ece": 0.00023271665926616336, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ESCALATE URGENT": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0013836833875753131, "brier": 4.1422660085089114e-06, "expected_brier": 4.142266008533291e-06, "multiclass_ece": 0.0013826467181969893, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/ISOLATE HOST": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0002855956510961497, "brier": 1.6308317046307555e-07, "expected_brier": 1.6308317052349253e-07, "multiclass_ece": 0.00028555487254033896, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/NOTIFY USER": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.904535505125093e-05, "brier": 1.2495348578555861e-08, "expected_brier": 1.2495348533647643e-08, "multiclass_ece": 7.904223104948649e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/PURGE MAILBOXES": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010198711645387576, "brier": 2.3042223271747317e-08, "expected_brier": 2.304222335158812e-08, "multiclass_ece": 0.00010198135543015852, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE FILE": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00010438255928594226, "brier": 2.1789162858668298e-08, "expected_brier": 2.1789162918395277e-08, "multiclass_ece": 0.0001043771116161496, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/QUARANTINE MESSAGE": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.188187484536114e-05, "brier": 1.0335978104854301e-08, "expected_brier": 1.0335978184450312e-08, "multiclass_ece": 7.187929072693944e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REQUIRE REAUTH": { "count": 1, "hard_count": 1, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 7.33652641481388e-05, "brier": 1.0764134229369825e-08, "expected_brier": 1.0764134250607071e-08, "multiclass_ece": 7.336257298296012e-05, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE ACCESS KEY": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00013298919415552046, "brier": 3.777764137765114e-08, "expected_brier": 3.777764134849093e-08, "multiclass_ece": 0.00013297974880099872, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "workflow-controls-v1/security_incidents/REVOKE SESSIONS": { "count": 2, "hard_count": 2, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0003416104805716653, "brier": 3.16814822362037e-07, "expected_brier": 3.168148223764433e-07, "multiclass_ece": 0.0003415312493121325, "ece_bins": 15, "coverage": [ { "threshold": 0.5, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 2, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } } } }