diff --git "a/evaluation/full-data.json" "b/evaluation/full-data.json" new file mode 100644--- /dev/null +++ "b/evaluation/full-data.json" @@ -0,0 +1,5340 @@ +{ + "model": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "checkpoint": { + "model": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "checkpoint_sha256": "a691105dd5751bfcf72d276a6ce075d45054db77ae3272e87b5f07b2bd7e2193", + "temperature": 1.8969118766347646, + "calibration": { + "temperature": 1.8969118766347646, + "split": "calibration", + "n": 512, + "ids_sha256": "6ff7db59c1ae2485179b80316b3cb4f0161df77704692dcab48871a295ab94f1" + }, + "max_length": 4096, + "training_completion": { + "steps": 20204, + "trained_rows_consumed": 80816, + "checkpoint_reload_max_error": 0.0, + "run_sha256": "e5a119cc2ec1f78615375824352f71a68bcc78eb30d28e14bcdbcd442075b42b", + "summary_sha256": "789ae2a85fc3dec77388b11c039f56e3110052b77c55e1eb371d5d5284e72ebc" + } + }, + "package_bound_inference_files": { + "checkpoint/adapter/adapter_config.json": { + "sha256": "3b2d3b4c9e95a7ca0dfb6132a8a76af281363032eeffbbd08ba34fc76cb3b9be", + "bytes": 1170 + }, + "checkpoint/adapter/adapter_model.safetensors": { + "sha256": "f85650a8fb97c6d0ac3e948cdca2f30a0ca6ace8b8a43aebca1c152fe38aeb75", + "bytes": 23613816 + }, + "checkpoint/head.pt": { + "sha256": "229fe9800384e824135e59d1af59bda7346da031aea61640be5f456b9be6ce70", + "bytes": 18189 + }, + "checkpoint/model.json": { + "sha256": "a3a8a691a88b2af64ecb6952a7dd006977fd8c544df09afcbe1e4303b91a8daa", + "bytes": 186 + }, + "checkpoint/temperature.json": { + "sha256": "8cfe40a9f42cc2d0607a3381f2a4e0374cb3842506ac7263f7fff35facfca034", + "bytes": 162 + } + }, + "data_identity": { + "manifest_sha256": "56105dc9fc89ef74919f5beb60bb6ae8c6e17bb95699dab59205f67d8b338d97", + "split_sha256": { + "train": "80a9ec19da9c4065b72f7a9e5ff8d67e62c1d1dd2572dfd296e46bfe2ea5a182", + "calibration": "dd21b26912a6b26a09c3b00ae66a7985ee760e737b94ada4fcc6320406d4db63", + "validation": "1dc97079d4fc034a1ba16315f801b0a4715b30c2a3874ae05aeb60dd1e40e8bb", + "test": "fa8d62775b96c1514f238c2d8a8f8b66515dada67d490978376545296593e29b", + "ood": "9152bff7f3d9c83cce3f1423cc6e0e6b33286b3092b3f544cb18dcc37b35a7ee" + }, + "counts": { + "test": 10532, + "ood": 15920 + }, + "ordered_ids_sha256": { + "test": "98e1af223471c08d84e44c8864e51cefa1bc671687c00bfbf64b7e5a22e8bf52", + "ood": "52fa6cc83c0a172516609257b6048aaece9a3fef75f3f5f694bc6213e5c30a42" + } + }, + "coverage": { + "expected_rows": 26452, + "unique_ids": 26452, + "shard_rows": [ + 6613, + 6613, + 6613, + 6613 + ], + "missing": 0, + "duplicate": 0, + "failed": 0, + "merged_original_order_verified": true + }, + "metrics": { + "test": { + "count": 10532, + "successful": 10532, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 10046, + "accuracy_all_rows_failures_incorrect": 0.9754130997411905, + "expected_accuracy_all_rows_failures_zero": 0.9471531754526437, + "coverage": [ + { + "threshold": 0.5, + "selected": 9945, + "total": 10532, + "coverage": 0.9442650968477022 + }, + { + "threshold": 0.7, + "selected": 9636, + "total": 10532, + "coverage": 0.9149259399924041 + }, + { + "threshold": 0.8, + "selected": 9552, + "total": 10532, + "coverage": 0.906950246866692 + }, + { + "threshold": 0.9, + "selected": 9409, + "total": 10532, + "coverage": 0.893372578807444 + }, + { + "threshold": 0.95, + "selected": 9208, + "total": 10532, + "coverage": 0.8742878845423472 + }, + { + "threshold": 0.99, + "selected": 8674, + "total": 10532, + "coverage": 0.823585263957463 + } + ], + "probability_metrics_all_rows": { + "count": 10532, + "hard_count": 10046, + "accuracy": 0.9754130997411905, + "expected_accuracy": 0.9471531754526437, + "nll": 0.1309471525831089, + "brier": 0.03911290032779779, + "expected_brier": 0.0671638646396812, + "ece_bins": 15, + "multiclass_ece": 0.007706544094318023, + "coverage": [ + { + "threshold": 0.5, + "selected": 9945, + "coverage": 0.9442650968477022, + "accuracy": 0.9885468861846815, + "expected_accuracy": 0.9797553209317914, + "expected_risk": 0.020244679068208593 + }, + { + "threshold": 0.7, + "selected": 9636, + "coverage": 0.9149259399924041, + "accuracy": 0.9936570656129771, + "expected_accuracy": 0.9926490936764909, + "expected_risk": 0.007350906323509099 + }, + { + "threshold": 0.8, + "selected": 9552, + "coverage": 0.906950246866692, + "accuracy": 0.9952859836580766, + "expected_accuracy": 0.9949748743718593, + "expected_risk": 0.005025125628140725 + }, + { + "threshold": 0.9, + "selected": 9409, + "coverage": 0.893372578807444, + "accuracy": 0.9977680943777234, + "expected_accuracy": 0.9977680943777234, + "expected_risk": 0.0022319056222765976 + }, + { + "threshold": 0.95, + "selected": 9208, + "coverage": 0.8742878845423472, + "accuracy": 0.9993483927019983, + "expected_accuracy": 0.9993483927019983, + "expected_risk": 0.0006516072980017329 + }, + { + "threshold": 0.99, + "selected": 8674, + "coverage": 0.823585263957463, + "accuracy": 0.9997694258704174, + "expected_accuracy": 0.9997694258704174, + "expected_risk": 0.0002305741295826058 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 10532, + "hard_count": 10046, + "accuracy": 0.9754130997411905, + "expected_accuracy": 0.9471531754526437, + "nll": 0.1309471525831089, + "brier": 0.03911290032779779, + "expected_brier": 0.0671638646396812, + "ece_bins": 15, + "multiclass_ece": 0.007706544094318023, + "coverage": [ + { + "threshold": 0.5, + "selected": 9945, + "coverage": 0.9442650968477022, + "accuracy": 0.9885468861846815, + "expected_accuracy": 0.9797553209317914, + "expected_risk": 0.020244679068208593 + }, + { + "threshold": 0.7, + "selected": 9636, + "coverage": 0.9149259399924041, + "accuracy": 0.9936570656129771, + "expected_accuracy": 0.9926490936764909, + "expected_risk": 0.007350906323509099 + }, + { + "threshold": 0.8, + "selected": 9552, + "coverage": 0.906950246866692, + "accuracy": 0.9952859836580766, + "expected_accuracy": 0.9949748743718593, + "expected_risk": 0.005025125628140725 + }, + { + "threshold": 0.9, + "selected": 9409, + "coverage": 0.893372578807444, + "accuracy": 0.9977680943777234, + "expected_accuracy": 0.9977680943777234, + "expected_risk": 0.0022319056222765976 + }, + { + "threshold": 0.95, + "selected": 9208, + "coverage": 0.8742878845423472, + "accuracy": 0.9993483927019983, + "expected_accuracy": 0.9993483927019983, + "expected_risk": 0.0006516072980017329 + }, + { + "threshold": 0.99, + "selected": 8674, + "coverage": 0.823585263957463, + "accuracy": 0.9997694258704174, + "expected_accuracy": 0.9997694258704174, + "expected_risk": 0.0002305741295826058 + } + ] + }, + "by_source": { + "customer-control-v1": { + "count": 606, + "successful": 606, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 574, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 0.9722222222222222, + "coverage": [ + { + "threshold": 0.5, + "selected": 596, + "total": 606, + "coverage": 0.9834983498349835 + }, + { + "threshold": 0.7, + "selected": 574, + "total": 606, + "coverage": 0.9471947194719472 + }, + { + "threshold": 0.8, + "selected": 574, + "total": 606, + "coverage": 0.9471947194719472 + }, + { + "threshold": 0.9, + "selected": 574, + "total": 606, + "coverage": 0.9471947194719472 + }, + { + "threshold": 0.95, + "selected": 574, + "total": 606, + "coverage": 0.9471947194719472 + }, + { + "threshold": 0.99, + "selected": 564, + "total": 606, + "coverage": 0.9306930693069307 + } + ], + "probability_metrics_all_rows": { + "count": 606, + "hard_count": 574, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.04199953432734906, + "brier": 6.843414604449945e-05, + "expected_brier": 0.027846211923822282, + "ece_bins": 15, + "multiclass_ece": 0.0019262574800517217, + "coverage": [ + { + "threshold": 0.5, + "selected": 596, + "coverage": 0.9834983498349835, + "accuracy": 1.0, + "expected_accuracy": 0.9815436241610739, + "expected_risk": 0.01845637583892612 + }, + { + "threshold": 0.7, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 564, + "coverage": 0.9306930693069307, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 606, + "hard_count": 574, + "accuracy": 1.0, + "expected_accuracy": 0.9722222222222222, + "nll": 0.04199953432734906, + "brier": 6.843414604449945e-05, + "expected_brier": 0.027846211923822282, + "ece_bins": 15, + "multiclass_ece": 0.0019262574800517217, + "coverage": [ + { + "threshold": 0.5, + "selected": 596, + "coverage": 0.9834983498349835, + "accuracy": 1.0, + "expected_accuracy": 0.9815436241610739, + "expected_risk": 0.01845637583892612 + }, + { + "threshold": 0.7, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 574, + "coverage": 0.9471947194719472, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 564, + "coverage": 0.9306930693069307, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "painting-geometry-v1": { + "count": 3072, + "successful": 3072, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 3072, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 1.0, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "total": 3072, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 3071, + "total": 3072, + "coverage": 0.9996744791666666 + }, + { + "threshold": 0.8, + "selected": 3071, + "total": 3072, + "coverage": 0.9996744791666666 + }, + { + "threshold": 0.9, + "selected": 3071, + "total": 3072, + "coverage": 0.9996744791666666 + }, + { + "threshold": 0.95, + "selected": 3068, + "total": 3072, + "coverage": 0.9986979166666666 + }, + { + "threshold": 0.99, + "selected": 3031, + "total": 3072, + "coverage": 0.9866536458333334 + } + ], + "probability_metrics_all_rows": { + "count": 3072, + "hard_count": 3072, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013689140515597267, + "brier": 7.957005332993052e-05, + "expected_brier": 7.957005332993308e-05, + "ece_bins": 15, + "multiclass_ece": 0.0013415313830535288, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3068, + "coverage": 0.9986979166666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3031, + "coverage": 0.9866536458333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3072, + "hard_count": 3072, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0013689140515597267, + "brier": 7.957005332993052e-05, + "expected_brier": 7.957005332993308e-05, + "ece_bins": 15, + "multiclass_ece": 0.0013415313830535288, + "coverage": [ + { + "threshold": 0.5, + "selected": 3072, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 3071, + "coverage": 0.9996744791666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3068, + "coverage": 0.9986979166666666, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 3031, + "coverage": 0.9866536458333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "reasoning-control-v1": { + "count": 681, + "successful": 681, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 681, + "accuracy_all_rows_failures_incorrect": 0.8722466960352423, + "expected_accuracy_all_rows_failures_zero": 0.8722466960352423, + "coverage": [ + { + "threshold": 0.5, + "selected": 627, + "total": 681, + "coverage": 0.920704845814978 + }, + { + "threshold": 0.7, + "selected": 576, + "total": 681, + "coverage": 0.8458149779735683 + }, + { + "threshold": 0.8, + "selected": 554, + "total": 681, + "coverage": 0.8135095447870778 + }, + { + "threshold": 0.9, + "selected": 512, + "total": 681, + "coverage": 0.7518355359765051 + }, + { + "threshold": 0.95, + "selected": 489, + "total": 681, + "coverage": 0.7180616740088106 + }, + { + "threshold": 0.99, + "selected": 419, + "total": 681, + "coverage": 0.6152716593245228 + } + ], + "probability_metrics_all_rows": { + "count": 681, + "hard_count": 681, + "accuracy": 0.8722466960352423, + "expected_accuracy": 0.8722466960352423, + "nll": 0.30288131001112933, + "brier": 0.1731250583140691, + "expected_brier": 0.1731250583140691, + "ece_bins": 15, + "multiclass_ece": 0.0369016945577258, + "coverage": [ + { + "threshold": 0.5, + "selected": 627, + "coverage": 0.920704845814978, + "accuracy": 0.9138755980861244, + "expected_accuracy": 0.9138755980861244, + "expected_risk": 0.0861244019138756 + }, + { + "threshold": 0.7, + "selected": 576, + "coverage": 0.8458149779735683, + "accuracy": 0.9461805555555556, + "expected_accuracy": 0.9461805555555556, + "expected_risk": 0.05381944444444442 + }, + { + "threshold": 0.8, + "selected": 554, + "coverage": 0.8135095447870778, + "accuracy": 0.9620938628158845, + "expected_accuracy": 0.9620938628158845, + "expected_risk": 0.03790613718411551 + }, + { + "threshold": 0.9, + "selected": 512, + "coverage": 0.7518355359765051, + "accuracy": 0.986328125, + "expected_accuracy": 0.986328125, + "expected_risk": 0.013671875 + }, + { + "threshold": 0.95, + "selected": 489, + "coverage": 0.7180616740088106, + "accuracy": 0.9959100204498977, + "expected_accuracy": 0.9959100204498977, + "expected_risk": 0.004089979550102263 + }, + { + "threshold": 0.99, + "selected": 419, + "coverage": 0.6152716593245228, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 681, + "hard_count": 681, + "accuracy": 0.8722466960352423, + "expected_accuracy": 0.8722466960352423, + "nll": 0.30288131001112933, + "brier": 0.1731250583140691, + "expected_brier": 0.1731250583140691, + "ece_bins": 15, + "multiclass_ece": 0.0369016945577258, + "coverage": [ + { + "threshold": 0.5, + "selected": 627, + "coverage": 0.920704845814978, + "accuracy": 0.9138755980861244, + "expected_accuracy": 0.9138755980861244, + "expected_risk": 0.0861244019138756 + }, + { + "threshold": 0.7, + "selected": 576, + "coverage": 0.8458149779735683, + "accuracy": 0.9461805555555556, + "expected_accuracy": 0.9461805555555556, + "expected_risk": 0.05381944444444442 + }, + { + "threshold": 0.8, + "selected": 554, + "coverage": 0.8135095447870778, + "accuracy": 0.9620938628158845, + "expected_accuracy": 0.9620938628158845, + "expected_risk": 0.03790613718411551 + }, + { + "threshold": 0.9, + "selected": 512, + "coverage": 0.7518355359765051, + "accuracy": 0.986328125, + "expected_accuracy": 0.986328125, + "expected_risk": 0.013671875 + }, + { + "threshold": 0.95, + "selected": 489, + "coverage": 0.7180616740088106, + "accuracy": 0.9959100204498977, + "expected_accuracy": 0.9959100204498977, + "expected_risk": 0.004089979550102263 + }, + { + "threshold": 0.99, + "selected": 419, + "coverage": 0.6152716593245228, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "snake-v1": { + "count": 1392, + "successful": 1392, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1279, + "accuracy_all_rows_failures_incorrect": 0.9976544175136826, + "expected_accuracy_all_rows_failures_zero": 0.9567768199233716, + "coverage": [ + { + "threshold": 0.5, + "selected": 1340, + "total": 1392, + "coverage": 0.9626436781609196 + }, + { + "threshold": 0.7, + "selected": 1268, + "total": 1392, + "coverage": 0.9109195402298851 + }, + { + "threshold": 0.8, + "selected": 1254, + "total": 1392, + "coverage": 0.9008620689655172 + }, + { + "threshold": 0.9, + "selected": 1201, + "total": 1392, + "coverage": 0.8627873563218391 + }, + { + "threshold": 0.95, + "selected": 1152, + "total": 1392, + "coverage": 0.8275862068965517 + }, + { + "threshold": 0.99, + "selected": 1076, + "total": 1392, + "coverage": 0.7729885057471264 + } + ], + "probability_metrics_all_rows": { + "count": 1392, + "hard_count": 1279, + "accuracy": 0.9976544175136826, + "expected_accuracy": 0.9567768199233716, + "nll": 0.08536951007564594, + "brier": 0.01017585055261676, + "expected_brier": 0.05124385821545201, + "ece_bins": 15, + "multiclass_ece": 0.014056450259188788, + "coverage": [ + { + "threshold": 0.5, + "selected": 1340, + "coverage": 0.9626436781609196, + "accuracy": 0.9976525821596244, + "expected_accuracy": 0.9743781094527364, + "expected_risk": 0.025621890547263604 + }, + { + "threshold": 0.7, + "selected": 1268, + "coverage": 0.9109195402298851, + "accuracy": 0.9984164687252574, + "expected_accuracy": 0.9963196635120924, + "expected_risk": 0.003680336487907554 + }, + { + "threshold": 0.8, + "selected": 1254, + "coverage": 0.9008620689655172, + "accuracy": 0.9984012789768185, + "expected_accuracy": 0.9972089314194578, + "expected_risk": 0.002791068580542233 + }, + { + "threshold": 0.9, + "selected": 1201, + "coverage": 0.8627873563218391, + "accuracy": 0.9991673605328892, + "expected_accuracy": 0.9991673605328892, + "expected_risk": 0.0008326394671107629 + }, + { + "threshold": 0.95, + "selected": 1152, + "coverage": 0.8275862068965517, + "accuracy": 0.9991319444444444, + "expected_accuracy": 0.9991319444444444, + "expected_risk": 0.0008680555555555802 + }, + { + "threshold": 0.99, + "selected": 1076, + "coverage": 0.7729885057471264, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1392, + "hard_count": 1279, + "accuracy": 0.9976544175136826, + "expected_accuracy": 0.9567768199233716, + "nll": 0.08536951007564594, + "brier": 0.01017585055261676, + "expected_brier": 0.05124385821545201, + "ece_bins": 15, + "multiclass_ece": 0.014056450259188788, + "coverage": [ + { + "threshold": 0.5, + "selected": 1340, + "coverage": 0.9626436781609196, + "accuracy": 0.9976525821596244, + "expected_accuracy": 0.9743781094527364, + "expected_risk": 0.025621890547263604 + }, + { + "threshold": 0.7, + "selected": 1268, + "coverage": 0.9109195402298851, + "accuracy": 0.9984164687252574, + "expected_accuracy": 0.9963196635120924, + "expected_risk": 0.003680336487907554 + }, + { + "threshold": 0.8, + "selected": 1254, + "coverage": 0.9008620689655172, + "accuracy": 0.9984012789768185, + "expected_accuracy": 0.9972089314194578, + "expected_risk": 0.002791068580542233 + }, + { + "threshold": 0.9, + "selected": 1201, + "coverage": 0.8627873563218391, + "accuracy": 0.9991673605328892, + "expected_accuracy": 0.9991673605328892, + "expected_risk": 0.0008326394671107629 + }, + { + "threshold": 0.95, + "selected": 1152, + "coverage": 0.8275862068965517, + "accuracy": 0.9991319444444444, + "expected_accuracy": 0.9991319444444444, + "expected_risk": 0.0008680555555555802 + }, + { + "threshold": 0.99, + "selected": 1076, + "coverage": 0.7729885057471264, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "tic_tac_toe-v1": { + "count": 472, + "successful": 472, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 256, + "accuracy_all_rows_failures_incorrect": 0.64453125, + "expected_accuracy_all_rows_failures_zero": 0.5206315577078289, + "coverage": [ + { + "threshold": 0.5, + "selected": 174, + "total": 472, + "coverage": 0.3686440677966102 + }, + { + "threshold": 0.7, + "selected": 34, + "total": 472, + "coverage": 0.07203389830508475 + }, + { + "threshold": 0.8, + "selected": 7, + "total": 472, + "coverage": 0.014830508474576272 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 472, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 472, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 472, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 472, + "hard_count": 256, + "accuracy": 0.64453125, + "expected_accuracy": 0.5206315577078289, + "nll": 1.0984003514512244, + "brier": 0.3480926885371523, + "expected_brier": 0.6180684754620918, + "ece_bins": 15, + "multiclass_ece": 0.06619460047533682, + "coverage": [ + { + "threshold": 0.5, + "selected": 174, + "coverage": 0.3686440677966102, + "accuracy": 0.8152173913043478, + "expected_accuracy": 0.632183908045977, + "expected_risk": 0.367816091954023 + }, + { + "threshold": 0.7, + "selected": 34, + "coverage": 0.07203389830508475, + "accuracy": 1.0, + "expected_accuracy": 0.7892156862745098, + "expected_risk": 0.21078431372549022 + }, + { + "threshold": 0.8, + "selected": 7, + "coverage": 0.014830508474576272, + "accuracy": 1.0, + "expected_accuracy": 0.7857142857142857, + "expected_risk": 0.2142857142857143 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 472, + "hard_count": 256, + "accuracy": 0.64453125, + "expected_accuracy": 0.5206315577078289, + "nll": 1.0984003514512244, + "brier": 0.3480926885371523, + "expected_brier": 0.6180684754620918, + "ece_bins": 15, + "multiclass_ece": 0.06619460047533682, + "coverage": [ + { + "threshold": 0.5, + "selected": 174, + "coverage": 0.3686440677966102, + "accuracy": 0.8152173913043478, + "expected_accuracy": 0.632183908045977, + "expected_risk": 0.367816091954023 + }, + { + "threshold": 0.7, + "selected": 34, + "coverage": 0.07203389830508475, + "accuracy": 1.0, + "expected_accuracy": 0.7892156862745098, + "expected_risk": 0.21078431372549022 + }, + { + "threshold": 0.8, + "selected": 7, + "coverage": 0.014830508474576272, + "accuracy": 1.0, + "expected_accuracy": 0.7857142857142857, + "expected_risk": 0.2142857142857143 + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "tile_platformer-v1": { + "count": 202, + "successful": 202, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 202, + "accuracy_all_rows_failures_incorrect": 0.9752475247524752, + "expected_accuracy_all_rows_failures_zero": 0.9752475247524752, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 202, + "total": 202, + "coverage": 1.0 + }, + { + "threshold": 0.95, + "selected": 164, + "total": 202, + "coverage": 0.8118811881188119 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 202, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 202, + "hard_count": 202, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "nll": 0.13968696905117425, + "brier": 0.047840610560813734, + "expected_brier": 0.047840610560813734, + "ece_bins": 15, + "multiclass_ece": 0.035747153706735625, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.7, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.8, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.9, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.95, + "selected": 164, + "coverage": 0.8118811881188119, + "accuracy": 0.9939024390243902, + "expected_accuracy": 0.9939024390243902, + "expected_risk": 0.0060975609756097615 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 202, + "hard_count": 202, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "nll": 0.13968696905117425, + "brier": 0.047840610560813734, + "expected_brier": 0.047840610560813734, + "ece_bins": 15, + "multiclass_ece": 0.035747153706735625, + "coverage": [ + { + "threshold": 0.5, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.7, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.8, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.9, + "selected": 202, + "coverage": 1.0, + "accuracy": 0.9752475247524752, + "expected_accuracy": 0.9752475247524752, + "expected_risk": 0.024752475247524774 + }, + { + "threshold": 0.95, + "selected": 164, + "coverage": 0.8118811881188119, + "accuracy": 0.9939024390243902, + "expected_accuracy": 0.9939024390243902, + "expected_risk": 0.0060975609756097615 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "trex_runner-v1": { + "count": 4, + "successful": 4, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 4, + "accuracy_all_rows_failures_incorrect": 0.75, + "expected_accuracy_all_rows_failures_zero": 0.75, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "total": 4, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 4, + "total": 4, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 4, + "total": 4, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 3, + "total": 4, + "coverage": 0.75 + }, + { + "threshold": 0.95, + "selected": 3, + "total": 4, + "coverage": 0.75 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 4, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.7668877262100418, + "brier": 0.4059017097317489, + "expected_brier": 0.40590170973174894, + "ece_bins": 15, + "multiclass_ece": 0.2272462205045263, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 4, + "hard_count": 4, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "nll": 0.7668877262100418, + "brier": 0.4059017097317489, + "expected_brier": 0.40590170973174894, + "ece_bins": 15, + "multiclass_ece": 0.2272462205045263, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.8, + "selected": 4, + "coverage": 1.0, + "accuracy": 0.75, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.9, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 3, + "coverage": 0.75, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "vizdoom-basic-v1": { + "count": 858, + "successful": 858, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 858, + "accuracy_all_rows_failures_incorrect": 0.9766899766899767, + "expected_accuracy_all_rows_failures_zero": 0.9766899766899767, + "coverage": [ + { + "threshold": 0.5, + "selected": 858, + "total": 858, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 842, + "total": 858, + "coverage": 0.9813519813519813 + }, + { + "threshold": 0.8, + "selected": 823, + "total": 858, + "coverage": 0.9592074592074592 + }, + { + "threshold": 0.9, + "selected": 791, + "total": 858, + "coverage": 0.921911421911422 + }, + { + "threshold": 0.95, + "selected": 718, + "total": 858, + "coverage": 0.8368298368298368 + }, + { + "threshold": 0.99, + "selected": 564, + "total": 858, + "coverage": 0.6573426573426573 + } + ], + "probability_metrics_all_rows": { + "count": 858, + "hard_count": 858, + "accuracy": 0.9766899766899767, + "expected_accuracy": 0.9766899766899767, + "nll": 0.06761322089895994, + "brier": 0.034824071109677196, + "expected_brier": 0.034824071109677196, + "ece_bins": 15, + "multiclass_ece": 0.012718603010498264, + "coverage": [ + { + "threshold": 0.5, + "selected": 858, + "coverage": 1.0, + "accuracy": 0.9766899766899767, + "expected_accuracy": 0.9766899766899767, + "expected_risk": 0.023310023310023298 + }, + { + "threshold": 0.7, + "selected": 842, + "coverage": 0.9813519813519813, + "accuracy": 0.9857482185273159, + "expected_accuracy": 0.9857482185273159, + "expected_risk": 0.014251781472684133 + }, + { + "threshold": 0.8, + "selected": 823, + "coverage": 0.9592074592074592, + "accuracy": 0.9902794653705954, + "expected_accuracy": 0.9902794653705954, + "expected_risk": 0.009720534629404587 + }, + { + "threshold": 0.9, + "selected": 791, + "coverage": 0.921911421911422, + "accuracy": 0.9974715549936789, + "expected_accuracy": 0.9974715549936789, + "expected_risk": 0.0025284450063211006 + }, + { + "threshold": 0.95, + "selected": 718, + "coverage": 0.8368298368298368, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 564, + "coverage": 0.6573426573426573, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 858, + "hard_count": 858, + "accuracy": 0.9766899766899767, + "expected_accuracy": 0.9766899766899767, + "nll": 0.06761322089895994, + "brier": 0.034824071109677196, + "expected_brier": 0.034824071109677196, + "ece_bins": 15, + "multiclass_ece": 0.012718603010498264, + "coverage": [ + { + "threshold": 0.5, + "selected": 858, + "coverage": 1.0, + "accuracy": 0.9766899766899767, + "expected_accuracy": 0.9766899766899767, + "expected_risk": 0.023310023310023298 + }, + { + "threshold": 0.7, + "selected": 842, + "coverage": 0.9813519813519813, + "accuracy": 0.9857482185273159, + "expected_accuracy": 0.9857482185273159, + "expected_risk": 0.014251781472684133 + }, + { + "threshold": 0.8, + "selected": 823, + "coverage": 0.9592074592074592, + "accuracy": 0.9902794653705954, + "expected_accuracy": 0.9902794653705954, + "expected_risk": 0.009720534629404587 + }, + { + "threshold": 0.9, + "selected": 791, + "coverage": 0.921911421911422, + "accuracy": 0.9974715549936789, + "expected_accuracy": 0.9974715549936789, + "expected_risk": 0.0025284450063211006 + }, + { + "threshold": 0.95, + "selected": 718, + "coverage": 0.8368298368298368, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 564, + "coverage": 0.6573426573426573, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "wikispeedia-v1": { + "count": 176, + "successful": 176, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 51, + "accuracy_all_rows_failures_incorrect": 0.45098039215686275, + "expected_accuracy_all_rows_failures_zero": 0.2709042535747081, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "total": 176, + "coverage": 0.017045454545454544 + }, + { + "threshold": 0.7, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 176, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 176, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 176, + "hard_count": 51, + "accuracy": 0.45098039215686275, + "expected_accuracy": 0.2709042535747081, + "nll": 2.1680481914874337, + "brier": 0.31628613125660077, + "expected_brier": 0.8504008170758321, + "ece_bins": 15, + "multiclass_ece": 0.0965942356793526, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 0.017045454545454544, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 176, + "hard_count": 51, + "accuracy": 0.45098039215686275, + "expected_accuracy": 0.2709042535747081, + "nll": 2.1680481914874337, + "brier": 0.31628613125660077, + "expected_brier": 0.8504008170758321, + "ece_bins": 15, + "multiclass_ece": 0.0965942356793526, + "coverage": [ + { + "threshold": 0.5, + "selected": 3, + "coverage": 0.017045454545454544, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 357, + "successful": 357, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 357, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 1.0, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.95, + "selected": 357, + "total": 357, + "coverage": 1.0 + }, + { + "threshold": 0.99, + "selected": 357, + "total": 357, + "coverage": 1.0 + } + ], + "probability_metrics_all_rows": { + "count": 357, + "hard_count": 357, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00044687482748441643, + "brier": 2.946417322061725e-06, + "expected_brier": 2.946417322060909e-06, + "ece_bins": 15, + "multiclass_ece": 0.0004461355302124166, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 357, + "hard_count": 357, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.00044687482748441643, + "brier": 2.946417322061725e-06, + "expected_brier": 2.946417322060909e-06, + "ece_bins": 15, + "multiclass_ece": 0.0004461355302124166, + "coverage": [ + { + "threshold": 0.5, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 357, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/customer_service": { + "count": 456, + "successful": 456, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 456, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 1.0, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.95, + "selected": 456, + "total": 456, + "coverage": 1.0 + }, + { + "threshold": 0.99, + "selected": 455, + "total": 456, + "coverage": 0.9978070175438597 + } + ], + "probability_metrics_all_rows": { + "count": 456, + "hard_count": 456, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004573076748327324, + "brier": 8.826622033700542e-06, + "expected_brier": 8.826622033703088e-06, + "ece_bins": 15, + "multiclass_ece": 0.0004550441009002813, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 455, + "coverage": 0.9978070175438597, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 456, + "hard_count": 456, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.0004573076748327324, + "brier": 8.826622033700542e-06, + "expected_brier": 8.826622033703088e-06, + "ece_bins": 15, + "multiclass_ece": 0.0004550441009002813, + "coverage": [ + { + "threshold": 0.5, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 456, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 455, + "coverage": 0.9978070175438597, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/invoice_processing": { + "count": 930, + "successful": 930, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 930, + "accuracy_all_rows_failures_incorrect": 0.9967741935483871, + "expected_accuracy_all_rows_failures_zero": 0.9967741935483871, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "total": 930, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 927, + "total": 930, + "coverage": 0.9967741935483871 + }, + { + "threshold": 0.8, + "selected": 925, + "total": 930, + "coverage": 0.9946236559139785 + }, + { + "threshold": 0.9, + "selected": 924, + "total": 930, + "coverage": 0.9935483870967742 + }, + { + "threshold": 0.95, + "selected": 924, + "total": 930, + "coverage": 0.9935483870967742 + }, + { + "threshold": 0.99, + "selected": 917, + "total": 930, + "coverage": 0.986021505376344 + } + ], + "probability_metrics_all_rows": { + "count": 930, + "hard_count": 930, + "accuracy": 0.9967741935483871, + "expected_accuracy": 0.9967741935483871, + "nll": 0.005458157001187219, + "brier": 0.003648645648216325, + "expected_brier": 0.0036486456482163225, + "ece_bins": 15, + "multiclass_ece": 0.0037258424516831987, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "coverage": 1.0, + "accuracy": 0.9967741935483871, + "expected_accuracy": 0.9967741935483871, + "expected_risk": 0.003225806451612856 + }, + { + "threshold": 0.7, + "selected": 927, + "coverage": 0.9967741935483871, + "accuracy": 0.9978425026968716, + "expected_accuracy": 0.9978425026968716, + "expected_risk": 0.0021574973031284195 + }, + { + "threshold": 0.8, + "selected": 925, + "coverage": 0.9946236559139785, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 917, + "coverage": 0.986021505376344, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 930, + "hard_count": 930, + "accuracy": 0.9967741935483871, + "expected_accuracy": 0.9967741935483871, + "nll": 0.005458157001187219, + "brier": 0.003648645648216325, + "expected_brier": 0.0036486456482163225, + "ece_bins": 15, + "multiclass_ece": 0.0037258424516831987, + "coverage": [ + { + "threshold": 0.5, + "selected": 930, + "coverage": 1.0, + "accuracy": 0.9967741935483871, + "expected_accuracy": 0.9967741935483871, + "expected_risk": 0.003225806451612856 + }, + { + "threshold": 0.7, + "selected": 927, + "coverage": 0.9967741935483871, + "accuracy": 0.9978425026968716, + "expected_accuracy": 0.9978425026968716, + "expected_risk": 0.0021574973031284195 + }, + { + "threshold": 0.8, + "selected": 925, + "coverage": 0.9946236559139785, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 924, + "coverage": 0.9935483870967742, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 917, + "coverage": 0.986021505376344, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/security_incidents": { + "count": 1326, + "successful": 1326, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1326, + "accuracy_all_rows_failures_incorrect": 0.9932126696832579, + "expected_accuracy_all_rows_failures_zero": 0.9932126696832579, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "total": 1326, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 1325, + "total": 1326, + "coverage": 0.9992458521870287 + }, + { + "threshold": 0.8, + "selected": 1325, + "total": 1326, + "coverage": 0.9992458521870287 + }, + { + "threshold": 0.9, + "selected": 1318, + "total": 1326, + "coverage": 0.9939668174962293 + }, + { + "threshold": 0.95, + "selected": 1303, + "total": 1326, + "coverage": 0.9826546003016591 + }, + { + "threshold": 0.99, + "selected": 1291, + "total": 1326, + "coverage": 0.973604826546003 + } + ], + "probability_metrics_all_rows": { + "count": 1326, + "hard_count": 1326, + "accuracy": 0.9932126696832579, + "expected_accuracy": 0.9932126696832579, + "nll": 0.022337704900579736, + "brier": 0.011355443994287867, + "expected_brier": 0.011355443994287865, + "ece_bins": 15, + "multiclass_ece": 0.004366133269192267, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "coverage": 1.0, + "accuracy": 0.9932126696832579, + "expected_accuracy": 0.9932126696832579, + "expected_risk": 0.006787330316742057 + }, + { + "threshold": 0.7, + "selected": 1325, + "coverage": 0.9992458521870287, + "accuracy": 0.9939622641509434, + "expected_accuracy": 0.9939622641509434, + "expected_risk": 0.006037735849056647 + }, + { + "threshold": 0.8, + "selected": 1325, + "coverage": 0.9992458521870287, + "accuracy": 0.9939622641509434, + "expected_accuracy": 0.9939622641509434, + "expected_risk": 0.006037735849056647 + }, + { + "threshold": 0.9, + "selected": 1318, + "coverage": 0.9939668174962293, + "accuracy": 0.9954476479514416, + "expected_accuracy": 0.9954476479514416, + "expected_risk": 0.004552352048558417 + }, + { + "threshold": 0.95, + "selected": 1303, + "coverage": 0.9826546003016591, + "accuracy": 0.9984650805832693, + "expected_accuracy": 0.9984650805832693, + "expected_risk": 0.0015349194167306734 + }, + { + "threshold": 0.99, + "selected": 1291, + "coverage": 0.973604826546003, + "accuracy": 0.9984508133230054, + "expected_accuracy": 0.9984508133230054, + "expected_risk": 0.0015491866769945517 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1326, + "hard_count": 1326, + "accuracy": 0.9932126696832579, + "expected_accuracy": 0.9932126696832579, + "nll": 0.022337704900579736, + "brier": 0.011355443994287867, + "expected_brier": 0.011355443994287865, + "ece_bins": 15, + "multiclass_ece": 0.004366133269192267, + "coverage": [ + { + "threshold": 0.5, + "selected": 1326, + "coverage": 1.0, + "accuracy": 0.9932126696832579, + "expected_accuracy": 0.9932126696832579, + "expected_risk": 0.006787330316742057 + }, + { + "threshold": 0.7, + "selected": 1325, + "coverage": 0.9992458521870287, + "accuracy": 0.9939622641509434, + "expected_accuracy": 0.9939622641509434, + "expected_risk": 0.006037735849056647 + }, + { + "threshold": 0.8, + "selected": 1325, + "coverage": 0.9992458521870287, + "accuracy": 0.9939622641509434, + "expected_accuracy": 0.9939622641509434, + "expected_risk": 0.006037735849056647 + }, + { + "threshold": 0.9, + "selected": 1318, + "coverage": 0.9939668174962293, + "accuracy": 0.9954476479514416, + "expected_accuracy": 0.9954476479514416, + "expected_risk": 0.004552352048558417 + }, + { + "threshold": 0.95, + "selected": 1303, + "coverage": 0.9826546003016591, + "accuracy": 0.9984650805832693, + "expected_accuracy": 0.9984650805832693, + "expected_risk": 0.0015349194167306734 + }, + { + "threshold": 0.99, + "selected": 1291, + "coverage": 0.973604826546003, + "accuracy": 0.9984508133230054, + "expected_accuracy": 0.9984508133230054, + "expected_risk": 0.0015491866769945517 + } + ] + } + } + }, + "by_kind": { + "choice": { + "count": 2584, + "successful": 2584, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2119, + "accuracy_all_rows_failures_incorrect": 0.9188296366210477, + "expected_accuracy_all_rows_failures_zero": 0.8180149292571892, + "coverage": [ + { + "threshold": 0.5, + "selected": 2050, + "total": 2584, + "coverage": 0.7933436532507739 + }, + { + "threshold": 0.7, + "selected": 1783, + "total": 2584, + "coverage": 0.690015479876161 + }, + { + "threshold": 0.8, + "selected": 1715, + "total": 2584, + "coverage": 0.6636996904024768 + }, + { + "threshold": 0.9, + "selected": 1620, + "total": 2584, + "coverage": 0.6269349845201239 + }, + { + "threshold": 0.95, + "selected": 1458, + "total": 2584, + "coverage": 0.5642414860681114 + }, + { + "threshold": 0.99, + "selected": 1056, + "total": 2584, + "coverage": 0.4086687306501548 + } + ], + "probability_metrics_all_rows": { + "count": 2584, + "hard_count": 2119, + "accuracy": 0.9188296366210477, + "expected_accuracy": 0.8180149292571892, + "nll": 0.45266508836146735, + "brier": 0.11881345254759784, + "expected_brier": 0.228759049606198, + "ece_bins": 15, + "multiclass_ece": 0.027110901433906452, + "coverage": [ + { + "threshold": 0.5, + "selected": 2050, + "coverage": 0.7933436532507739, + "accuracy": 0.9651898734177216, + "expected_accuracy": 0.9271544715447155, + "expected_risk": 0.07284552845528447 + }, + { + "threshold": 0.7, + "selected": 1783, + "coverage": 0.690015479876161, + "accuracy": 0.985827664399093, + "expected_accuracy": 0.9804636380631894, + "expected_risk": 0.01953636193681063 + }, + { + "threshold": 0.8, + "selected": 1715, + "coverage": 0.6636996904024768, + "accuracy": 0.9900526623756583, + "expected_accuracy": 0.9883381924198251, + "expected_risk": 0.011661807580174877 + }, + { + "threshold": 0.9, + "selected": 1620, + "coverage": 0.6269349845201239, + "accuracy": 0.9950617283950617, + "expected_accuracy": 0.9950617283950617, + "expected_risk": 0.004938271604938316 + }, + { + "threshold": 0.95, + "selected": 1458, + "coverage": 0.5642414860681114, + "accuracy": 0.9986282578875172, + "expected_accuracy": 0.9986282578875172, + "expected_risk": 0.00137174211248281 + }, + { + "threshold": 0.99, + "selected": 1056, + "coverage": 0.4086687306501548, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 2584, + "hard_count": 2119, + "accuracy": 0.9188296366210477, + "expected_accuracy": 0.8180149292571892, + "nll": 0.45266508836146735, + "brier": 0.11881345254759784, + "expected_brier": 0.228759049606198, + "ece_bins": 15, + "multiclass_ece": 0.027110901433906452, + "coverage": [ + { + "threshold": 0.5, + "selected": 2050, + "coverage": 0.7933436532507739, + "accuracy": 0.9651898734177216, + "expected_accuracy": 0.9271544715447155, + "expected_risk": 0.07284552845528447 + }, + { + "threshold": 0.7, + "selected": 1783, + "coverage": 0.690015479876161, + "accuracy": 0.985827664399093, + "expected_accuracy": 0.9804636380631894, + "expected_risk": 0.01953636193681063 + }, + { + "threshold": 0.8, + "selected": 1715, + "coverage": 0.6636996904024768, + "accuracy": 0.9900526623756583, + "expected_accuracy": 0.9883381924198251, + "expected_risk": 0.011661807580174877 + }, + { + "threshold": 0.9, + "selected": 1620, + "coverage": 0.6269349845201239, + "accuracy": 0.9950617283950617, + "expected_accuracy": 0.9950617283950617, + "expected_risk": 0.004938271604938316 + }, + { + "threshold": 0.95, + "selected": 1458, + "coverage": 0.5642414860681114, + "accuracy": 0.9986282578875172, + "expected_accuracy": 0.9986282578875172, + "expected_risk": 0.00137174211248281 + }, + { + "threshold": 0.99, + "selected": 1056, + "coverage": 0.4086687306501548, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "noul": { + "count": 6364, + "successful": 6364, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 6364, + "accuracy_all_rows_failures_incorrect": 0.9951288497800126, + "expected_accuracy_all_rows_failures_zero": 0.9951288497800126, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "total": 6364, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 6347, + "total": 6364, + "coverage": 0.9973287240729101 + }, + { + "threshold": 0.8, + "selected": 6338, + "total": 6364, + "coverage": 0.9959145191703331 + }, + { + "threshold": 0.9, + "selected": 6301, + "total": 6364, + "coverage": 0.990100565681961 + }, + { + "threshold": 0.95, + "selected": 6274, + "total": 6364, + "coverage": 0.98585795097423 + }, + { + "threshold": 0.99, + "selected": 6176, + "total": 6364, + "coverage": 0.9704588309239472 + } + ], + "probability_metrics_all_rows": { + "count": 6364, + "hard_count": 6364, + "accuracy": 0.9951288497800126, + "expected_accuracy": 0.9951288497800126, + "nll": 0.014164294737203246, + "brier": 0.007765035196910086, + "expected_brier": 0.007765035196910086, + "ece_bins": 15, + "multiclass_ece": 0.002491467968884566, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "coverage": 1.0, + "accuracy": 0.9951288497800126, + "expected_accuracy": 0.9951288497800126, + "expected_risk": 0.004871150219987408 + }, + { + "threshold": 0.7, + "selected": 6347, + "coverage": 0.9973287240729101, + "accuracy": 0.996061131243107, + "expected_accuracy": 0.996061131243107, + "expected_risk": 0.003938868756892977 + }, + { + "threshold": 0.8, + "selected": 6338, + "coverage": 0.9959145191703331, + "accuracy": 0.9965288734616599, + "expected_accuracy": 0.9965288734616599, + "expected_risk": 0.0034711265383401413 + }, + { + "threshold": 0.9, + "selected": 6301, + "coverage": 0.990100565681961, + "accuracy": 0.9980955403904143, + "expected_accuracy": 0.9980955403904143, + "expected_risk": 0.0019044596095857402 + }, + { + "threshold": 0.95, + "selected": 6274, + "coverage": 0.98585795097423, + "accuracy": 0.9993624481989162, + "expected_accuracy": 0.9993624481989162, + "expected_risk": 0.0006375518010838244 + }, + { + "threshold": 0.99, + "selected": 6176, + "coverage": 0.9704588309239472, + "accuracy": 0.9996761658031088, + "expected_accuracy": 0.9996761658031088, + "expected_risk": 0.00032383419689119286 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 6364, + "hard_count": 6364, + "accuracy": 0.9951288497800126, + "expected_accuracy": 0.9951288497800126, + "nll": 0.014164294737203246, + "brier": 0.007765035196910086, + "expected_brier": 0.007765035196910086, + "ece_bins": 15, + "multiclass_ece": 0.002491467968884566, + "coverage": [ + { + "threshold": 0.5, + "selected": 6364, + "coverage": 1.0, + "accuracy": 0.9951288497800126, + "expected_accuracy": 0.9951288497800126, + "expected_risk": 0.004871150219987408 + }, + { + "threshold": 0.7, + "selected": 6347, + "coverage": 0.9973287240729101, + "accuracy": 0.996061131243107, + "expected_accuracy": 0.996061131243107, + "expected_risk": 0.003938868756892977 + }, + { + "threshold": 0.8, + "selected": 6338, + "coverage": 0.9959145191703331, + "accuracy": 0.9965288734616599, + "expected_accuracy": 0.9965288734616599, + "expected_risk": 0.0034711265383401413 + }, + { + "threshold": 0.9, + "selected": 6301, + "coverage": 0.990100565681961, + "accuracy": 0.9980955403904143, + "expected_accuracy": 0.9980955403904143, + "expected_risk": 0.0019044596095857402 + }, + { + "threshold": 0.95, + "selected": 6274, + "coverage": 0.98585795097423, + "accuracy": 0.9993624481989162, + "expected_accuracy": 0.9993624481989162, + "expected_risk": 0.0006375518010838244 + }, + { + "threshold": 0.99, + "selected": 6176, + "coverage": 0.9704588309239472, + "accuracy": 0.9996761658031088, + "expected_accuracy": 0.9996761658031088, + "expected_risk": 0.00032383419689119286 + } + ] + } + }, + "score": { + "count": 1584, + "successful": 1584, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1563, + "accuracy_all_rows_failures_incorrect": 0.9718490083173384, + "expected_accuracy_all_rows_failures_zero": 0.9650673400673401, + "coverage": [ + { + "threshold": 0.5, + "selected": 1531, + "total": 1584, + "coverage": 0.9665404040404041 + }, + { + "threshold": 0.7, + "selected": 1506, + "total": 1584, + "coverage": 0.9507575757575758 + }, + { + "threshold": 0.8, + "selected": 1499, + "total": 1584, + "coverage": 0.9463383838383839 + }, + { + "threshold": 0.9, + "selected": 1488, + "total": 1584, + "coverage": 0.9393939393939394 + }, + { + "threshold": 0.95, + "selected": 1476, + "total": 1584, + "coverage": 0.9318181818181818 + }, + { + "threshold": 0.99, + "selected": 1442, + "total": 1584, + "coverage": 0.9103535353535354 + } + ], + "probability_metrics_all_rows": { + "count": 1584, + "hard_count": 1563, + "accuracy": 0.9718490083173384, + "expected_accuracy": 0.9650673400673401, + "nll": 0.07532023419931186, + "brier": 0.03504193237136221, + "expected_brier": 0.042196814526244374, + "ece_bins": 15, + "multiclass_ece": 0.010182639374983222, + "coverage": [ + { + "threshold": 0.5, + "selected": 1531, + "coverage": 0.9665404040404041, + "accuracy": 0.990125082290981, + "expected_accuracy": 0.9862834748530372, + "expected_risk": 0.013716525146962755 + }, + { + "threshold": 0.7, + "selected": 1506, + "coverage": 0.9507575757575758, + "accuracy": 0.9926958831341302, + "expected_accuracy": 0.9926958831341302, + "expected_risk": 0.007304116865869847 + }, + { + "threshold": 0.8, + "selected": 1499, + "coverage": 0.9463383838383839, + "accuracy": 0.9959973315543695, + "expected_accuracy": 0.9959973315543695, + "expected_risk": 0.004002668445630464 + }, + { + "threshold": 0.9, + "selected": 1488, + "coverage": 0.9393939393939394, + "accuracy": 0.9993279569892473, + "expected_accuracy": 0.9993279569892473, + "expected_risk": 0.0006720430107527431 + }, + { + "threshold": 0.95, + "selected": 1476, + "coverage": 0.9318181818181818, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1442, + "coverage": 0.9103535353535354, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1584, + "hard_count": 1563, + "accuracy": 0.9718490083173384, + "expected_accuracy": 0.9650673400673401, + "nll": 0.07532023419931186, + "brier": 0.03504193237136221, + "expected_brier": 0.042196814526244374, + "ece_bins": 15, + "multiclass_ece": 0.010182639374983222, + "coverage": [ + { + "threshold": 0.5, + "selected": 1531, + "coverage": 0.9665404040404041, + "accuracy": 0.990125082290981, + "expected_accuracy": 0.9862834748530372, + "expected_risk": 0.013716525146962755 + }, + { + "threshold": 0.7, + "selected": 1506, + "coverage": 0.9507575757575758, + "accuracy": 0.9926958831341302, + "expected_accuracy": 0.9926958831341302, + "expected_risk": 0.007304116865869847 + }, + { + "threshold": 0.8, + "selected": 1499, + "coverage": 0.9463383838383839, + "accuracy": 0.9959973315543695, + "expected_accuracy": 0.9959973315543695, + "expected_risk": 0.004002668445630464 + }, + { + "threshold": 0.9, + "selected": 1488, + "coverage": 0.9393939393939394, + "accuracy": 0.9993279569892473, + "expected_accuracy": 0.9993279569892473, + "expected_risk": 0.0006720430107527431 + }, + { + "threshold": 0.95, + "selected": 1476, + "coverage": 0.9318181818181818, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 1442, + "coverage": 0.9103535353535354, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + } + } + }, + "ood": { + "count": 15920, + "successful": 15920, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 15446, + "accuracy_all_rows_failures_incorrect": 0.9196555742587078, + "expected_accuracy_all_rows_failures_zero": 0.9040437668138673, + "coverage": [ + { + "threshold": 0.5, + "selected": 15349, + "total": 15920, + "coverage": 0.9641331658291458 + }, + { + "threshold": 0.7, + "selected": 14632, + "total": 15920, + "coverage": 0.9190954773869346 + }, + { + "threshold": 0.8, + "selected": 14183, + "total": 15920, + "coverage": 0.8908919597989949 + }, + { + "threshold": 0.9, + "selected": 13442, + "total": 15920, + "coverage": 0.8443467336683417 + }, + { + "threshold": 0.95, + "selected": 12751, + "total": 15920, + "coverage": 0.8009422110552764 + }, + { + "threshold": 0.99, + "selected": 10671, + "total": 15920, + "coverage": 0.6702889447236181 + } + ], + "probability_metrics_all_rows": { + "count": 15920, + "hard_count": 15446, + "accuracy": 0.9196555742587078, + "expected_accuracy": 0.9040437668138673, + "nll": 0.2994407995801986, + "brier": 0.1266465263060649, + "expected_brier": 0.1439772659530734, + "ece_bins": 15, + "multiclass_ece": 0.037398104632606045, + "coverage": [ + { + "threshold": 0.5, + "selected": 15349, + "coverage": 0.9641331658291458, + "accuracy": 0.9263081777544044, + "expected_accuracy": 0.9224922361934546, + "expected_risk": 0.07750776380654545 + }, + { + "threshold": 0.7, + "selected": 14632, + "coverage": 0.9190954773869346, + "accuracy": 0.939354574046219, + "expected_accuracy": 0.9391630216876253, + "expected_risk": 0.06083697831237467 + }, + { + "threshold": 0.8, + "selected": 14183, + "coverage": 0.8908919597989949, + "accuracy": 0.9484557890283458, + "expected_accuracy": 0.9484241697807234, + "expected_risk": 0.051575830219276586 + }, + { + "threshold": 0.9, + "selected": 13442, + "coverage": 0.8443467336683417, + "accuracy": 0.9607172085410312, + "expected_accuracy": 0.9606829340871894, + "expected_risk": 0.03931706591281059 + }, + { + "threshold": 0.95, + "selected": 12751, + "coverage": 0.8009422110552764, + "accuracy": 0.9714509803921568, + "expected_accuracy": 0.971414006744569, + "expected_risk": 0.028585993255430986 + }, + { + "threshold": 0.99, + "selected": 10671, + "coverage": 0.6702889447236181, + "accuracy": 0.9867866179364633, + "expected_accuracy": 0.9867866179364633, + "expected_risk": 0.013213382063536727 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 15920, + "hard_count": 15446, + "accuracy": 0.9196555742587078, + "expected_accuracy": 0.9040437668138673, + "nll": 0.2994407995801986, + "brier": 0.1266465263060649, + "expected_brier": 0.1439772659530734, + "ece_bins": 15, + "multiclass_ece": 0.037398104632606045, + "coverage": [ + { + "threshold": 0.5, + "selected": 15349, + "coverage": 0.9641331658291458, + "accuracy": 0.9263081777544044, + "expected_accuracy": 0.9224922361934546, + "expected_risk": 0.07750776380654545 + }, + { + "threshold": 0.7, + "selected": 14632, + "coverage": 0.9190954773869346, + "accuracy": 0.939354574046219, + "expected_accuracy": 0.9391630216876253, + "expected_risk": 0.06083697831237467 + }, + { + "threshold": 0.8, + "selected": 14183, + "coverage": 0.8908919597989949, + "accuracy": 0.9484557890283458, + "expected_accuracy": 0.9484241697807234, + "expected_risk": 0.051575830219276586 + }, + { + "threshold": 0.9, + "selected": 13442, + "coverage": 0.8443467336683417, + "accuracy": 0.9607172085410312, + "expected_accuracy": 0.9606829340871894, + "expected_risk": 0.03931706591281059 + }, + { + "threshold": 0.95, + "selected": 12751, + "coverage": 0.8009422110552764, + "accuracy": 0.9714509803921568, + "expected_accuracy": 0.971414006744569, + "expected_risk": 0.028585993255430986 + }, + { + "threshold": 0.99, + "selected": 10671, + "coverage": 0.6702889447236181, + "accuracy": 0.9867866179364633, + "expected_accuracy": 0.9867866179364633, + "expected_risk": 0.013213382063536727 + } + ] + }, + "by_source": { + "customer-control-v1": { + "count": 600, + "successful": 600, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 560, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 0.9633333333333334, + "coverage": [ + { + "threshold": 0.5, + "selected": 586, + "total": 600, + "coverage": 0.9766666666666667 + }, + { + "threshold": 0.7, + "selected": 559, + "total": 600, + "coverage": 0.9316666666666666 + }, + { + "threshold": 0.8, + "selected": 555, + "total": 600, + "coverage": 0.925 + }, + { + "threshold": 0.9, + "selected": 548, + "total": 600, + "coverage": 0.9133333333333333 + }, + { + "threshold": 0.95, + "selected": 540, + "total": 600, + "coverage": 0.9 + }, + { + "threshold": 0.99, + "selected": 512, + "total": 600, + "coverage": 0.8533333333333334 + } + ], + "probability_metrics_all_rows": { + "count": 600, + "hard_count": 560, + "accuracy": 1.0, + "expected_accuracy": 0.9633333333333334, + "nll": 0.06367998388489432, + "brier": 0.002432497344975918, + "expected_brier": 0.03909916401164259, + "ece_bins": 15, + "multiclass_ece": 0.007595238345599102, + "coverage": [ + { + "threshold": 0.5, + "selected": 586, + "coverage": 0.9766666666666667, + "accuracy": 1.0, + "expected_accuracy": 0.976962457337884, + "expected_risk": 0.023037542662115995 + }, + { + "threshold": 0.7, + "selected": 559, + "coverage": 0.9316666666666666, + "accuracy": 1.0, + "expected_accuracy": 0.9991055456171736, + "expected_risk": 0.0008944543828264262 + }, + { + "threshold": 0.8, + "selected": 555, + "coverage": 0.925, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 548, + "coverage": 0.9133333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 540, + "coverage": 0.9, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 512, + "coverage": 0.8533333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 600, + "hard_count": 560, + "accuracy": 1.0, + "expected_accuracy": 0.9633333333333334, + "nll": 0.06367998388489432, + "brier": 0.002432497344975918, + "expected_brier": 0.03909916401164259, + "ece_bins": 15, + "multiclass_ece": 0.007595238345599102, + "coverage": [ + { + "threshold": 0.5, + "selected": 586, + "coverage": 0.9766666666666667, + "accuracy": 1.0, + "expected_accuracy": 0.976962457337884, + "expected_risk": 0.023037542662115995 + }, + { + "threshold": 0.7, + "selected": 559, + "coverage": 0.9316666666666666, + "accuracy": 1.0, + "expected_accuracy": 0.9991055456171736, + "expected_risk": 0.0008944543828264262 + }, + { + "threshold": 0.8, + "selected": 555, + "coverage": 0.925, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 548, + "coverage": 0.9133333333333333, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 540, + "coverage": 0.9, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 512, + "coverage": 0.8533333333333334, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "painting-geometry-v1": { + "count": 6912, + "successful": 6912, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 6912, + "accuracy_all_rows_failures_incorrect": 0.8629918981481481, + "expected_accuracy_all_rows_failures_zero": 0.8629918981481481, + "coverage": [ + { + "threshold": 0.5, + "selected": 6842, + "total": 6912, + "coverage": 0.9898726851851852 + }, + { + "threshold": 0.7, + "selected": 6420, + "total": 6912, + "coverage": 0.9288194444444444 + }, + { + "threshold": 0.8, + "selected": 6102, + "total": 6912, + "coverage": 0.8828125 + }, + { + "threshold": 0.9, + "selected": 5666, + "total": 6912, + "coverage": 0.8197337962962963 + }, + { + "threshold": 0.95, + "selected": 5246, + "total": 6912, + "coverage": 0.7589699074074074 + }, + { + "threshold": 0.99, + "selected": 3677, + "total": 6912, + "coverage": 0.5319733796296297 + } + ], + "probability_metrics_all_rows": { + "count": 6912, + "hard_count": 6912, + "accuracy": 0.8629918981481481, + "expected_accuracy": 0.8629918981481481, + "nll": 0.4601822966946938, + "brier": 0.22129407318457964, + "expected_brier": 0.22129407318457964, + "ece_bins": 15, + "multiclass_ece": 0.0793900158186277, + "coverage": [ + { + "threshold": 0.5, + "selected": 6842, + "coverage": 0.9898726851851852, + "accuracy": 0.8661210172464192, + "expected_accuracy": 0.8661210172464192, + "expected_risk": 0.1338789827535808 + }, + { + "threshold": 0.7, + "selected": 6420, + "coverage": 0.9288194444444444, + "accuracy": 0.8848909657320873, + "expected_accuracy": 0.8848909657320873, + "expected_risk": 0.11510903426791275 + }, + { + "threshold": 0.8, + "selected": 6102, + "coverage": 0.8828125, + "accuracy": 0.8982300884955752, + "expected_accuracy": 0.8982300884955752, + "expected_risk": 0.10176991150442483 + }, + { + "threshold": 0.9, + "selected": 5666, + "coverage": 0.8197337962962963, + "accuracy": 0.9200494175785386, + "expected_accuracy": 0.9200494175785386, + "expected_risk": 0.07995058242146136 + }, + { + "threshold": 0.95, + "selected": 5246, + "coverage": 0.7589699074074074, + "accuracy": 0.9412886008387342, + "expected_accuracy": 0.9412886008387342, + "expected_risk": 0.058711399161265776 + }, + { + "threshold": 0.99, + "selected": 3677, + "coverage": 0.5319733796296297, + "accuracy": 0.9709001903725863, + "expected_accuracy": 0.9709001903725863, + "expected_risk": 0.029099809627413653 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 6912, + "hard_count": 6912, + "accuracy": 0.8629918981481481, + "expected_accuracy": 0.8629918981481481, + "nll": 0.4601822966946938, + "brier": 0.22129407318457964, + "expected_brier": 0.22129407318457964, + "ece_bins": 15, + "multiclass_ece": 0.0793900158186277, + "coverage": [ + { + "threshold": 0.5, + "selected": 6842, + "coverage": 0.9898726851851852, + "accuracy": 0.8661210172464192, + "expected_accuracy": 0.8661210172464192, + "expected_risk": 0.1338789827535808 + }, + { + "threshold": 0.7, + "selected": 6420, + "coverage": 0.9288194444444444, + "accuracy": 0.8848909657320873, + "expected_accuracy": 0.8848909657320873, + "expected_risk": 0.11510903426791275 + }, + { + "threshold": 0.8, + "selected": 6102, + "coverage": 0.8828125, + "accuracy": 0.8982300884955752, + "expected_accuracy": 0.8982300884955752, + "expected_risk": 0.10176991150442483 + }, + { + "threshold": 0.9, + "selected": 5666, + "coverage": 0.8197337962962963, + "accuracy": 0.9200494175785386, + "expected_accuracy": 0.9200494175785386, + "expected_risk": 0.07995058242146136 + }, + { + "threshold": 0.95, + "selected": 5246, + "coverage": 0.7589699074074074, + "accuracy": 0.9412886008387342, + "expected_accuracy": 0.9412886008387342, + "expected_risk": 0.058711399161265776 + }, + { + "threshold": 0.99, + "selected": 3677, + "coverage": 0.5319733796296297, + "accuracy": 0.9709001903725863, + "expected_accuracy": 0.9709001903725863, + "expected_risk": 0.029099809627413653 + } + ] + } + }, + "reasoning-control-v1": { + "count": 750, + "successful": 750, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 750, + "accuracy_all_rows_failures_incorrect": 0.736, + "expected_accuracy_all_rows_failures_zero": 0.736, + "coverage": [ + { + "threshold": 0.5, + "selected": 644, + "total": 750, + "coverage": 0.8586666666666667 + }, + { + "threshold": 0.7, + "selected": 542, + "total": 750, + "coverage": 0.7226666666666667 + }, + { + "threshold": 0.8, + "selected": 492, + "total": 750, + "coverage": 0.656 + }, + { + "threshold": 0.9, + "selected": 414, + "total": 750, + "coverage": 0.552 + }, + { + "threshold": 0.95, + "selected": 373, + "total": 750, + "coverage": 0.49733333333333335 + }, + { + "threshold": 0.99, + "selected": 245, + "total": 750, + "coverage": 0.32666666666666666 + } + ], + "probability_metrics_all_rows": { + "count": 750, + "hard_count": 750, + "accuracy": 0.736, + "expected_accuracy": 0.736, + "nll": 0.6801700551809211, + "brier": 0.3601332438976597, + "expected_brier": 0.3601332438976597, + "ece_bins": 15, + "multiclass_ece": 0.0941763492512841, + "coverage": [ + { + "threshold": 0.5, + "selected": 644, + "coverage": 0.8586666666666667, + "accuracy": 0.8012422360248447, + "expected_accuracy": 0.8012422360248447, + "expected_risk": 0.19875776397515532 + }, + { + "threshold": 0.7, + "selected": 542, + "coverage": 0.7226666666666667, + "accuracy": 0.8431734317343174, + "expected_accuracy": 0.8431734317343174, + "expected_risk": 0.15682656826568264 + }, + { + "threshold": 0.8, + "selected": 492, + "coverage": 0.656, + "accuracy": 0.8760162601626016, + "expected_accuracy": 0.8760162601626016, + "expected_risk": 0.12398373983739841 + }, + { + "threshold": 0.9, + "selected": 414, + "coverage": 0.552, + "accuracy": 0.9106280193236715, + "expected_accuracy": 0.9106280193236715, + "expected_risk": 0.08937198067632846 + }, + { + "threshold": 0.95, + "selected": 373, + "coverage": 0.49733333333333335, + "accuracy": 0.9276139410187667, + "expected_accuracy": 0.9276139410187667, + "expected_risk": 0.07238605898123329 + }, + { + "threshold": 0.99, + "selected": 245, + "coverage": 0.32666666666666666, + "accuracy": 0.9591836734693877, + "expected_accuracy": 0.9591836734693877, + "expected_risk": 0.04081632653061229 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 750, + "hard_count": 750, + "accuracy": 0.736, + "expected_accuracy": 0.736, + "nll": 0.6801700551809211, + "brier": 0.3601332438976597, + "expected_brier": 0.3601332438976597, + "ece_bins": 15, + "multiclass_ece": 0.0941763492512841, + "coverage": [ + { + "threshold": 0.5, + "selected": 644, + "coverage": 0.8586666666666667, + "accuracy": 0.8012422360248447, + "expected_accuracy": 0.8012422360248447, + "expected_risk": 0.19875776397515532 + }, + { + "threshold": 0.7, + "selected": 542, + "coverage": 0.7226666666666667, + "accuracy": 0.8431734317343174, + "expected_accuracy": 0.8431734317343174, + "expected_risk": 0.15682656826568264 + }, + { + "threshold": 0.8, + "selected": 492, + "coverage": 0.656, + "accuracy": 0.8760162601626016, + "expected_accuracy": 0.8760162601626016, + "expected_risk": 0.12398373983739841 + }, + { + "threshold": 0.9, + "selected": 414, + "coverage": 0.552, + "accuracy": 0.9106280193236715, + "expected_accuracy": 0.9106280193236715, + "expected_risk": 0.08937198067632846 + }, + { + "threshold": 0.95, + "selected": 373, + "coverage": 0.49733333333333335, + "accuracy": 0.9276139410187667, + "expected_accuracy": 0.9276139410187667, + "expected_risk": 0.07238605898123329 + }, + { + "threshold": 0.99, + "selected": 245, + "coverage": 0.32666666666666666, + "accuracy": 0.9591836734693877, + "expected_accuracy": 0.9591836734693877, + "expected_risk": 0.04081632653061229 + } + ] + } + }, + "snake-v1": { + "count": 3464, + "successful": 3464, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 3192, + "accuracy_all_rows_failures_incorrect": 0.9899749373433584, + "expected_accuracy_all_rows_failures_zero": 0.9514049268668205, + "coverage": [ + { + "threshold": 0.5, + "selected": 3299, + "total": 3464, + "coverage": 0.9523672055427251 + }, + { + "threshold": 0.7, + "selected": 3151, + "total": 3464, + "coverage": 0.9096420323325635 + }, + { + "threshold": 0.8, + "selected": 3094, + "total": 3464, + "coverage": 0.8931870669745958 + }, + { + "threshold": 0.9, + "selected": 2929, + "total": 3464, + "coverage": 0.8455542725173211 + }, + { + "threshold": 0.95, + "selected": 2779, + "total": 3464, + "coverage": 0.8022517321016166 + }, + { + "threshold": 0.99, + "selected": 2592, + "total": 3464, + "coverage": 0.7482678983833718 + } + ], + "probability_metrics_all_rows": { + "count": 3464, + "hard_count": 3192, + "accuracy": 0.9899749373433584, + "expected_accuracy": 0.9514049268668205, + "nll": 0.10915249045149827, + "brier": 0.020795628523113556, + "expected_brier": 0.06015282636760932, + "ece_bins": 15, + "multiclass_ece": 0.013368579791205319, + "coverage": [ + { + "threshold": 0.5, + "selected": 3299, + "coverage": 0.9523672055427251, + "accuracy": 0.9899717956753369, + "expected_accuracy": 0.9738809740325352, + "expected_risk": 0.02611902596746485 + }, + { + "threshold": 0.7, + "selected": 3151, + "coverage": 0.9096420323325635, + "accuracy": 0.993006993006993, + "expected_accuracy": 0.9921717973130224, + "expected_risk": 0.007828202686977592 + }, + { + "threshold": 0.8, + "selected": 3094, + "coverage": 0.8931870669745958, + "accuracy": 0.9938570966698997, + "expected_accuracy": 0.9936974789915967, + "expected_risk": 0.006302521008403339 + }, + { + "threshold": 0.9, + "selected": 2929, + "coverage": 0.8455542725173211, + "accuracy": 0.994535519125683, + "expected_accuracy": 0.9943666780471151, + "expected_risk": 0.0056333219528849066 + }, + { + "threshold": 0.95, + "selected": 2779, + "coverage": 0.8022517321016166, + "accuracy": 0.9953203743700504, + "expected_accuracy": 0.9951421374595179, + "expected_risk": 0.00485786254048215 + }, + { + "threshold": 0.99, + "selected": 2592, + "coverage": 0.7482678983833718, + "accuracy": 0.9965277777777778, + "expected_accuracy": 0.9965277777777778, + "expected_risk": 0.00347222222222221 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3464, + "hard_count": 3192, + "accuracy": 0.9899749373433584, + "expected_accuracy": 0.9514049268668205, + "nll": 0.10915249045149827, + "brier": 0.020795628523113556, + "expected_brier": 0.06015282636760932, + "ece_bins": 15, + "multiclass_ece": 0.013368579791205319, + "coverage": [ + { + "threshold": 0.5, + "selected": 3299, + "coverage": 0.9523672055427251, + "accuracy": 0.9899717956753369, + "expected_accuracy": 0.9738809740325352, + "expected_risk": 0.02611902596746485 + }, + { + "threshold": 0.7, + "selected": 3151, + "coverage": 0.9096420323325635, + "accuracy": 0.993006993006993, + "expected_accuracy": 0.9921717973130224, + "expected_risk": 0.007828202686977592 + }, + { + "threshold": 0.8, + "selected": 3094, + "coverage": 0.8931870669745958, + "accuracy": 0.9938570966698997, + "expected_accuracy": 0.9936974789915967, + "expected_risk": 0.006302521008403339 + }, + { + "threshold": 0.9, + "selected": 2929, + "coverage": 0.8455542725173211, + "accuracy": 0.994535519125683, + "expected_accuracy": 0.9943666780471151, + "expected_risk": 0.0056333219528849066 + }, + { + "threshold": 0.95, + "selected": 2779, + "coverage": 0.8022517321016166, + "accuracy": 0.9953203743700504, + "expected_accuracy": 0.9951421374595179, + "expected_risk": 0.00485786254048215 + }, + { + "threshold": 0.99, + "selected": 2592, + "coverage": 0.7482678983833718, + "accuracy": 0.9965277777777778, + "expected_accuracy": 0.9965277777777778, + "expected_risk": 0.00347222222222221 + } + ] + } + }, + "vizdoom-basic-v1": { + "count": 825, + "successful": 825, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 825, + "accuracy_all_rows_failures_incorrect": 0.9745454545454545, + "expected_accuracy_all_rows_failures_zero": 0.9745454545454545, + "coverage": [ + { + "threshold": 0.5, + "selected": 824, + "total": 825, + "coverage": 0.9987878787878788 + }, + { + "threshold": 0.7, + "selected": 812, + "total": 825, + "coverage": 0.9842424242424243 + }, + { + "threshold": 0.8, + "selected": 794, + "total": 825, + "coverage": 0.9624242424242424 + }, + { + "threshold": 0.9, + "selected": 748, + "total": 825, + "coverage": 0.9066666666666666 + }, + { + "threshold": 0.95, + "selected": 688, + "total": 825, + "coverage": 0.833939393939394 + }, + { + "threshold": 0.99, + "selected": 534, + "total": 825, + "coverage": 0.6472727272727272 + } + ], + "probability_metrics_all_rows": { + "count": 825, + "hard_count": 825, + "accuracy": 0.9745454545454545, + "expected_accuracy": 0.9745454545454545, + "nll": 0.07717523550004791, + "brier": 0.0398977614679421, + "expected_brier": 0.0398977614679421, + "ece_bins": 15, + "multiclass_ece": 0.021055164766154082, + "coverage": [ + { + "threshold": 0.5, + "selected": 824, + "coverage": 0.9987878787878788, + "accuracy": 0.9745145631067961, + "expected_accuracy": 0.9745145631067961, + "expected_risk": 0.025485436893203928 + }, + { + "threshold": 0.7, + "selected": 812, + "coverage": 0.9842424242424243, + "accuracy": 0.9778325123152709, + "expected_accuracy": 0.9778325123152709, + "expected_risk": 0.022167487684729092 + }, + { + "threshold": 0.8, + "selected": 794, + "coverage": 0.9624242424242424, + "accuracy": 0.9911838790931989, + "expected_accuracy": 0.9911838790931989, + "expected_risk": 0.008816120906801062 + }, + { + "threshold": 0.9, + "selected": 748, + "coverage": 0.9066666666666666, + "accuracy": 0.9973262032085561, + "expected_accuracy": 0.9973262032085561, + "expected_risk": 0.002673796791443861 + }, + { + "threshold": 0.95, + "selected": 688, + "coverage": 0.833939393939394, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 534, + "coverage": 0.6472727272727272, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 825, + "hard_count": 825, + "accuracy": 0.9745454545454545, + "expected_accuracy": 0.9745454545454545, + "nll": 0.07717523550004791, + "brier": 0.0398977614679421, + "expected_brier": 0.0398977614679421, + "ece_bins": 15, + "multiclass_ece": 0.021055164766154082, + "coverage": [ + { + "threshold": 0.5, + "selected": 824, + "coverage": 0.9987878787878788, + "accuracy": 0.9745145631067961, + "expected_accuracy": 0.9745145631067961, + "expected_risk": 0.025485436893203928 + }, + { + "threshold": 0.7, + "selected": 812, + "coverage": 0.9842424242424243, + "accuracy": 0.9778325123152709, + "expected_accuracy": 0.9778325123152709, + "expected_risk": 0.022167487684729092 + }, + { + "threshold": 0.8, + "selected": 794, + "coverage": 0.9624242424242424, + "accuracy": 0.9911838790931989, + "expected_accuracy": 0.9911838790931989, + "expected_risk": 0.008816120906801062 + }, + { + "threshold": 0.9, + "selected": 748, + "coverage": 0.9066666666666666, + "accuracy": 0.9973262032085561, + "expected_accuracy": 0.9973262032085561, + "expected_risk": 0.002673796791443861 + }, + { + "threshold": 0.95, + "selected": 688, + "coverage": 0.833939393939394, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 534, + "coverage": 0.6472727272727272, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "wikispeedia-v1": { + "count": 219, + "successful": 219, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 57, + "accuracy_all_rows_failures_incorrect": 0.6666666666666666, + "expected_accuracy_all_rows_failures_zero": 0.3274433836077672, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "total": 219, + "coverage": 0.0182648401826484 + }, + { + "threshold": 0.7, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.8, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.9, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.95, + "selected": 0, + "total": 219, + "coverage": 0.0 + }, + { + "threshold": 0.99, + "selected": 0, + "total": 219, + "coverage": 0.0 + } + ], + "probability_metrics_all_rows": { + "count": 219, + "hard_count": 57, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.3274433836077672, + "nll": 2.111795103417709, + "brier": 0.29905520762617627, + "expected_brier": 0.8359138461971435, + "ece_bins": 15, + "multiclass_ece": 0.14396741883440964, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 0.0182648401826484, + "accuracy": 1.0, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 219, + "hard_count": 57, + "accuracy": 0.6666666666666666, + "expected_accuracy": 0.3274433836077672, + "nll": 2.111795103417709, + "brier": 0.29905520762617627, + "expected_brier": 0.8359138461971435, + "ece_bins": 15, + "multiclass_ece": 0.14396741883440964, + "coverage": [ + { + "threshold": 0.5, + "selected": 4, + "coverage": 0.0182648401826484, + "accuracy": 1.0, + "expected_accuracy": 0.75, + "expected_risk": 0.25 + }, + { + "threshold": 0.7, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.8, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.9, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.95, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + }, + { + "threshold": 0.99, + "selected": 0, + "coverage": 0.0, + "accuracy": null, + "expected_accuracy": null, + "expected_risk": null + } + ] + } + }, + "workflow-controls-v1/agent_trace_observability": { + "count": 525, + "successful": 525, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 525, + "accuracy_all_rows_failures_incorrect": 1.0, + "expected_accuracy_all_rows_failures_zero": 1.0, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.95, + "selected": 525, + "total": 525, + "coverage": 1.0 + }, + { + "threshold": 0.99, + "selected": 524, + "total": 525, + "coverage": 0.9980952380952381 + } + ], + "probability_metrics_all_rows": { + "count": 525, + "hard_count": 525, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000545160068456591, + "brier": 4.895507169664063e-06, + "expected_brier": 4.8955071696642534e-06, + "ece_bins": 15, + "multiclass_ece": 0.0005439303275837422, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 524, + "coverage": 0.9980952380952381, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 525, + "hard_count": 525, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "nll": 0.000545160068456591, + "brier": 4.895507169664063e-06, + "expected_brier": 4.8955071696642534e-06, + "ece_bins": 15, + "multiclass_ece": 0.0005439303275837422, + "coverage": [ + { + "threshold": 0.5, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.7, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.8, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.9, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.95, + "selected": 525, + "coverage": 1.0, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 524, + "coverage": 0.9980952380952381, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/customer_service": { + "count": 600, + "successful": 600, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 600, + "accuracy_all_rows_failures_incorrect": 0.9783333333333334, + "expected_accuracy_all_rows_failures_zero": 0.9783333333333334, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.95, + "selected": 600, + "total": 600, + "coverage": 1.0 + }, + { + "threshold": 0.99, + "selected": 597, + "total": 600, + "coverage": 0.995 + } + ], + "probability_metrics_all_rows": { + "count": 600, + "hard_count": 600, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "nll": 0.16283998438208533, + "brier": 0.04328014604238859, + "expected_brier": 0.04328014604238859, + "ece_bins": 15, + "multiclass_ece": 0.021130186926374108, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.7, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.8, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.9, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.95, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.99, + "selected": 597, + "coverage": 0.995, + "accuracy": 0.9782244556113903, + "expected_accuracy": 0.9782244556113903, + "expected_risk": 0.021775544388609736 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 600, + "hard_count": 600, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "nll": 0.16283998438208533, + "brier": 0.04328014604238859, + "expected_brier": 0.04328014604238859, + "ece_bins": 15, + "multiclass_ece": 0.021130186926374108, + "coverage": [ + { + "threshold": 0.5, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.7, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.8, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.9, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.95, + "selected": 600, + "coverage": 1.0, + "accuracy": 0.9783333333333334, + "expected_accuracy": 0.9783333333333334, + "expected_risk": 0.021666666666666612 + }, + { + "threshold": 0.99, + "selected": 597, + "coverage": 0.995, + "accuracy": 0.9782244556113903, + "expected_accuracy": 0.9782244556113903, + "expected_risk": 0.021775544388609736 + } + ] + } + }, + "workflow-controls-v1/invoice_processing": { + "count": 750, + "successful": 750, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 750, + "accuracy_all_rows_failures_incorrect": 0.9973333333333333, + "expected_accuracy_all_rows_failures_zero": 0.9973333333333333, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "total": 750, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 748, + "total": 750, + "coverage": 0.9973333333333333 + }, + { + "threshold": 0.8, + "selected": 746, + "total": 750, + "coverage": 0.9946666666666667 + }, + { + "threshold": 0.9, + "selected": 746, + "total": 750, + "coverage": 0.9946666666666667 + }, + { + "threshold": 0.95, + "selected": 744, + "total": 750, + "coverage": 0.992 + }, + { + "threshold": 0.99, + "selected": 743, + "total": 750, + "coverage": 0.9906666666666667 + } + ], + "probability_metrics_all_rows": { + "count": 750, + "hard_count": 750, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "nll": 0.006416542163341697, + "brier": 0.003983146029347859, + "expected_brier": 0.003983146029347861, + "ece_bins": 15, + "multiclass_ece": 0.002603556053682448, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "coverage": 1.0, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "expected_risk": 0.002666666666666706 + }, + { + "threshold": 0.7, + "selected": 748, + "coverage": 0.9973333333333333, + "accuracy": 0.9986631016042781, + "expected_accuracy": 0.9986631016042781, + "expected_risk": 0.0013368983957219305 + }, + { + "threshold": 0.8, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9986595174262735, + "expected_accuracy": 0.9986595174262735, + "expected_risk": 0.0013404825737265424 + }, + { + "threshold": 0.9, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9986595174262735, + "expected_accuracy": 0.9986595174262735, + "expected_risk": 0.0013404825737265424 + }, + { + "threshold": 0.95, + "selected": 744, + "coverage": 0.992, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 743, + "coverage": 0.9906666666666667, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 750, + "hard_count": 750, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "nll": 0.006416542163341697, + "brier": 0.003983146029347859, + "expected_brier": 0.003983146029347861, + "ece_bins": 15, + "multiclass_ece": 0.002603556053682448, + "coverage": [ + { + "threshold": 0.5, + "selected": 750, + "coverage": 1.0, + "accuracy": 0.9973333333333333, + "expected_accuracy": 0.9973333333333333, + "expected_risk": 0.002666666666666706 + }, + { + "threshold": 0.7, + "selected": 748, + "coverage": 0.9973333333333333, + "accuracy": 0.9986631016042781, + "expected_accuracy": 0.9986631016042781, + "expected_risk": 0.0013368983957219305 + }, + { + "threshold": 0.8, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9986595174262735, + "expected_accuracy": 0.9986595174262735, + "expected_risk": 0.0013404825737265424 + }, + { + "threshold": 0.9, + "selected": 746, + "coverage": 0.9946666666666667, + "accuracy": 0.9986595174262735, + "expected_accuracy": 0.9986595174262735, + "expected_risk": 0.0013404825737265424 + }, + { + "threshold": 0.95, + "selected": 744, + "coverage": 0.992, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + }, + { + "threshold": 0.99, + "selected": 743, + "coverage": 0.9906666666666667, + "accuracy": 1.0, + "expected_accuracy": 1.0, + "expected_risk": 0.0 + } + ] + } + }, + "workflow-controls-v1/security_incidents": { + "count": 1275, + "successful": 1275, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 1275, + "accuracy_all_rows_failures_incorrect": 0.9929411764705882, + "expected_accuracy_all_rows_failures_zero": 0.9929411764705882, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "total": 1275, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 1275, + "total": 1275, + "coverage": 1.0 + }, + { + "threshold": 0.8, + "selected": 1275, + "total": 1275, + "coverage": 1.0 + }, + { + "threshold": 0.9, + "selected": 1266, + "total": 1275, + "coverage": 0.9929411764705882 + }, + { + "threshold": 0.95, + "selected": 1256, + "total": 1275, + "coverage": 0.9850980392156863 + }, + { + "threshold": 0.99, + "selected": 1247, + "total": 1275, + "coverage": 0.9780392156862745 + } + ], + "probability_metrics_all_rows": { + "count": 1275, + "hard_count": 1275, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "nll": 0.02425288839114142, + "brier": 0.012286914067882482, + "expected_brier": 0.012286914067882486, + "ece_bins": 15, + "multiclass_ece": 0.004988858941316103, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.7, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.8, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.9, + "selected": 1266, + "coverage": 0.9929411764705882, + "accuracy": 0.995260663507109, + "expected_accuracy": 0.995260663507109, + "expected_risk": 0.004739336492891044 + }, + { + "threshold": 0.95, + "selected": 1256, + "coverage": 0.9850980392156863, + "accuracy": 0.9976114649681529, + "expected_accuracy": 0.9976114649681529, + "expected_risk": 0.0023885350318471055 + }, + { + "threshold": 0.99, + "selected": 1247, + "coverage": 0.9780392156862745, + "accuracy": 0.9983961507618284, + "expected_accuracy": 0.9983961507618284, + "expected_risk": 0.001603849238171584 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 1275, + "hard_count": 1275, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "nll": 0.02425288839114142, + "brier": 0.012286914067882482, + "expected_brier": 0.012286914067882486, + "ece_bins": 15, + "multiclass_ece": 0.004988858941316103, + "coverage": [ + { + "threshold": 0.5, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.7, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.8, + "selected": 1275, + "coverage": 1.0, + "accuracy": 0.9929411764705882, + "expected_accuracy": 0.9929411764705882, + "expected_risk": 0.007058823529411784 + }, + { + "threshold": 0.9, + "selected": 1266, + "coverage": 0.9929411764705882, + "accuracy": 0.995260663507109, + "expected_accuracy": 0.995260663507109, + "expected_risk": 0.004739336492891044 + }, + { + "threshold": 0.95, + "selected": 1256, + "coverage": 0.9850980392156863, + "accuracy": 0.9976114649681529, + "expected_accuracy": 0.9976114649681529, + "expected_risk": 0.0023885350318471055 + }, + { + "threshold": 0.99, + "selected": 1247, + "coverage": 0.9780392156862745, + "accuracy": 0.9983961507618284, + "expected_accuracy": 0.9983961507618284, + "expected_risk": 0.001603849238171584 + } + ] + } + } + }, + "by_kind": { + "choice": { + "count": 3438, + "successful": 3438, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2993, + "accuracy_all_rows_failures_incorrect": 0.863013698630137, + "expected_accuracy_all_rows_failures_zero": 0.8021747433614798, + "coverage": [ + { + "threshold": 0.5, + "selected": 2972, + "total": 3438, + "coverage": 0.8644560791157649 + }, + { + "threshold": 0.7, + "selected": 2643, + "total": 3438, + "coverage": 0.768760907504363 + }, + { + "threshold": 0.8, + "selected": 2483, + "total": 3438, + "coverage": 0.7222222222222222 + }, + { + "threshold": 0.9, + "selected": 2147, + "total": 3438, + "coverage": 0.6244909831297266 + }, + { + "threshold": 0.95, + "selected": 1778, + "total": 3438, + "coverage": 0.5171611401977894 + }, + { + "threshold": 0.99, + "selected": 863, + "total": 3438, + "coverage": 0.2510180337405468 + } + ], + "probability_metrics_all_rows": { + "count": 3438, + "hard_count": 2993, + "accuracy": 0.863013698630137, + "expected_accuracy": 0.8021747433614798, + "nll": 0.6259869551780559, + "brier": 0.20650286979189394, + "expected_brier": 0.28195527676698856, + "ece_bins": 15, + "multiclass_ece": 0.05203723428051684, + "coverage": [ + { + "threshold": 0.5, + "selected": 2972, + "coverage": 0.8644560791157649, + "accuracy": 0.8779803646563815, + "expected_accuracy": 0.8626626289816062, + "expected_risk": 0.13733737101839383 + }, + { + "threshold": 0.7, + "selected": 2643, + "coverage": 0.768760907504363, + "accuracy": 0.8976109215017065, + "expected_accuracy": 0.8966452263841594, + "expected_risk": 0.10335477361584056 + }, + { + "threshold": 0.8, + "selected": 2483, + "coverage": 0.7222222222222222, + "accuracy": 0.9129734085414988, + "expected_accuracy": 0.9128070881997583, + "expected_risk": 0.08719291180024169 + }, + { + "threshold": 0.9, + "selected": 2147, + "coverage": 0.6244909831297266, + "accuracy": 0.9263746505125815, + "expected_accuracy": 0.9261760596180717, + "expected_risk": 0.0738239403819283 + }, + { + "threshold": 0.95, + "selected": 1778, + "coverage": 0.5171611401977894, + "accuracy": 0.9465391108610017, + "expected_accuracy": 0.9462879640044994, + "expected_risk": 0.05371203599550056 + }, + { + "threshold": 0.99, + "selected": 863, + "coverage": 0.2510180337405468, + "accuracy": 0.9733487833140209, + "expected_accuracy": 0.9733487833140209, + "expected_risk": 0.02665121668597914 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 3438, + "hard_count": 2993, + "accuracy": 0.863013698630137, + "expected_accuracy": 0.8021747433614798, + "nll": 0.6259869551780559, + "brier": 0.20650286979189394, + "expected_brier": 0.28195527676698856, + "ece_bins": 15, + "multiclass_ece": 0.05203723428051684, + "coverage": [ + { + "threshold": 0.5, + "selected": 2972, + "coverage": 0.8644560791157649, + "accuracy": 0.8779803646563815, + "expected_accuracy": 0.8626626289816062, + "expected_risk": 0.13733737101839383 + }, + { + "threshold": 0.7, + "selected": 2643, + "coverage": 0.768760907504363, + "accuracy": 0.8976109215017065, + "expected_accuracy": 0.8966452263841594, + "expected_risk": 0.10335477361584056 + }, + { + "threshold": 0.8, + "selected": 2483, + "coverage": 0.7222222222222222, + "accuracy": 0.9129734085414988, + "expected_accuracy": 0.9128070881997583, + "expected_risk": 0.08719291180024169 + }, + { + "threshold": 0.9, + "selected": 2147, + "coverage": 0.6244909831297266, + "accuracy": 0.9263746505125815, + "expected_accuracy": 0.9261760596180717, + "expected_risk": 0.0738239403819283 + }, + { + "threshold": 0.95, + "selected": 1778, + "coverage": 0.5171611401977894, + "accuracy": 0.9465391108610017, + "expected_accuracy": 0.9462879640044994, + "expected_risk": 0.05371203599550056 + }, + { + "threshold": 0.99, + "selected": 863, + "coverage": 0.2510180337405468, + "accuracy": 0.9733487833140209, + "expected_accuracy": 0.9733487833140209, + "expected_risk": 0.02665121668597914 + } + ] + } + }, + "noul": { + "count": 9929, + "successful": 9929, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 9929, + "accuracy_all_rows_failures_incorrect": 0.9417866854668144, + "expected_accuracy_all_rows_failures_zero": 0.9417866854668144, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "total": 9929, + "coverage": 1.0 + }, + { + "threshold": 0.7, + "selected": 9661, + "total": 9929, + "coverage": 0.973008359351395 + }, + { + "threshold": 0.8, + "selected": 9448, + "total": 9929, + "coverage": 0.9515560479403766 + }, + { + "threshold": 0.9, + "selected": 9151, + "total": 9929, + "coverage": 0.9216436700574075 + }, + { + "threshold": 0.95, + "selected": 8928, + "total": 9929, + "coverage": 0.899184207875919 + }, + { + "threshold": 0.99, + "selected": 8067, + "total": 9929, + "coverage": 0.8124685265384228 + } + ], + "probability_metrics_all_rows": { + "count": 9929, + "hard_count": 9929, + "accuracy": 0.9417866854668144, + "expected_accuracy": 0.9417866854668144, + "nll": 0.1586976512072927, + "brier": 0.08958742899221635, + "expected_brier": 0.08958742899221635, + "ece_bins": 15, + "multiclass_ece": 0.03298841702250886, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "coverage": 1.0, + "accuracy": 0.9417866854668144, + "expected_accuracy": 0.9417866854668144, + "expected_risk": 0.05821331453318557 + }, + { + "threshold": 0.7, + "selected": 9661, + "coverage": 0.973008359351395, + "accuracy": 0.9533174619604596, + "expected_accuracy": 0.9533174619604596, + "expected_risk": 0.04668253803954037 + }, + { + "threshold": 0.8, + "selected": 9448, + "coverage": 0.9515560479403766, + "accuracy": 0.9613674851820491, + "expected_accuracy": 0.9613674851820491, + "expected_risk": 0.038632514817950936 + }, + { + "threshold": 0.9, + "selected": 9151, + "coverage": 0.9216436700574075, + "accuracy": 0.973117692055513, + "expected_accuracy": 0.973117692055513, + "expected_risk": 0.026882307944486983 + }, + { + "threshold": 0.95, + "selected": 8928, + "coverage": 0.899184207875919, + "accuracy": 0.9816308243727598, + "expected_accuracy": 0.9816308243727598, + "expected_risk": 0.018369175627240164 + }, + { + "threshold": 0.99, + "selected": 8067, + "coverage": 0.8124685265384228, + "accuracy": 0.9935539853725053, + "expected_accuracy": 0.9935539853725053, + "expected_risk": 0.006446014627494723 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 9929, + "hard_count": 9929, + "accuracy": 0.9417866854668144, + "expected_accuracy": 0.9417866854668144, + "nll": 0.1586976512072927, + "brier": 0.08958742899221635, + "expected_brier": 0.08958742899221635, + "ece_bins": 15, + "multiclass_ece": 0.03298841702250886, + "coverage": [ + { + "threshold": 0.5, + "selected": 9929, + "coverage": 1.0, + "accuracy": 0.9417866854668144, + "expected_accuracy": 0.9417866854668144, + "expected_risk": 0.05821331453318557 + }, + { + "threshold": 0.7, + "selected": 9661, + "coverage": 0.973008359351395, + "accuracy": 0.9533174619604596, + "expected_accuracy": 0.9533174619604596, + "expected_risk": 0.04668253803954037 + }, + { + "threshold": 0.8, + "selected": 9448, + "coverage": 0.9515560479403766, + "accuracy": 0.9613674851820491, + "expected_accuracy": 0.9613674851820491, + "expected_risk": 0.038632514817950936 + }, + { + "threshold": 0.9, + "selected": 9151, + "coverage": 0.9216436700574075, + "accuracy": 0.973117692055513, + "expected_accuracy": 0.973117692055513, + "expected_risk": 0.026882307944486983 + }, + { + "threshold": 0.95, + "selected": 8928, + "coverage": 0.899184207875919, + "accuracy": 0.9816308243727598, + "expected_accuracy": 0.9816308243727598, + "expected_risk": 0.018369175627240164 + }, + { + "threshold": 0.99, + "selected": 8067, + "coverage": 0.8124685265384228, + "accuracy": 0.9935539853725053, + "expected_accuracy": 0.9935539853725053, + "expected_risk": 0.006446014627494723 + } + ] + } + }, + "score": { + "count": 2553, + "successful": 2553, + "failed": 0, + "inference_coverage": 1.0, + "hard_count": 2524, + "accuracy_all_rows_failures_incorrect": 0.8997622820919176, + "expected_accuracy_all_rows_failures_zero": 0.8944379161770466, + "coverage": [ + { + "threshold": 0.5, + "selected": 2448, + "total": 2553, + "coverage": 0.9588719153936546 + }, + { + "threshold": 0.7, + "selected": 2328, + "total": 2553, + "coverage": 0.9118683901292597 + }, + { + "threshold": 0.8, + "selected": 2252, + "total": 2553, + "coverage": 0.8820994907951429 + }, + { + "threshold": 0.9, + "selected": 2144, + "total": 2553, + "coverage": 0.8397963180571876 + }, + { + "threshold": 0.95, + "selected": 2045, + "total": 2553, + "coverage": 0.8010184097140619 + }, + { + "threshold": 0.99, + "selected": 1741, + "total": 2553, + "coverage": 0.681942812377595 + } + ], + "probability_metrics_all_rows": { + "count": 2553, + "hard_count": 2524, + "accuracy": 0.8997622820919176, + "expected_accuracy": 0.8944379161770466, + "nll": 0.40706831123282244, + "brier": 0.16323629063231726, + "expected_brier": 0.16969927535617155, + "ece_bins": 15, + "multiclass_ece": 0.05413140129609939, + "coverage": [ + { + "threshold": 0.5, + "selected": 2448, + "coverage": 0.9588719153936546, + "accuracy": 0.9197860962566845, + "expected_accuracy": 0.9168709150326797, + "expected_risk": 0.0831290849673203 + }, + { + "threshold": 0.7, + "selected": 2328, + "coverage": 0.9118683901292597, + "accuracy": 0.9286941580756014, + "expected_accuracy": 0.9286941580756014, + "expected_risk": 0.0713058419243986 + }, + { + "threshold": 0.8, + "selected": 2252, + "coverage": 0.8820994907951429, + "accuracy": 0.933392539964476, + "expected_accuracy": 0.933392539964476, + "expected_risk": 0.06660746003552398 + }, + { + "threshold": 0.9, + "selected": 2144, + "coverage": 0.8397963180571876, + "accuracy": 0.9421641791044776, + "expected_accuracy": 0.9421641791044776, + "expected_risk": 0.057835820895522416 + }, + { + "threshold": 0.95, + "selected": 2045, + "coverage": 0.8010184097140619, + "accuracy": 0.9486552567237164, + "expected_accuracy": 0.9486552567237164, + "expected_risk": 0.05134474327628358 + }, + { + "threshold": 0.99, + "selected": 1741, + "coverage": 0.681942812377595, + "accuracy": 0.9620907524411257, + "expected_accuracy": 0.9620907524411257, + "expected_risk": 0.03790924755887426 + } + ] + }, + "probability_metrics_successful_rows_only": { + "count": 2553, + "hard_count": 2524, + "accuracy": 0.8997622820919176, + "expected_accuracy": 0.8944379161770466, + "nll": 0.40706831123282244, + "brier": 0.16323629063231726, + "expected_brier": 0.16969927535617155, + "ece_bins": 15, + "multiclass_ece": 0.05413140129609939, + "coverage": [ + { + "threshold": 0.5, + "selected": 2448, + "coverage": 0.9588719153936546, + "accuracy": 0.9197860962566845, + "expected_accuracy": 0.9168709150326797, + "expected_risk": 0.0831290849673203 + }, + { + "threshold": 0.7, + "selected": 2328, + "coverage": 0.9118683901292597, + "accuracy": 0.9286941580756014, + "expected_accuracy": 0.9286941580756014, + "expected_risk": 0.0713058419243986 + }, + { + "threshold": 0.8, + "selected": 2252, + "coverage": 0.8820994907951429, + "accuracy": 0.933392539964476, + "expected_accuracy": 0.933392539964476, + "expected_risk": 0.06660746003552398 + }, + { + "threshold": 0.9, + "selected": 2144, + "coverage": 0.8397963180571876, + "accuracy": 0.9421641791044776, + "expected_accuracy": 0.9421641791044776, + "expected_risk": 0.057835820895522416 + }, + { + "threshold": 0.95, + "selected": 2045, + "coverage": 0.8010184097140619, + "accuracy": 0.9486552567237164, + "expected_accuracy": 0.9486552567237164, + "expected_risk": 0.05134474327628358 + }, + { + "threshold": 0.99, + "selected": 1741, + "coverage": 0.681942812377595, + "accuracy": 0.9620907524411257, + "expected_accuracy": 0.9620907524411257, + "expected_risk": 0.03790924755887426 + } + ] + } + } + } + } + }, + "full_data_baseline_evaluated": false, + "full_data_training_gain": null, + "limitations": [ + "No full-data baseline exists in this run. Earlier 512-row-per-split baseline results cannot establish full-data training gain.", + "Per-group metrics were recomputed and compared for every group; compact report retains source/kind metrics and group counts/hashes. Original complete summary remains in ignored evidence.", + "Snapshot/checkpoint hashes bind this evaluation to the packaged inference files; the trainer's historical reload lacked a contemporaneous weight digest.", + "Synthetic held-out decision accuracy is not end-to-end game, browser or flight performance. Four-replica wall time is not single-GPU latency." + ], + "schema_version": 1, + "status": "passed", + "source_audit_sha256": "d490dd1636acebaf393fe181dfd0f65e66eb54edb7247b4abffd74a3d0891100", + "source_audit_path": "reports/full-data-eval-n1-v1/9b-audit.json", + "scope": "Existing completed full release-v2 held-out evaluation, bound to the unchanged five packaged inference files. No new model inference was run for this Hugging Face upload." +}