| { |
| "model": "Qwen/Qwen3.5-9B", |
| "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", |
| "checkpoint": { |
| "model": "Qwen/Qwen3.5-9B", |
| "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", |
| "checkpoint_sha256": "a691105dd5751bfcf72d276a6ce075d45054db77ae3272e87b5f07b2bd7e2193", |
| "temperature": 1.8969118766347646, |
| "calibration": { |
| "temperature": 1.8969118766347646, |
| "split": "calibration", |
| "n": 512, |
| "ids_sha256": "6ff7db59c1ae2485179b80316b3cb4f0161df77704692dcab48871a295ab94f1" |
| }, |
| "max_length": 4096, |
| "training_completion": { |
| "steps": 20204, |
| "trained_rows_consumed": 80816, |
| "checkpoint_reload_max_error": 0.0, |
| "run_sha256": "e5a119cc2ec1f78615375824352f71a68bcc78eb30d28e14bcdbcd442075b42b", |
| "summary_sha256": "789ae2a85fc3dec77388b11c039f56e3110052b77c55e1eb371d5d5284e72ebc" |
| } |
| }, |
| "package_bound_inference_files": { |
| "checkpoint/adapter/adapter_config.json": { |
| "sha256": "3b2d3b4c9e95a7ca0dfb6132a8a76af281363032eeffbbd08ba34fc76cb3b9be", |
| "bytes": 1170 |
| }, |
| "checkpoint/adapter/adapter_model.safetensors": { |
| "sha256": "f85650a8fb97c6d0ac3e948cdca2f30a0ca6ace8b8a43aebca1c152fe38aeb75", |
| "bytes": 23613816 |
| }, |
| "checkpoint/head.pt": { |
| "sha256": "229fe9800384e824135e59d1af59bda7346da031aea61640be5f456b9be6ce70", |
| "bytes": 18189 |
| }, |
| "checkpoint/model.json": { |
| "sha256": "a3a8a691a88b2af64ecb6952a7dd006977fd8c544df09afcbe1e4303b91a8daa", |
| "bytes": 186 |
| }, |
| "checkpoint/temperature.json": { |
| "sha256": "8cfe40a9f42cc2d0607a3381f2a4e0374cb3842506ac7263f7fff35facfca034", |
| "bytes": 162 |
| } |
| }, |
| "data_identity": { |
| "manifest_sha256": "56105dc9fc89ef74919f5beb60bb6ae8c6e17bb95699dab59205f67d8b338d97", |
| "split_sha256": { |
| "train": "80a9ec19da9c4065b72f7a9e5ff8d67e62c1d1dd2572dfd296e46bfe2ea5a182", |
| "calibration": "dd21b26912a6b26a09c3b00ae66a7985ee760e737b94ada4fcc6320406d4db63", |
| "validation": "1dc97079d4fc034a1ba16315f801b0a4715b30c2a3874ae05aeb60dd1e40e8bb", |
| "test": "fa8d62775b96c1514f238c2d8a8f8b66515dada67d490978376545296593e29b", |
| "ood": "9152bff7f3d9c83cce3f1423cc6e0e6b33286b3092b3f544cb18dcc37b35a7ee" |
| }, |
| "counts": { |
| "test": 10532, |
| "ood": 15920 |
| }, |
| "ordered_ids_sha256": { |
| "test": "98e1af223471c08d84e44c8864e51cefa1bc671687c00bfbf64b7e5a22e8bf52", |
| "ood": "52fa6cc83c0a172516609257b6048aaece9a3fef75f3f5f694bc6213e5c30a42" |
| } |
| }, |
| "coverage": { |
| "expected_rows": 26452, |
| "unique_ids": 26452, |
| "shard_rows": [ |
| 6613, |
| 6613, |
| 6613, |
| 6613 |
| ], |
| "missing": 0, |
| "duplicate": 0, |
| "failed": 0, |
| "merged_original_order_verified": true |
| }, |
| "metrics": { |
| "test": { |
| "count": 10532, |
| "successful": 10532, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 10046, |
| "accuracy_all_rows_failures_incorrect": 0.9754130997411905, |
| "expected_accuracy_all_rows_failures_zero": 0.9471531754526437, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9945, |
| "total": 10532, |
| "coverage": 0.9442650968477022 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9636, |
| "total": 10532, |
| "coverage": 0.9149259399924041 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9552, |
| "total": 10532, |
| "coverage": 0.906950246866692 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9409, |
| "total": 10532, |
| "coverage": 0.893372578807444 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 9208, |
| "total": 10532, |
| "coverage": 0.8742878845423472 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8674, |
| "total": 10532, |
| "coverage": 0.823585263957463 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 10532, |
| "hard_count": 10046, |
| "accuracy": 0.9754130997411905, |
| "expected_accuracy": 0.9471531754526437, |
| "nll": 0.1309471525831089, |
| "brier": 0.03911290032779779, |
| "expected_brier": 0.0671638646396812, |
| "ece_bins": 15, |
| "multiclass_ece": 0.007706544094318023, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9945, |
| "coverage": 0.9442650968477022, |
| "accuracy": 0.9885468861846815, |
| "expected_accuracy": 0.9797553209317914, |
| "expected_risk": 0.020244679068208593 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9636, |
| "coverage": 0.9149259399924041, |
| "accuracy": 0.9936570656129771, |
| "expected_accuracy": 0.9926490936764909, |
| "expected_risk": 0.007350906323509099 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9552, |
| "coverage": 0.906950246866692, |
| "accuracy": 0.9952859836580766, |
| "expected_accuracy": 0.9949748743718593, |
| "expected_risk": 0.005025125628140725 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9409, |
| "coverage": 0.893372578807444, |
| "accuracy": 0.9977680943777234, |
| "expected_accuracy": 0.9977680943777234, |
| "expected_risk": 0.0022319056222765976 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 9208, |
| "coverage": 0.8742878845423472, |
| "accuracy": 0.9993483927019983, |
| "expected_accuracy": 0.9993483927019983, |
| "expected_risk": 0.0006516072980017329 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8674, |
| "coverage": 0.823585263957463, |
| "accuracy": 0.9997694258704174, |
| "expected_accuracy": 0.9997694258704174, |
| "expected_risk": 0.0002305741295826058 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 10532, |
| "hard_count": 10046, |
| "accuracy": 0.9754130997411905, |
| "expected_accuracy": 0.9471531754526437, |
| "nll": 0.1309471525831089, |
| "brier": 0.03911290032779779, |
| "expected_brier": 0.0671638646396812, |
| "ece_bins": 15, |
| "multiclass_ece": 0.007706544094318023, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9945, |
| "coverage": 0.9442650968477022, |
| "accuracy": 0.9885468861846815, |
| "expected_accuracy": 0.9797553209317914, |
| "expected_risk": 0.020244679068208593 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9636, |
| "coverage": 0.9149259399924041, |
| "accuracy": 0.9936570656129771, |
| "expected_accuracy": 0.9926490936764909, |
| "expected_risk": 0.007350906323509099 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9552, |
| "coverage": 0.906950246866692, |
| "accuracy": 0.9952859836580766, |
| "expected_accuracy": 0.9949748743718593, |
| "expected_risk": 0.005025125628140725 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9409, |
| "coverage": 0.893372578807444, |
| "accuracy": 0.9977680943777234, |
| "expected_accuracy": 0.9977680943777234, |
| "expected_risk": 0.0022319056222765976 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 9208, |
| "coverage": 0.8742878845423472, |
| "accuracy": 0.9993483927019983, |
| "expected_accuracy": 0.9993483927019983, |
| "expected_risk": 0.0006516072980017329 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8674, |
| "coverage": 0.823585263957463, |
| "accuracy": 0.9997694258704174, |
| "expected_accuracy": 0.9997694258704174, |
| "expected_risk": 0.0002305741295826058 |
| } |
| ] |
| }, |
| "by_source": { |
| "customer-control-v1": { |
| "count": 606, |
| "successful": 606, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 574, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 0.9722222222222222, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 596, |
| "total": 606, |
| "coverage": 0.9834983498349835 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 574, |
| "total": 606, |
| "coverage": 0.9471947194719472 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 574, |
| "total": 606, |
| "coverage": 0.9471947194719472 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 574, |
| "total": 606, |
| "coverage": 0.9471947194719472 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 574, |
| "total": 606, |
| "coverage": 0.9471947194719472 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "total": 606, |
| "coverage": 0.9306930693069307 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 606, |
| "hard_count": 574, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9722222222222222, |
| "nll": 0.04199953432734906, |
| "brier": 6.843414604449945e-05, |
| "expected_brier": 0.027846211923822282, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0019262574800517217, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 596, |
| "coverage": 0.9834983498349835, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9815436241610739, |
| "expected_risk": 0.01845637583892612 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "coverage": 0.9306930693069307, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 606, |
| "hard_count": 574, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9722222222222222, |
| "nll": 0.04199953432734906, |
| "brier": 6.843414604449945e-05, |
| "expected_brier": 0.027846211923822282, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0019262574800517217, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 596, |
| "coverage": 0.9834983498349835, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9815436241610739, |
| "expected_risk": 0.01845637583892612 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 574, |
| "coverage": 0.9471947194719472, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "coverage": 0.9306930693069307, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "painting-geometry-v1": { |
| "count": 3072, |
| "successful": 3072, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 3072, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 1.0, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "total": 3072, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3071, |
| "total": 3072, |
| "coverage": 0.9996744791666666 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3071, |
| "total": 3072, |
| "coverage": 0.9996744791666666 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3071, |
| "total": 3072, |
| "coverage": 0.9996744791666666 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3068, |
| "total": 3072, |
| "coverage": 0.9986979166666666 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3031, |
| "total": 3072, |
| "coverage": 0.9866536458333334 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3072, |
| "hard_count": 3072, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.0013689140515597267, |
| "brier": 7.957005332993052e-05, |
| "expected_brier": 7.957005332993308e-05, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0013415313830535288, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3068, |
| "coverage": 0.9986979166666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3031, |
| "coverage": 0.9866536458333334, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3072, |
| "hard_count": 3072, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.0013689140515597267, |
| "brier": 7.957005332993052e-05, |
| "expected_brier": 7.957005332993308e-05, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0013415313830535288, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3072, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3071, |
| "coverage": 0.9996744791666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3068, |
| "coverage": 0.9986979166666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3031, |
| "coverage": 0.9866536458333334, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "reasoning-control-v1": { |
| "count": 681, |
| "successful": 681, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 681, |
| "accuracy_all_rows_failures_incorrect": 0.8722466960352423, |
| "expected_accuracy_all_rows_failures_zero": 0.8722466960352423, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 627, |
| "total": 681, |
| "coverage": 0.920704845814978 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "total": 681, |
| "coverage": 0.8458149779735683 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 554, |
| "total": 681, |
| "coverage": 0.8135095447870778 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 512, |
| "total": 681, |
| "coverage": 0.7518355359765051 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 489, |
| "total": 681, |
| "coverage": 0.7180616740088106 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 419, |
| "total": 681, |
| "coverage": 0.6152716593245228 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 681, |
| "hard_count": 681, |
| "accuracy": 0.8722466960352423, |
| "expected_accuracy": 0.8722466960352423, |
| "nll": 0.30288131001112933, |
| "brier": 0.1731250583140691, |
| "expected_brier": 0.1731250583140691, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0369016945577258, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 627, |
| "coverage": 0.920704845814978, |
| "accuracy": 0.9138755980861244, |
| "expected_accuracy": 0.9138755980861244, |
| "expected_risk": 0.0861244019138756 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "coverage": 0.8458149779735683, |
| "accuracy": 0.9461805555555556, |
| "expected_accuracy": 0.9461805555555556, |
| "expected_risk": 0.05381944444444442 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 554, |
| "coverage": 0.8135095447870778, |
| "accuracy": 0.9620938628158845, |
| "expected_accuracy": 0.9620938628158845, |
| "expected_risk": 0.03790613718411551 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 512, |
| "coverage": 0.7518355359765051, |
| "accuracy": 0.986328125, |
| "expected_accuracy": 0.986328125, |
| "expected_risk": 0.013671875 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 489, |
| "coverage": 0.7180616740088106, |
| "accuracy": 0.9959100204498977, |
| "expected_accuracy": 0.9959100204498977, |
| "expected_risk": 0.004089979550102263 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 419, |
| "coverage": 0.6152716593245228, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 681, |
| "hard_count": 681, |
| "accuracy": 0.8722466960352423, |
| "expected_accuracy": 0.8722466960352423, |
| "nll": 0.30288131001112933, |
| "brier": 0.1731250583140691, |
| "expected_brier": 0.1731250583140691, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0369016945577258, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 627, |
| "coverage": 0.920704845814978, |
| "accuracy": 0.9138755980861244, |
| "expected_accuracy": 0.9138755980861244, |
| "expected_risk": 0.0861244019138756 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 576, |
| "coverage": 0.8458149779735683, |
| "accuracy": 0.9461805555555556, |
| "expected_accuracy": 0.9461805555555556, |
| "expected_risk": 0.05381944444444442 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 554, |
| "coverage": 0.8135095447870778, |
| "accuracy": 0.9620938628158845, |
| "expected_accuracy": 0.9620938628158845, |
| "expected_risk": 0.03790613718411551 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 512, |
| "coverage": 0.7518355359765051, |
| "accuracy": 0.986328125, |
| "expected_accuracy": 0.986328125, |
| "expected_risk": 0.013671875 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 489, |
| "coverage": 0.7180616740088106, |
| "accuracy": 0.9959100204498977, |
| "expected_accuracy": 0.9959100204498977, |
| "expected_risk": 0.004089979550102263 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 419, |
| "coverage": 0.6152716593245228, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "snake-v1": { |
| "count": 1392, |
| "successful": 1392, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1279, |
| "accuracy_all_rows_failures_incorrect": 0.9976544175136826, |
| "expected_accuracy_all_rows_failures_zero": 0.9567768199233716, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1340, |
| "total": 1392, |
| "coverage": 0.9626436781609196 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1268, |
| "total": 1392, |
| "coverage": 0.9109195402298851 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1254, |
| "total": 1392, |
| "coverage": 0.9008620689655172 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1201, |
| "total": 1392, |
| "coverage": 0.8627873563218391 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1152, |
| "total": 1392, |
| "coverage": 0.8275862068965517 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1076, |
| "total": 1392, |
| "coverage": 0.7729885057471264 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1392, |
| "hard_count": 1279, |
| "accuracy": 0.9976544175136826, |
| "expected_accuracy": 0.9567768199233716, |
| "nll": 0.08536951007564594, |
| "brier": 0.01017585055261676, |
| "expected_brier": 0.05124385821545201, |
| "ece_bins": 15, |
| "multiclass_ece": 0.014056450259188788, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1340, |
| "coverage": 0.9626436781609196, |
| "accuracy": 0.9976525821596244, |
| "expected_accuracy": 0.9743781094527364, |
| "expected_risk": 0.025621890547263604 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1268, |
| "coverage": 0.9109195402298851, |
| "accuracy": 0.9984164687252574, |
| "expected_accuracy": 0.9963196635120924, |
| "expected_risk": 0.003680336487907554 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1254, |
| "coverage": 0.9008620689655172, |
| "accuracy": 0.9984012789768185, |
| "expected_accuracy": 0.9972089314194578, |
| "expected_risk": 0.002791068580542233 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1201, |
| "coverage": 0.8627873563218391, |
| "accuracy": 0.9991673605328892, |
| "expected_accuracy": 0.9991673605328892, |
| "expected_risk": 0.0008326394671107629 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1152, |
| "coverage": 0.8275862068965517, |
| "accuracy": 0.9991319444444444, |
| "expected_accuracy": 0.9991319444444444, |
| "expected_risk": 0.0008680555555555802 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1076, |
| "coverage": 0.7729885057471264, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1392, |
| "hard_count": 1279, |
| "accuracy": 0.9976544175136826, |
| "expected_accuracy": 0.9567768199233716, |
| "nll": 0.08536951007564594, |
| "brier": 0.01017585055261676, |
| "expected_brier": 0.05124385821545201, |
| "ece_bins": 15, |
| "multiclass_ece": 0.014056450259188788, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1340, |
| "coverage": 0.9626436781609196, |
| "accuracy": 0.9976525821596244, |
| "expected_accuracy": 0.9743781094527364, |
| "expected_risk": 0.025621890547263604 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1268, |
| "coverage": 0.9109195402298851, |
| "accuracy": 0.9984164687252574, |
| "expected_accuracy": 0.9963196635120924, |
| "expected_risk": 0.003680336487907554 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1254, |
| "coverage": 0.9008620689655172, |
| "accuracy": 0.9984012789768185, |
| "expected_accuracy": 0.9972089314194578, |
| "expected_risk": 0.002791068580542233 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1201, |
| "coverage": 0.8627873563218391, |
| "accuracy": 0.9991673605328892, |
| "expected_accuracy": 0.9991673605328892, |
| "expected_risk": 0.0008326394671107629 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1152, |
| "coverage": 0.8275862068965517, |
| "accuracy": 0.9991319444444444, |
| "expected_accuracy": 0.9991319444444444, |
| "expected_risk": 0.0008680555555555802 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1076, |
| "coverage": 0.7729885057471264, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "tic_tac_toe-v1": { |
| "count": 472, |
| "successful": 472, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 256, |
| "accuracy_all_rows_failures_incorrect": 0.64453125, |
| "expected_accuracy_all_rows_failures_zero": 0.5206315577078289, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 174, |
| "total": 472, |
| "coverage": 0.3686440677966102 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 34, |
| "total": 472, |
| "coverage": 0.07203389830508475 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 7, |
| "total": 472, |
| "coverage": 0.014830508474576272 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 472, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 472, |
| "hard_count": 256, |
| "accuracy": 0.64453125, |
| "expected_accuracy": 0.5206315577078289, |
| "nll": 1.0984003514512244, |
| "brier": 0.3480926885371523, |
| "expected_brier": 0.6180684754620918, |
| "ece_bins": 15, |
| "multiclass_ece": 0.06619460047533682, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 174, |
| "coverage": 0.3686440677966102, |
| "accuracy": 0.8152173913043478, |
| "expected_accuracy": 0.632183908045977, |
| "expected_risk": 0.367816091954023 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 34, |
| "coverage": 0.07203389830508475, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.7892156862745098, |
| "expected_risk": 0.21078431372549022 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 7, |
| "coverage": 0.014830508474576272, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.7857142857142857, |
| "expected_risk": 0.2142857142857143 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 472, |
| "hard_count": 256, |
| "accuracy": 0.64453125, |
| "expected_accuracy": 0.5206315577078289, |
| "nll": 1.0984003514512244, |
| "brier": 0.3480926885371523, |
| "expected_brier": 0.6180684754620918, |
| "ece_bins": 15, |
| "multiclass_ece": 0.06619460047533682, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 174, |
| "coverage": 0.3686440677966102, |
| "accuracy": 0.8152173913043478, |
| "expected_accuracy": 0.632183908045977, |
| "expected_risk": 0.367816091954023 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 34, |
| "coverage": 0.07203389830508475, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.7892156862745098, |
| "expected_risk": 0.21078431372549022 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 7, |
| "coverage": 0.014830508474576272, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.7857142857142857, |
| "expected_risk": 0.2142857142857143 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "tile_platformer-v1": { |
| "count": 202, |
| "successful": 202, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 202, |
| "accuracy_all_rows_failures_incorrect": 0.9752475247524752, |
| "expected_accuracy_all_rows_failures_zero": 0.9752475247524752, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 202, |
| "total": 202, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 164, |
| "total": 202, |
| "coverage": 0.8118811881188119 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 202, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 202, |
| "hard_count": 202, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "nll": 0.13968696905117425, |
| "brier": 0.047840610560813734, |
| "expected_brier": 0.047840610560813734, |
| "ece_bins": 15, |
| "multiclass_ece": 0.035747153706735625, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 164, |
| "coverage": 0.8118811881188119, |
| "accuracy": 0.9939024390243902, |
| "expected_accuracy": 0.9939024390243902, |
| "expected_risk": 0.0060975609756097615 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 202, |
| "hard_count": 202, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "nll": 0.13968696905117425, |
| "brier": 0.047840610560813734, |
| "expected_brier": 0.047840610560813734, |
| "ece_bins": 15, |
| "multiclass_ece": 0.035747153706735625, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 202, |
| "coverage": 1.0, |
| "accuracy": 0.9752475247524752, |
| "expected_accuracy": 0.9752475247524752, |
| "expected_risk": 0.024752475247524774 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 164, |
| "coverage": 0.8118811881188119, |
| "accuracy": 0.9939024390243902, |
| "expected_accuracy": 0.9939024390243902, |
| "expected_risk": 0.0060975609756097615 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "trex_runner-v1": { |
| "count": 4, |
| "successful": 4, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 4, |
| "accuracy_all_rows_failures_incorrect": 0.75, |
| "expected_accuracy_all_rows_failures_zero": 0.75, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "total": 4, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 4, |
| "total": 4, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 4, |
| "total": 4, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3, |
| "total": 4, |
| "coverage": 0.75 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3, |
| "total": 4, |
| "coverage": 0.75 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 4, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 4, |
| "hard_count": 4, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "nll": 0.7668877262100418, |
| "brier": 0.4059017097317489, |
| "expected_brier": 0.40590170973174894, |
| "ece_bins": 15, |
| "multiclass_ece": 0.2272462205045263, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3, |
| "coverage": 0.75, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3, |
| "coverage": 0.75, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 4, |
| "hard_count": 4, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "nll": 0.7668877262100418, |
| "brier": 0.4059017097317489, |
| "expected_brier": 0.40590170973174894, |
| "ece_bins": 15, |
| "multiclass_ece": 0.2272462205045263, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 4, |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 3, |
| "coverage": 0.75, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 3, |
| "coverage": 0.75, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "vizdoom-basic-v1": { |
| "count": 858, |
| "successful": 858, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 858, |
| "accuracy_all_rows_failures_incorrect": 0.9766899766899767, |
| "expected_accuracy_all_rows_failures_zero": 0.9766899766899767, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 858, |
| "total": 858, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 842, |
| "total": 858, |
| "coverage": 0.9813519813519813 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 823, |
| "total": 858, |
| "coverage": 0.9592074592074592 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 791, |
| "total": 858, |
| "coverage": 0.921911421911422 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 718, |
| "total": 858, |
| "coverage": 0.8368298368298368 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "total": 858, |
| "coverage": 0.6573426573426573 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 858, |
| "hard_count": 858, |
| "accuracy": 0.9766899766899767, |
| "expected_accuracy": 0.9766899766899767, |
| "nll": 0.06761322089895994, |
| "brier": 0.034824071109677196, |
| "expected_brier": 0.034824071109677196, |
| "ece_bins": 15, |
| "multiclass_ece": 0.012718603010498264, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 858, |
| "coverage": 1.0, |
| "accuracy": 0.9766899766899767, |
| "expected_accuracy": 0.9766899766899767, |
| "expected_risk": 0.023310023310023298 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 842, |
| "coverage": 0.9813519813519813, |
| "accuracy": 0.9857482185273159, |
| "expected_accuracy": 0.9857482185273159, |
| "expected_risk": 0.014251781472684133 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 823, |
| "coverage": 0.9592074592074592, |
| "accuracy": 0.9902794653705954, |
| "expected_accuracy": 0.9902794653705954, |
| "expected_risk": 0.009720534629404587 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 791, |
| "coverage": 0.921911421911422, |
| "accuracy": 0.9974715549936789, |
| "expected_accuracy": 0.9974715549936789, |
| "expected_risk": 0.0025284450063211006 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 718, |
| "coverage": 0.8368298368298368, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "coverage": 0.6573426573426573, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 858, |
| "hard_count": 858, |
| "accuracy": 0.9766899766899767, |
| "expected_accuracy": 0.9766899766899767, |
| "nll": 0.06761322089895994, |
| "brier": 0.034824071109677196, |
| "expected_brier": 0.034824071109677196, |
| "ece_bins": 15, |
| "multiclass_ece": 0.012718603010498264, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 858, |
| "coverage": 1.0, |
| "accuracy": 0.9766899766899767, |
| "expected_accuracy": 0.9766899766899767, |
| "expected_risk": 0.023310023310023298 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 842, |
| "coverage": 0.9813519813519813, |
| "accuracy": 0.9857482185273159, |
| "expected_accuracy": 0.9857482185273159, |
| "expected_risk": 0.014251781472684133 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 823, |
| "coverage": 0.9592074592074592, |
| "accuracy": 0.9902794653705954, |
| "expected_accuracy": 0.9902794653705954, |
| "expected_risk": 0.009720534629404587 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 791, |
| "coverage": 0.921911421911422, |
| "accuracy": 0.9974715549936789, |
| "expected_accuracy": 0.9974715549936789, |
| "expected_risk": 0.0025284450063211006 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 718, |
| "coverage": 0.8368298368298368, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 564, |
| "coverage": 0.6573426573426573, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "wikispeedia-v1": { |
| "count": 176, |
| "successful": 176, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 51, |
| "accuracy_all_rows_failures_incorrect": 0.45098039215686275, |
| "expected_accuracy_all_rows_failures_zero": 0.2709042535747081, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3, |
| "total": 176, |
| "coverage": 0.017045454545454544 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 176, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 176, |
| "hard_count": 51, |
| "accuracy": 0.45098039215686275, |
| "expected_accuracy": 0.2709042535747081, |
| "nll": 2.1680481914874337, |
| "brier": 0.31628613125660077, |
| "expected_brier": 0.8504008170758321, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0965942356793526, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3, |
| "coverage": 0.017045454545454544, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 176, |
| "hard_count": 51, |
| "accuracy": 0.45098039215686275, |
| "expected_accuracy": 0.2709042535747081, |
| "nll": 2.1680481914874337, |
| "brier": 0.31628613125660077, |
| "expected_brier": 0.8504008170758321, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0965942356793526, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3, |
| "coverage": 0.017045454545454544, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/agent_trace_observability": { |
| "count": 357, |
| "successful": 357, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 357, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 1.0, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 357, |
| "total": 357, |
| "coverage": 1.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 357, |
| "hard_count": 357, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.00044687482748441643, |
| "brier": 2.946417322061725e-06, |
| "expected_brier": 2.946417322060909e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0004461355302124166, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 357, |
| "hard_count": 357, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.00044687482748441643, |
| "brier": 2.946417322061725e-06, |
| "expected_brier": 2.946417322060909e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0004461355302124166, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 357, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/customer_service": { |
| "count": 456, |
| "successful": 456, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 456, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 1.0, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 456, |
| "total": 456, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 455, |
| "total": 456, |
| "coverage": 0.9978070175438597 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 456, |
| "hard_count": 456, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.0004573076748327324, |
| "brier": 8.826622033700542e-06, |
| "expected_brier": 8.826622033703088e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0004550441009002813, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 455, |
| "coverage": 0.9978070175438597, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 456, |
| "hard_count": 456, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.0004573076748327324, |
| "brier": 8.826622033700542e-06, |
| "expected_brier": 8.826622033703088e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0004550441009002813, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 456, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 455, |
| "coverage": 0.9978070175438597, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/invoice_processing": { |
| "count": 930, |
| "successful": 930, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 930, |
| "accuracy_all_rows_failures_incorrect": 0.9967741935483871, |
| "expected_accuracy_all_rows_failures_zero": 0.9967741935483871, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "total": 930, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 927, |
| "total": 930, |
| "coverage": 0.9967741935483871 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 925, |
| "total": 930, |
| "coverage": 0.9946236559139785 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 924, |
| "total": 930, |
| "coverage": 0.9935483870967742 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 924, |
| "total": 930, |
| "coverage": 0.9935483870967742 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 917, |
| "total": 930, |
| "coverage": 0.986021505376344 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 930, |
| "hard_count": 930, |
| "accuracy": 0.9967741935483871, |
| "expected_accuracy": 0.9967741935483871, |
| "nll": 0.005458157001187219, |
| "brier": 0.003648645648216325, |
| "expected_brier": 0.0036486456482163225, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0037258424516831987, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "coverage": 1.0, |
| "accuracy": 0.9967741935483871, |
| "expected_accuracy": 0.9967741935483871, |
| "expected_risk": 0.003225806451612856 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 927, |
| "coverage": 0.9967741935483871, |
| "accuracy": 0.9978425026968716, |
| "expected_accuracy": 0.9978425026968716, |
| "expected_risk": 0.0021574973031284195 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 925, |
| "coverage": 0.9946236559139785, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 917, |
| "coverage": 0.986021505376344, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 930, |
| "hard_count": 930, |
| "accuracy": 0.9967741935483871, |
| "expected_accuracy": 0.9967741935483871, |
| "nll": 0.005458157001187219, |
| "brier": 0.003648645648216325, |
| "expected_brier": 0.0036486456482163225, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0037258424516831987, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 930, |
| "coverage": 1.0, |
| "accuracy": 0.9967741935483871, |
| "expected_accuracy": 0.9967741935483871, |
| "expected_risk": 0.003225806451612856 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 927, |
| "coverage": 0.9967741935483871, |
| "accuracy": 0.9978425026968716, |
| "expected_accuracy": 0.9978425026968716, |
| "expected_risk": 0.0021574973031284195 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 925, |
| "coverage": 0.9946236559139785, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 924, |
| "coverage": 0.9935483870967742, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 917, |
| "coverage": 0.986021505376344, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/security_incidents": { |
| "count": 1326, |
| "successful": 1326, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1326, |
| "accuracy_all_rows_failures_incorrect": 0.9932126696832579, |
| "expected_accuracy_all_rows_failures_zero": 0.9932126696832579, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "total": 1326, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1325, |
| "total": 1326, |
| "coverage": 0.9992458521870287 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1325, |
| "total": 1326, |
| "coverage": 0.9992458521870287 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1318, |
| "total": 1326, |
| "coverage": 0.9939668174962293 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1303, |
| "total": 1326, |
| "coverage": 0.9826546003016591 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1291, |
| "total": 1326, |
| "coverage": 0.973604826546003 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1326, |
| "hard_count": 1326, |
| "accuracy": 0.9932126696832579, |
| "expected_accuracy": 0.9932126696832579, |
| "nll": 0.022337704900579736, |
| "brier": 0.011355443994287867, |
| "expected_brier": 0.011355443994287865, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004366133269192267, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "coverage": 1.0, |
| "accuracy": 0.9932126696832579, |
| "expected_accuracy": 0.9932126696832579, |
| "expected_risk": 0.006787330316742057 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1325, |
| "coverage": 0.9992458521870287, |
| "accuracy": 0.9939622641509434, |
| "expected_accuracy": 0.9939622641509434, |
| "expected_risk": 0.006037735849056647 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1325, |
| "coverage": 0.9992458521870287, |
| "accuracy": 0.9939622641509434, |
| "expected_accuracy": 0.9939622641509434, |
| "expected_risk": 0.006037735849056647 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1318, |
| "coverage": 0.9939668174962293, |
| "accuracy": 0.9954476479514416, |
| "expected_accuracy": 0.9954476479514416, |
| "expected_risk": 0.004552352048558417 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1303, |
| "coverage": 0.9826546003016591, |
| "accuracy": 0.9984650805832693, |
| "expected_accuracy": 0.9984650805832693, |
| "expected_risk": 0.0015349194167306734 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1291, |
| "coverage": 0.973604826546003, |
| "accuracy": 0.9984508133230054, |
| "expected_accuracy": 0.9984508133230054, |
| "expected_risk": 0.0015491866769945517 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1326, |
| "hard_count": 1326, |
| "accuracy": 0.9932126696832579, |
| "expected_accuracy": 0.9932126696832579, |
| "nll": 0.022337704900579736, |
| "brier": 0.011355443994287867, |
| "expected_brier": 0.011355443994287865, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004366133269192267, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1326, |
| "coverage": 1.0, |
| "accuracy": 0.9932126696832579, |
| "expected_accuracy": 0.9932126696832579, |
| "expected_risk": 0.006787330316742057 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1325, |
| "coverage": 0.9992458521870287, |
| "accuracy": 0.9939622641509434, |
| "expected_accuracy": 0.9939622641509434, |
| "expected_risk": 0.006037735849056647 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1325, |
| "coverage": 0.9992458521870287, |
| "accuracy": 0.9939622641509434, |
| "expected_accuracy": 0.9939622641509434, |
| "expected_risk": 0.006037735849056647 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1318, |
| "coverage": 0.9939668174962293, |
| "accuracy": 0.9954476479514416, |
| "expected_accuracy": 0.9954476479514416, |
| "expected_risk": 0.004552352048558417 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1303, |
| "coverage": 0.9826546003016591, |
| "accuracy": 0.9984650805832693, |
| "expected_accuracy": 0.9984650805832693, |
| "expected_risk": 0.0015349194167306734 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1291, |
| "coverage": 0.973604826546003, |
| "accuracy": 0.9984508133230054, |
| "expected_accuracy": 0.9984508133230054, |
| "expected_risk": 0.0015491866769945517 |
| } |
| ] |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "count": 2584, |
| "successful": 2584, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2119, |
| "accuracy_all_rows_failures_incorrect": 0.9188296366210477, |
| "expected_accuracy_all_rows_failures_zero": 0.8180149292571892, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2050, |
| "total": 2584, |
| "coverage": 0.7933436532507739 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1783, |
| "total": 2584, |
| "coverage": 0.690015479876161 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1715, |
| "total": 2584, |
| "coverage": 0.6636996904024768 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1620, |
| "total": 2584, |
| "coverage": 0.6269349845201239 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1458, |
| "total": 2584, |
| "coverage": 0.5642414860681114 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1056, |
| "total": 2584, |
| "coverage": 0.4086687306501548 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 2584, |
| "hard_count": 2119, |
| "accuracy": 0.9188296366210477, |
| "expected_accuracy": 0.8180149292571892, |
| "nll": 0.45266508836146735, |
| "brier": 0.11881345254759784, |
| "expected_brier": 0.228759049606198, |
| "ece_bins": 15, |
| "multiclass_ece": 0.027110901433906452, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2050, |
| "coverage": 0.7933436532507739, |
| "accuracy": 0.9651898734177216, |
| "expected_accuracy": 0.9271544715447155, |
| "expected_risk": 0.07284552845528447 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1783, |
| "coverage": 0.690015479876161, |
| "accuracy": 0.985827664399093, |
| "expected_accuracy": 0.9804636380631894, |
| "expected_risk": 0.01953636193681063 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1715, |
| "coverage": 0.6636996904024768, |
| "accuracy": 0.9900526623756583, |
| "expected_accuracy": 0.9883381924198251, |
| "expected_risk": 0.011661807580174877 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1620, |
| "coverage": 0.6269349845201239, |
| "accuracy": 0.9950617283950617, |
| "expected_accuracy": 0.9950617283950617, |
| "expected_risk": 0.004938271604938316 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1458, |
| "coverage": 0.5642414860681114, |
| "accuracy": 0.9986282578875172, |
| "expected_accuracy": 0.9986282578875172, |
| "expected_risk": 0.00137174211248281 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1056, |
| "coverage": 0.4086687306501548, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 2584, |
| "hard_count": 2119, |
| "accuracy": 0.9188296366210477, |
| "expected_accuracy": 0.8180149292571892, |
| "nll": 0.45266508836146735, |
| "brier": 0.11881345254759784, |
| "expected_brier": 0.228759049606198, |
| "ece_bins": 15, |
| "multiclass_ece": 0.027110901433906452, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2050, |
| "coverage": 0.7933436532507739, |
| "accuracy": 0.9651898734177216, |
| "expected_accuracy": 0.9271544715447155, |
| "expected_risk": 0.07284552845528447 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1783, |
| "coverage": 0.690015479876161, |
| "accuracy": 0.985827664399093, |
| "expected_accuracy": 0.9804636380631894, |
| "expected_risk": 0.01953636193681063 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1715, |
| "coverage": 0.6636996904024768, |
| "accuracy": 0.9900526623756583, |
| "expected_accuracy": 0.9883381924198251, |
| "expected_risk": 0.011661807580174877 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1620, |
| "coverage": 0.6269349845201239, |
| "accuracy": 0.9950617283950617, |
| "expected_accuracy": 0.9950617283950617, |
| "expected_risk": 0.004938271604938316 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1458, |
| "coverage": 0.5642414860681114, |
| "accuracy": 0.9986282578875172, |
| "expected_accuracy": 0.9986282578875172, |
| "expected_risk": 0.00137174211248281 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1056, |
| "coverage": 0.4086687306501548, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "noul": { |
| "count": 6364, |
| "successful": 6364, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 6364, |
| "accuracy_all_rows_failures_incorrect": 0.9951288497800126, |
| "expected_accuracy_all_rows_failures_zero": 0.9951288497800126, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "total": 6364, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6347, |
| "total": 6364, |
| "coverage": 0.9973287240729101 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6338, |
| "total": 6364, |
| "coverage": 0.9959145191703331 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6301, |
| "total": 6364, |
| "coverage": 0.990100565681961 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6274, |
| "total": 6364, |
| "coverage": 0.98585795097423 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6176, |
| "total": 6364, |
| "coverage": 0.9704588309239472 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 6364, |
| "hard_count": 6364, |
| "accuracy": 0.9951288497800126, |
| "expected_accuracy": 0.9951288497800126, |
| "nll": 0.014164294737203246, |
| "brier": 0.007765035196910086, |
| "expected_brier": 0.007765035196910086, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002491467968884566, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "coverage": 1.0, |
| "accuracy": 0.9951288497800126, |
| "expected_accuracy": 0.9951288497800126, |
| "expected_risk": 0.004871150219987408 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6347, |
| "coverage": 0.9973287240729101, |
| "accuracy": 0.996061131243107, |
| "expected_accuracy": 0.996061131243107, |
| "expected_risk": 0.003938868756892977 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6338, |
| "coverage": 0.9959145191703331, |
| "accuracy": 0.9965288734616599, |
| "expected_accuracy": 0.9965288734616599, |
| "expected_risk": 0.0034711265383401413 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6301, |
| "coverage": 0.990100565681961, |
| "accuracy": 0.9980955403904143, |
| "expected_accuracy": 0.9980955403904143, |
| "expected_risk": 0.0019044596095857402 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6274, |
| "coverage": 0.98585795097423, |
| "accuracy": 0.9993624481989162, |
| "expected_accuracy": 0.9993624481989162, |
| "expected_risk": 0.0006375518010838244 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6176, |
| "coverage": 0.9704588309239472, |
| "accuracy": 0.9996761658031088, |
| "expected_accuracy": 0.9996761658031088, |
| "expected_risk": 0.00032383419689119286 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 6364, |
| "hard_count": 6364, |
| "accuracy": 0.9951288497800126, |
| "expected_accuracy": 0.9951288497800126, |
| "nll": 0.014164294737203246, |
| "brier": 0.007765035196910086, |
| "expected_brier": 0.007765035196910086, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002491467968884566, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6364, |
| "coverage": 1.0, |
| "accuracy": 0.9951288497800126, |
| "expected_accuracy": 0.9951288497800126, |
| "expected_risk": 0.004871150219987408 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6347, |
| "coverage": 0.9973287240729101, |
| "accuracy": 0.996061131243107, |
| "expected_accuracy": 0.996061131243107, |
| "expected_risk": 0.003938868756892977 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6338, |
| "coverage": 0.9959145191703331, |
| "accuracy": 0.9965288734616599, |
| "expected_accuracy": 0.9965288734616599, |
| "expected_risk": 0.0034711265383401413 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 6301, |
| "coverage": 0.990100565681961, |
| "accuracy": 0.9980955403904143, |
| "expected_accuracy": 0.9980955403904143, |
| "expected_risk": 0.0019044596095857402 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 6274, |
| "coverage": 0.98585795097423, |
| "accuracy": 0.9993624481989162, |
| "expected_accuracy": 0.9993624481989162, |
| "expected_risk": 0.0006375518010838244 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 6176, |
| "coverage": 0.9704588309239472, |
| "accuracy": 0.9996761658031088, |
| "expected_accuracy": 0.9996761658031088, |
| "expected_risk": 0.00032383419689119286 |
| } |
| ] |
| } |
| }, |
| "score": { |
| "count": 1584, |
| "successful": 1584, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1563, |
| "accuracy_all_rows_failures_incorrect": 0.9718490083173384, |
| "expected_accuracy_all_rows_failures_zero": 0.9650673400673401, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1531, |
| "total": 1584, |
| "coverage": 0.9665404040404041 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1506, |
| "total": 1584, |
| "coverage": 0.9507575757575758 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1499, |
| "total": 1584, |
| "coverage": 0.9463383838383839 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1488, |
| "total": 1584, |
| "coverage": 0.9393939393939394 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1476, |
| "total": 1584, |
| "coverage": 0.9318181818181818 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1442, |
| "total": 1584, |
| "coverage": 0.9103535353535354 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1584, |
| "hard_count": 1563, |
| "accuracy": 0.9718490083173384, |
| "expected_accuracy": 0.9650673400673401, |
| "nll": 0.07532023419931186, |
| "brier": 0.03504193237136221, |
| "expected_brier": 0.042196814526244374, |
| "ece_bins": 15, |
| "multiclass_ece": 0.010182639374983222, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1531, |
| "coverage": 0.9665404040404041, |
| "accuracy": 0.990125082290981, |
| "expected_accuracy": 0.9862834748530372, |
| "expected_risk": 0.013716525146962755 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1506, |
| "coverage": 0.9507575757575758, |
| "accuracy": 0.9926958831341302, |
| "expected_accuracy": 0.9926958831341302, |
| "expected_risk": 0.007304116865869847 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1499, |
| "coverage": 0.9463383838383839, |
| "accuracy": 0.9959973315543695, |
| "expected_accuracy": 0.9959973315543695, |
| "expected_risk": 0.004002668445630464 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1488, |
| "coverage": 0.9393939393939394, |
| "accuracy": 0.9993279569892473, |
| "expected_accuracy": 0.9993279569892473, |
| "expected_risk": 0.0006720430107527431 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1476, |
| "coverage": 0.9318181818181818, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1442, |
| "coverage": 0.9103535353535354, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1584, |
| "hard_count": 1563, |
| "accuracy": 0.9718490083173384, |
| "expected_accuracy": 0.9650673400673401, |
| "nll": 0.07532023419931186, |
| "brier": 0.03504193237136221, |
| "expected_brier": 0.042196814526244374, |
| "ece_bins": 15, |
| "multiclass_ece": 0.010182639374983222, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1531, |
| "coverage": 0.9665404040404041, |
| "accuracy": 0.990125082290981, |
| "expected_accuracy": 0.9862834748530372, |
| "expected_risk": 0.013716525146962755 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1506, |
| "coverage": 0.9507575757575758, |
| "accuracy": 0.9926958831341302, |
| "expected_accuracy": 0.9926958831341302, |
| "expected_risk": 0.007304116865869847 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1499, |
| "coverage": 0.9463383838383839, |
| "accuracy": 0.9959973315543695, |
| "expected_accuracy": 0.9959973315543695, |
| "expected_risk": 0.004002668445630464 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1488, |
| "coverage": 0.9393939393939394, |
| "accuracy": 0.9993279569892473, |
| "expected_accuracy": 0.9993279569892473, |
| "expected_risk": 0.0006720430107527431 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1476, |
| "coverage": 0.9318181818181818, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1442, |
| "coverage": 0.9103535353535354, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| } |
| } |
| }, |
| "ood": { |
| "count": 15920, |
| "successful": 15920, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 15446, |
| "accuracy_all_rows_failures_incorrect": 0.9196555742587078, |
| "expected_accuracy_all_rows_failures_zero": 0.9040437668138673, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15349, |
| "total": 15920, |
| "coverage": 0.9641331658291458 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14632, |
| "total": 15920, |
| "coverage": 0.9190954773869346 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14183, |
| "total": 15920, |
| "coverage": 0.8908919597989949 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 13442, |
| "total": 15920, |
| "coverage": 0.8443467336683417 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 12751, |
| "total": 15920, |
| "coverage": 0.8009422110552764 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 10671, |
| "total": 15920, |
| "coverage": 0.6702889447236181 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 15920, |
| "hard_count": 15446, |
| "accuracy": 0.9196555742587078, |
| "expected_accuracy": 0.9040437668138673, |
| "nll": 0.2994407995801986, |
| "brier": 0.1266465263060649, |
| "expected_brier": 0.1439772659530734, |
| "ece_bins": 15, |
| "multiclass_ece": 0.037398104632606045, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15349, |
| "coverage": 0.9641331658291458, |
| "accuracy": 0.9263081777544044, |
| "expected_accuracy": 0.9224922361934546, |
| "expected_risk": 0.07750776380654545 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14632, |
| "coverage": 0.9190954773869346, |
| "accuracy": 0.939354574046219, |
| "expected_accuracy": 0.9391630216876253, |
| "expected_risk": 0.06083697831237467 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14183, |
| "coverage": 0.8908919597989949, |
| "accuracy": 0.9484557890283458, |
| "expected_accuracy": 0.9484241697807234, |
| "expected_risk": 0.051575830219276586 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 13442, |
| "coverage": 0.8443467336683417, |
| "accuracy": 0.9607172085410312, |
| "expected_accuracy": 0.9606829340871894, |
| "expected_risk": 0.03931706591281059 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 12751, |
| "coverage": 0.8009422110552764, |
| "accuracy": 0.9714509803921568, |
| "expected_accuracy": 0.971414006744569, |
| "expected_risk": 0.028585993255430986 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 10671, |
| "coverage": 0.6702889447236181, |
| "accuracy": 0.9867866179364633, |
| "expected_accuracy": 0.9867866179364633, |
| "expected_risk": 0.013213382063536727 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 15920, |
| "hard_count": 15446, |
| "accuracy": 0.9196555742587078, |
| "expected_accuracy": 0.9040437668138673, |
| "nll": 0.2994407995801986, |
| "brier": 0.1266465263060649, |
| "expected_brier": 0.1439772659530734, |
| "ece_bins": 15, |
| "multiclass_ece": 0.037398104632606045, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 15349, |
| "coverage": 0.9641331658291458, |
| "accuracy": 0.9263081777544044, |
| "expected_accuracy": 0.9224922361934546, |
| "expected_risk": 0.07750776380654545 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 14632, |
| "coverage": 0.9190954773869346, |
| "accuracy": 0.939354574046219, |
| "expected_accuracy": 0.9391630216876253, |
| "expected_risk": 0.06083697831237467 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 14183, |
| "coverage": 0.8908919597989949, |
| "accuracy": 0.9484557890283458, |
| "expected_accuracy": 0.9484241697807234, |
| "expected_risk": 0.051575830219276586 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 13442, |
| "coverage": 0.8443467336683417, |
| "accuracy": 0.9607172085410312, |
| "expected_accuracy": 0.9606829340871894, |
| "expected_risk": 0.03931706591281059 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 12751, |
| "coverage": 0.8009422110552764, |
| "accuracy": 0.9714509803921568, |
| "expected_accuracy": 0.971414006744569, |
| "expected_risk": 0.028585993255430986 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 10671, |
| "coverage": 0.6702889447236181, |
| "accuracy": 0.9867866179364633, |
| "expected_accuracy": 0.9867866179364633, |
| "expected_risk": 0.013213382063536727 |
| } |
| ] |
| }, |
| "by_source": { |
| "customer-control-v1": { |
| "count": 600, |
| "successful": 600, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 560, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 0.9633333333333334, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 586, |
| "total": 600, |
| "coverage": 0.9766666666666667 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 559, |
| "total": 600, |
| "coverage": 0.9316666666666666 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 555, |
| "total": 600, |
| "coverage": 0.925 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 548, |
| "total": 600, |
| "coverage": 0.9133333333333333 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 540, |
| "total": 600, |
| "coverage": 0.9 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 512, |
| "total": 600, |
| "coverage": 0.8533333333333334 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 600, |
| "hard_count": 560, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9633333333333334, |
| "nll": 0.06367998388489432, |
| "brier": 0.002432497344975918, |
| "expected_brier": 0.03909916401164259, |
| "ece_bins": 15, |
| "multiclass_ece": 0.007595238345599102, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 586, |
| "coverage": 0.9766666666666667, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.976962457337884, |
| "expected_risk": 0.023037542662115995 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 559, |
| "coverage": 0.9316666666666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9991055456171736, |
| "expected_risk": 0.0008944543828264262 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 555, |
| "coverage": 0.925, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 548, |
| "coverage": 0.9133333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 540, |
| "coverage": 0.9, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 512, |
| "coverage": 0.8533333333333334, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 600, |
| "hard_count": 560, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9633333333333334, |
| "nll": 0.06367998388489432, |
| "brier": 0.002432497344975918, |
| "expected_brier": 0.03909916401164259, |
| "ece_bins": 15, |
| "multiclass_ece": 0.007595238345599102, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 586, |
| "coverage": 0.9766666666666667, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.976962457337884, |
| "expected_risk": 0.023037542662115995 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 559, |
| "coverage": 0.9316666666666666, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.9991055456171736, |
| "expected_risk": 0.0008944543828264262 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 555, |
| "coverage": 0.925, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 548, |
| "coverage": 0.9133333333333333, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 540, |
| "coverage": 0.9, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 512, |
| "coverage": 0.8533333333333334, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "painting-geometry-v1": { |
| "count": 6912, |
| "successful": 6912, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 6912, |
| "accuracy_all_rows_failures_incorrect": 0.8629918981481481, |
| "expected_accuracy_all_rows_failures_zero": 0.8629918981481481, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6842, |
| "total": 6912, |
| "coverage": 0.9898726851851852 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6420, |
| "total": 6912, |
| "coverage": 0.9288194444444444 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6102, |
| "total": 6912, |
| "coverage": 0.8828125 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 5666, |
| "total": 6912, |
| "coverage": 0.8197337962962963 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5246, |
| "total": 6912, |
| "coverage": 0.7589699074074074 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3677, |
| "total": 6912, |
| "coverage": 0.5319733796296297 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 6912, |
| "hard_count": 6912, |
| "accuracy": 0.8629918981481481, |
| "expected_accuracy": 0.8629918981481481, |
| "nll": 0.4601822966946938, |
| "brier": 0.22129407318457964, |
| "expected_brier": 0.22129407318457964, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0793900158186277, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6842, |
| "coverage": 0.9898726851851852, |
| "accuracy": 0.8661210172464192, |
| "expected_accuracy": 0.8661210172464192, |
| "expected_risk": 0.1338789827535808 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6420, |
| "coverage": 0.9288194444444444, |
| "accuracy": 0.8848909657320873, |
| "expected_accuracy": 0.8848909657320873, |
| "expected_risk": 0.11510903426791275 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6102, |
| "coverage": 0.8828125, |
| "accuracy": 0.8982300884955752, |
| "expected_accuracy": 0.8982300884955752, |
| "expected_risk": 0.10176991150442483 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 5666, |
| "coverage": 0.8197337962962963, |
| "accuracy": 0.9200494175785386, |
| "expected_accuracy": 0.9200494175785386, |
| "expected_risk": 0.07995058242146136 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5246, |
| "coverage": 0.7589699074074074, |
| "accuracy": 0.9412886008387342, |
| "expected_accuracy": 0.9412886008387342, |
| "expected_risk": 0.058711399161265776 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3677, |
| "coverage": 0.5319733796296297, |
| "accuracy": 0.9709001903725863, |
| "expected_accuracy": 0.9709001903725863, |
| "expected_risk": 0.029099809627413653 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 6912, |
| "hard_count": 6912, |
| "accuracy": 0.8629918981481481, |
| "expected_accuracy": 0.8629918981481481, |
| "nll": 0.4601822966946938, |
| "brier": 0.22129407318457964, |
| "expected_brier": 0.22129407318457964, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0793900158186277, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 6842, |
| "coverage": 0.9898726851851852, |
| "accuracy": 0.8661210172464192, |
| "expected_accuracy": 0.8661210172464192, |
| "expected_risk": 0.1338789827535808 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 6420, |
| "coverage": 0.9288194444444444, |
| "accuracy": 0.8848909657320873, |
| "expected_accuracy": 0.8848909657320873, |
| "expected_risk": 0.11510903426791275 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 6102, |
| "coverage": 0.8828125, |
| "accuracy": 0.8982300884955752, |
| "expected_accuracy": 0.8982300884955752, |
| "expected_risk": 0.10176991150442483 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 5666, |
| "coverage": 0.8197337962962963, |
| "accuracy": 0.9200494175785386, |
| "expected_accuracy": 0.9200494175785386, |
| "expected_risk": 0.07995058242146136 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 5246, |
| "coverage": 0.7589699074074074, |
| "accuracy": 0.9412886008387342, |
| "expected_accuracy": 0.9412886008387342, |
| "expected_risk": 0.058711399161265776 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 3677, |
| "coverage": 0.5319733796296297, |
| "accuracy": 0.9709001903725863, |
| "expected_accuracy": 0.9709001903725863, |
| "expected_risk": 0.029099809627413653 |
| } |
| ] |
| } |
| }, |
| "reasoning-control-v1": { |
| "count": 750, |
| "successful": 750, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 750, |
| "accuracy_all_rows_failures_incorrect": 0.736, |
| "expected_accuracy_all_rows_failures_zero": 0.736, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 644, |
| "total": 750, |
| "coverage": 0.8586666666666667 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 542, |
| "total": 750, |
| "coverage": 0.7226666666666667 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 492, |
| "total": 750, |
| "coverage": 0.656 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 414, |
| "total": 750, |
| "coverage": 0.552 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 373, |
| "total": 750, |
| "coverage": 0.49733333333333335 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 245, |
| "total": 750, |
| "coverage": 0.32666666666666666 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.736, |
| "expected_accuracy": 0.736, |
| "nll": 0.6801700551809211, |
| "brier": 0.3601332438976597, |
| "expected_brier": 0.3601332438976597, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0941763492512841, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 644, |
| "coverage": 0.8586666666666667, |
| "accuracy": 0.8012422360248447, |
| "expected_accuracy": 0.8012422360248447, |
| "expected_risk": 0.19875776397515532 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 542, |
| "coverage": 0.7226666666666667, |
| "accuracy": 0.8431734317343174, |
| "expected_accuracy": 0.8431734317343174, |
| "expected_risk": 0.15682656826568264 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 492, |
| "coverage": 0.656, |
| "accuracy": 0.8760162601626016, |
| "expected_accuracy": 0.8760162601626016, |
| "expected_risk": 0.12398373983739841 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 414, |
| "coverage": 0.552, |
| "accuracy": 0.9106280193236715, |
| "expected_accuracy": 0.9106280193236715, |
| "expected_risk": 0.08937198067632846 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 373, |
| "coverage": 0.49733333333333335, |
| "accuracy": 0.9276139410187667, |
| "expected_accuracy": 0.9276139410187667, |
| "expected_risk": 0.07238605898123329 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 245, |
| "coverage": 0.32666666666666666, |
| "accuracy": 0.9591836734693877, |
| "expected_accuracy": 0.9591836734693877, |
| "expected_risk": 0.04081632653061229 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.736, |
| "expected_accuracy": 0.736, |
| "nll": 0.6801700551809211, |
| "brier": 0.3601332438976597, |
| "expected_brier": 0.3601332438976597, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0941763492512841, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 644, |
| "coverage": 0.8586666666666667, |
| "accuracy": 0.8012422360248447, |
| "expected_accuracy": 0.8012422360248447, |
| "expected_risk": 0.19875776397515532 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 542, |
| "coverage": 0.7226666666666667, |
| "accuracy": 0.8431734317343174, |
| "expected_accuracy": 0.8431734317343174, |
| "expected_risk": 0.15682656826568264 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 492, |
| "coverage": 0.656, |
| "accuracy": 0.8760162601626016, |
| "expected_accuracy": 0.8760162601626016, |
| "expected_risk": 0.12398373983739841 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 414, |
| "coverage": 0.552, |
| "accuracy": 0.9106280193236715, |
| "expected_accuracy": 0.9106280193236715, |
| "expected_risk": 0.08937198067632846 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 373, |
| "coverage": 0.49733333333333335, |
| "accuracy": 0.9276139410187667, |
| "expected_accuracy": 0.9276139410187667, |
| "expected_risk": 0.07238605898123329 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 245, |
| "coverage": 0.32666666666666666, |
| "accuracy": 0.9591836734693877, |
| "expected_accuracy": 0.9591836734693877, |
| "expected_risk": 0.04081632653061229 |
| } |
| ] |
| } |
| }, |
| "snake-v1": { |
| "count": 3464, |
| "successful": 3464, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 3192, |
| "accuracy_all_rows_failures_incorrect": 0.9899749373433584, |
| "expected_accuracy_all_rows_failures_zero": 0.9514049268668205, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3299, |
| "total": 3464, |
| "coverage": 0.9523672055427251 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3151, |
| "total": 3464, |
| "coverage": 0.9096420323325635 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3094, |
| "total": 3464, |
| "coverage": 0.8931870669745958 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2929, |
| "total": 3464, |
| "coverage": 0.8455542725173211 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2779, |
| "total": 3464, |
| "coverage": 0.8022517321016166 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2592, |
| "total": 3464, |
| "coverage": 0.7482678983833718 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3464, |
| "hard_count": 3192, |
| "accuracy": 0.9899749373433584, |
| "expected_accuracy": 0.9514049268668205, |
| "nll": 0.10915249045149827, |
| "brier": 0.020795628523113556, |
| "expected_brier": 0.06015282636760932, |
| "ece_bins": 15, |
| "multiclass_ece": 0.013368579791205319, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3299, |
| "coverage": 0.9523672055427251, |
| "accuracy": 0.9899717956753369, |
| "expected_accuracy": 0.9738809740325352, |
| "expected_risk": 0.02611902596746485 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3151, |
| "coverage": 0.9096420323325635, |
| "accuracy": 0.993006993006993, |
| "expected_accuracy": 0.9921717973130224, |
| "expected_risk": 0.007828202686977592 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3094, |
| "coverage": 0.8931870669745958, |
| "accuracy": 0.9938570966698997, |
| "expected_accuracy": 0.9936974789915967, |
| "expected_risk": 0.006302521008403339 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2929, |
| "coverage": 0.8455542725173211, |
| "accuracy": 0.994535519125683, |
| "expected_accuracy": 0.9943666780471151, |
| "expected_risk": 0.0056333219528849066 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2779, |
| "coverage": 0.8022517321016166, |
| "accuracy": 0.9953203743700504, |
| "expected_accuracy": 0.9951421374595179, |
| "expected_risk": 0.00485786254048215 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2592, |
| "coverage": 0.7482678983833718, |
| "accuracy": 0.9965277777777778, |
| "expected_accuracy": 0.9965277777777778, |
| "expected_risk": 0.00347222222222221 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3464, |
| "hard_count": 3192, |
| "accuracy": 0.9899749373433584, |
| "expected_accuracy": 0.9514049268668205, |
| "nll": 0.10915249045149827, |
| "brier": 0.020795628523113556, |
| "expected_brier": 0.06015282636760932, |
| "ece_bins": 15, |
| "multiclass_ece": 0.013368579791205319, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 3299, |
| "coverage": 0.9523672055427251, |
| "accuracy": 0.9899717956753369, |
| "expected_accuracy": 0.9738809740325352, |
| "expected_risk": 0.02611902596746485 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 3151, |
| "coverage": 0.9096420323325635, |
| "accuracy": 0.993006993006993, |
| "expected_accuracy": 0.9921717973130224, |
| "expected_risk": 0.007828202686977592 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 3094, |
| "coverage": 0.8931870669745958, |
| "accuracy": 0.9938570966698997, |
| "expected_accuracy": 0.9936974789915967, |
| "expected_risk": 0.006302521008403339 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2929, |
| "coverage": 0.8455542725173211, |
| "accuracy": 0.994535519125683, |
| "expected_accuracy": 0.9943666780471151, |
| "expected_risk": 0.0056333219528849066 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2779, |
| "coverage": 0.8022517321016166, |
| "accuracy": 0.9953203743700504, |
| "expected_accuracy": 0.9951421374595179, |
| "expected_risk": 0.00485786254048215 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 2592, |
| "coverage": 0.7482678983833718, |
| "accuracy": 0.9965277777777778, |
| "expected_accuracy": 0.9965277777777778, |
| "expected_risk": 0.00347222222222221 |
| } |
| ] |
| } |
| }, |
| "vizdoom-basic-v1": { |
| "count": 825, |
| "successful": 825, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 825, |
| "accuracy_all_rows_failures_incorrect": 0.9745454545454545, |
| "expected_accuracy_all_rows_failures_zero": 0.9745454545454545, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 824, |
| "total": 825, |
| "coverage": 0.9987878787878788 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 812, |
| "total": 825, |
| "coverage": 0.9842424242424243 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 794, |
| "total": 825, |
| "coverage": 0.9624242424242424 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 748, |
| "total": 825, |
| "coverage": 0.9066666666666666 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 688, |
| "total": 825, |
| "coverage": 0.833939393939394 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 534, |
| "total": 825, |
| "coverage": 0.6472727272727272 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 825, |
| "hard_count": 825, |
| "accuracy": 0.9745454545454545, |
| "expected_accuracy": 0.9745454545454545, |
| "nll": 0.07717523550004791, |
| "brier": 0.0398977614679421, |
| "expected_brier": 0.0398977614679421, |
| "ece_bins": 15, |
| "multiclass_ece": 0.021055164766154082, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 824, |
| "coverage": 0.9987878787878788, |
| "accuracy": 0.9745145631067961, |
| "expected_accuracy": 0.9745145631067961, |
| "expected_risk": 0.025485436893203928 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 812, |
| "coverage": 0.9842424242424243, |
| "accuracy": 0.9778325123152709, |
| "expected_accuracy": 0.9778325123152709, |
| "expected_risk": 0.022167487684729092 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 794, |
| "coverage": 0.9624242424242424, |
| "accuracy": 0.9911838790931989, |
| "expected_accuracy": 0.9911838790931989, |
| "expected_risk": 0.008816120906801062 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 748, |
| "coverage": 0.9066666666666666, |
| "accuracy": 0.9973262032085561, |
| "expected_accuracy": 0.9973262032085561, |
| "expected_risk": 0.002673796791443861 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 688, |
| "coverage": 0.833939393939394, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 534, |
| "coverage": 0.6472727272727272, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 825, |
| "hard_count": 825, |
| "accuracy": 0.9745454545454545, |
| "expected_accuracy": 0.9745454545454545, |
| "nll": 0.07717523550004791, |
| "brier": 0.0398977614679421, |
| "expected_brier": 0.0398977614679421, |
| "ece_bins": 15, |
| "multiclass_ece": 0.021055164766154082, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 824, |
| "coverage": 0.9987878787878788, |
| "accuracy": 0.9745145631067961, |
| "expected_accuracy": 0.9745145631067961, |
| "expected_risk": 0.025485436893203928 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 812, |
| "coverage": 0.9842424242424243, |
| "accuracy": 0.9778325123152709, |
| "expected_accuracy": 0.9778325123152709, |
| "expected_risk": 0.022167487684729092 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 794, |
| "coverage": 0.9624242424242424, |
| "accuracy": 0.9911838790931989, |
| "expected_accuracy": 0.9911838790931989, |
| "expected_risk": 0.008816120906801062 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 748, |
| "coverage": 0.9066666666666666, |
| "accuracy": 0.9973262032085561, |
| "expected_accuracy": 0.9973262032085561, |
| "expected_risk": 0.002673796791443861 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 688, |
| "coverage": 0.833939393939394, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 534, |
| "coverage": 0.6472727272727272, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "wikispeedia-v1": { |
| "count": 219, |
| "successful": 219, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 57, |
| "accuracy_all_rows_failures_incorrect": 0.6666666666666666, |
| "expected_accuracy_all_rows_failures_zero": 0.3274433836077672, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "total": 219, |
| "coverage": 0.0182648401826484 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "total": 219, |
| "coverage": 0.0 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 219, |
| "hard_count": 57, |
| "accuracy": 0.6666666666666666, |
| "expected_accuracy": 0.3274433836077672, |
| "nll": 2.111795103417709, |
| "brier": 0.29905520762617627, |
| "expected_brier": 0.8359138461971435, |
| "ece_bins": 15, |
| "multiclass_ece": 0.14396741883440964, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 0.0182648401826484, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 219, |
| "hard_count": 57, |
| "accuracy": 0.6666666666666666, |
| "expected_accuracy": 0.3274433836077672, |
| "nll": 2.111795103417709, |
| "brier": 0.29905520762617627, |
| "expected_brier": 0.8359138461971435, |
| "ece_bins": 15, |
| "multiclass_ece": 0.14396741883440964, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 4, |
| "coverage": 0.0182648401826484, |
| "accuracy": 1.0, |
| "expected_accuracy": 0.75, |
| "expected_risk": 0.25 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 0, |
| "coverage": 0.0, |
| "accuracy": null, |
| "expected_accuracy": null, |
| "expected_risk": null |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/agent_trace_observability": { |
| "count": 525, |
| "successful": 525, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 525, |
| "accuracy_all_rows_failures_incorrect": 1.0, |
| "expected_accuracy_all_rows_failures_zero": 1.0, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 525, |
| "total": 525, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 524, |
| "total": 525, |
| "coverage": 0.9980952380952381 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 525, |
| "hard_count": 525, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.000545160068456591, |
| "brier": 4.895507169664063e-06, |
| "expected_brier": 4.8955071696642534e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0005439303275837422, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 524, |
| "coverage": 0.9980952380952381, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 525, |
| "hard_count": 525, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "nll": 0.000545160068456591, |
| "brier": 4.895507169664063e-06, |
| "expected_brier": 4.8955071696642534e-06, |
| "ece_bins": 15, |
| "multiclass_ece": 0.0005439303275837422, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 525, |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 524, |
| "coverage": 0.9980952380952381, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/customer_service": { |
| "count": 600, |
| "successful": 600, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 600, |
| "accuracy_all_rows_failures_incorrect": 0.9783333333333334, |
| "expected_accuracy_all_rows_failures_zero": 0.9783333333333334, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 600, |
| "total": 600, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 597, |
| "total": 600, |
| "coverage": 0.995 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 600, |
| "hard_count": 600, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "nll": 0.16283998438208533, |
| "brier": 0.04328014604238859, |
| "expected_brier": 0.04328014604238859, |
| "ece_bins": 15, |
| "multiclass_ece": 0.021130186926374108, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 597, |
| "coverage": 0.995, |
| "accuracy": 0.9782244556113903, |
| "expected_accuracy": 0.9782244556113903, |
| "expected_risk": 0.021775544388609736 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 600, |
| "hard_count": 600, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "nll": 0.16283998438208533, |
| "brier": 0.04328014604238859, |
| "expected_brier": 0.04328014604238859, |
| "ece_bins": 15, |
| "multiclass_ece": 0.021130186926374108, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 600, |
| "coverage": 1.0, |
| "accuracy": 0.9783333333333334, |
| "expected_accuracy": 0.9783333333333334, |
| "expected_risk": 0.021666666666666612 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 597, |
| "coverage": 0.995, |
| "accuracy": 0.9782244556113903, |
| "expected_accuracy": 0.9782244556113903, |
| "expected_risk": 0.021775544388609736 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/invoice_processing": { |
| "count": 750, |
| "successful": 750, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 750, |
| "accuracy_all_rows_failures_incorrect": 0.9973333333333333, |
| "expected_accuracy_all_rows_failures_zero": 0.9973333333333333, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "total": 750, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 748, |
| "total": 750, |
| "coverage": 0.9973333333333333 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 746, |
| "total": 750, |
| "coverage": 0.9946666666666667 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 746, |
| "total": 750, |
| "coverage": 0.9946666666666667 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 744, |
| "total": 750, |
| "coverage": 0.992 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 743, |
| "total": 750, |
| "coverage": 0.9906666666666667 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "nll": 0.006416542163341697, |
| "brier": 0.003983146029347859, |
| "expected_brier": 0.003983146029347861, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002603556053682448, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "coverage": 1.0, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "expected_risk": 0.002666666666666706 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 748, |
| "coverage": 0.9973333333333333, |
| "accuracy": 0.9986631016042781, |
| "expected_accuracy": 0.9986631016042781, |
| "expected_risk": 0.0013368983957219305 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9986595174262735, |
| "expected_accuracy": 0.9986595174262735, |
| "expected_risk": 0.0013404825737265424 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9986595174262735, |
| "expected_accuracy": 0.9986595174262735, |
| "expected_risk": 0.0013404825737265424 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 744, |
| "coverage": 0.992, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 743, |
| "coverage": 0.9906666666666667, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 750, |
| "hard_count": 750, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "nll": 0.006416542163341697, |
| "brier": 0.003983146029347859, |
| "expected_brier": 0.003983146029347861, |
| "ece_bins": 15, |
| "multiclass_ece": 0.002603556053682448, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 750, |
| "coverage": 1.0, |
| "accuracy": 0.9973333333333333, |
| "expected_accuracy": 0.9973333333333333, |
| "expected_risk": 0.002666666666666706 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 748, |
| "coverage": 0.9973333333333333, |
| "accuracy": 0.9986631016042781, |
| "expected_accuracy": 0.9986631016042781, |
| "expected_risk": 0.0013368983957219305 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9986595174262735, |
| "expected_accuracy": 0.9986595174262735, |
| "expected_risk": 0.0013404825737265424 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 746, |
| "coverage": 0.9946666666666667, |
| "accuracy": 0.9986595174262735, |
| "expected_accuracy": 0.9986595174262735, |
| "expected_risk": 0.0013404825737265424 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 744, |
| "coverage": 0.992, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 743, |
| "coverage": 0.9906666666666667, |
| "accuracy": 1.0, |
| "expected_accuracy": 1.0, |
| "expected_risk": 0.0 |
| } |
| ] |
| } |
| }, |
| "workflow-controls-v1/security_incidents": { |
| "count": 1275, |
| "successful": 1275, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 1275, |
| "accuracy_all_rows_failures_incorrect": 0.9929411764705882, |
| "expected_accuracy_all_rows_failures_zero": 0.9929411764705882, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "total": 1275, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1275, |
| "total": 1275, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1275, |
| "total": 1275, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1266, |
| "total": 1275, |
| "coverage": 0.9929411764705882 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1256, |
| "total": 1275, |
| "coverage": 0.9850980392156863 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1247, |
| "total": 1275, |
| "coverage": 0.9780392156862745 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 1275, |
| "hard_count": 1275, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "nll": 0.02425288839114142, |
| "brier": 0.012286914067882482, |
| "expected_brier": 0.012286914067882486, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004988858941316103, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1266, |
| "coverage": 0.9929411764705882, |
| "accuracy": 0.995260663507109, |
| "expected_accuracy": 0.995260663507109, |
| "expected_risk": 0.004739336492891044 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1256, |
| "coverage": 0.9850980392156863, |
| "accuracy": 0.9976114649681529, |
| "expected_accuracy": 0.9976114649681529, |
| "expected_risk": 0.0023885350318471055 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1247, |
| "coverage": 0.9780392156862745, |
| "accuracy": 0.9983961507618284, |
| "expected_accuracy": 0.9983961507618284, |
| "expected_risk": 0.001603849238171584 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 1275, |
| "hard_count": 1275, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "nll": 0.02425288839114142, |
| "brier": 0.012286914067882482, |
| "expected_brier": 0.012286914067882486, |
| "ece_bins": 15, |
| "multiclass_ece": 0.004988858941316103, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 1275, |
| "coverage": 1.0, |
| "accuracy": 0.9929411764705882, |
| "expected_accuracy": 0.9929411764705882, |
| "expected_risk": 0.007058823529411784 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 1266, |
| "coverage": 0.9929411764705882, |
| "accuracy": 0.995260663507109, |
| "expected_accuracy": 0.995260663507109, |
| "expected_risk": 0.004739336492891044 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1256, |
| "coverage": 0.9850980392156863, |
| "accuracy": 0.9976114649681529, |
| "expected_accuracy": 0.9976114649681529, |
| "expected_risk": 0.0023885350318471055 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1247, |
| "coverage": 0.9780392156862745, |
| "accuracy": 0.9983961507618284, |
| "expected_accuracy": 0.9983961507618284, |
| "expected_risk": 0.001603849238171584 |
| } |
| ] |
| } |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "count": 3438, |
| "successful": 3438, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2993, |
| "accuracy_all_rows_failures_incorrect": 0.863013698630137, |
| "expected_accuracy_all_rows_failures_zero": 0.8021747433614798, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2972, |
| "total": 3438, |
| "coverage": 0.8644560791157649 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2643, |
| "total": 3438, |
| "coverage": 0.768760907504363 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2483, |
| "total": 3438, |
| "coverage": 0.7222222222222222 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2147, |
| "total": 3438, |
| "coverage": 0.6244909831297266 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1778, |
| "total": 3438, |
| "coverage": 0.5171611401977894 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 863, |
| "total": 3438, |
| "coverage": 0.2510180337405468 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 3438, |
| "hard_count": 2993, |
| "accuracy": 0.863013698630137, |
| "expected_accuracy": 0.8021747433614798, |
| "nll": 0.6259869551780559, |
| "brier": 0.20650286979189394, |
| "expected_brier": 0.28195527676698856, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05203723428051684, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2972, |
| "coverage": 0.8644560791157649, |
| "accuracy": 0.8779803646563815, |
| "expected_accuracy": 0.8626626289816062, |
| "expected_risk": 0.13733737101839383 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2643, |
| "coverage": 0.768760907504363, |
| "accuracy": 0.8976109215017065, |
| "expected_accuracy": 0.8966452263841594, |
| "expected_risk": 0.10335477361584056 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2483, |
| "coverage": 0.7222222222222222, |
| "accuracy": 0.9129734085414988, |
| "expected_accuracy": 0.9128070881997583, |
| "expected_risk": 0.08719291180024169 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2147, |
| "coverage": 0.6244909831297266, |
| "accuracy": 0.9263746505125815, |
| "expected_accuracy": 0.9261760596180717, |
| "expected_risk": 0.0738239403819283 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1778, |
| "coverage": 0.5171611401977894, |
| "accuracy": 0.9465391108610017, |
| "expected_accuracy": 0.9462879640044994, |
| "expected_risk": 0.05371203599550056 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 863, |
| "coverage": 0.2510180337405468, |
| "accuracy": 0.9733487833140209, |
| "expected_accuracy": 0.9733487833140209, |
| "expected_risk": 0.02665121668597914 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 3438, |
| "hard_count": 2993, |
| "accuracy": 0.863013698630137, |
| "expected_accuracy": 0.8021747433614798, |
| "nll": 0.6259869551780559, |
| "brier": 0.20650286979189394, |
| "expected_brier": 0.28195527676698856, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05203723428051684, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2972, |
| "coverage": 0.8644560791157649, |
| "accuracy": 0.8779803646563815, |
| "expected_accuracy": 0.8626626289816062, |
| "expected_risk": 0.13733737101839383 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2643, |
| "coverage": 0.768760907504363, |
| "accuracy": 0.8976109215017065, |
| "expected_accuracy": 0.8966452263841594, |
| "expected_risk": 0.10335477361584056 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2483, |
| "coverage": 0.7222222222222222, |
| "accuracy": 0.9129734085414988, |
| "expected_accuracy": 0.9128070881997583, |
| "expected_risk": 0.08719291180024169 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2147, |
| "coverage": 0.6244909831297266, |
| "accuracy": 0.9263746505125815, |
| "expected_accuracy": 0.9261760596180717, |
| "expected_risk": 0.0738239403819283 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 1778, |
| "coverage": 0.5171611401977894, |
| "accuracy": 0.9465391108610017, |
| "expected_accuracy": 0.9462879640044994, |
| "expected_risk": 0.05371203599550056 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 863, |
| "coverage": 0.2510180337405468, |
| "accuracy": 0.9733487833140209, |
| "expected_accuracy": 0.9733487833140209, |
| "expected_risk": 0.02665121668597914 |
| } |
| ] |
| } |
| }, |
| "noul": { |
| "count": 9929, |
| "successful": 9929, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 9929, |
| "accuracy_all_rows_failures_incorrect": 0.9417866854668144, |
| "expected_accuracy_all_rows_failures_zero": 0.9417866854668144, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "total": 9929, |
| "coverage": 1.0 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9661, |
| "total": 9929, |
| "coverage": 0.973008359351395 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9448, |
| "total": 9929, |
| "coverage": 0.9515560479403766 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9151, |
| "total": 9929, |
| "coverage": 0.9216436700574075 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8928, |
| "total": 9929, |
| "coverage": 0.899184207875919 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8067, |
| "total": 9929, |
| "coverage": 0.8124685265384228 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 9929, |
| "hard_count": 9929, |
| "accuracy": 0.9417866854668144, |
| "expected_accuracy": 0.9417866854668144, |
| "nll": 0.1586976512072927, |
| "brier": 0.08958742899221635, |
| "expected_brier": 0.08958742899221635, |
| "ece_bins": 15, |
| "multiclass_ece": 0.03298841702250886, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "coverage": 1.0, |
| "accuracy": 0.9417866854668144, |
| "expected_accuracy": 0.9417866854668144, |
| "expected_risk": 0.05821331453318557 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9661, |
| "coverage": 0.973008359351395, |
| "accuracy": 0.9533174619604596, |
| "expected_accuracy": 0.9533174619604596, |
| "expected_risk": 0.04668253803954037 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9448, |
| "coverage": 0.9515560479403766, |
| "accuracy": 0.9613674851820491, |
| "expected_accuracy": 0.9613674851820491, |
| "expected_risk": 0.038632514817950936 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9151, |
| "coverage": 0.9216436700574075, |
| "accuracy": 0.973117692055513, |
| "expected_accuracy": 0.973117692055513, |
| "expected_risk": 0.026882307944486983 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8928, |
| "coverage": 0.899184207875919, |
| "accuracy": 0.9816308243727598, |
| "expected_accuracy": 0.9816308243727598, |
| "expected_risk": 0.018369175627240164 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8067, |
| "coverage": 0.8124685265384228, |
| "accuracy": 0.9935539853725053, |
| "expected_accuracy": 0.9935539853725053, |
| "expected_risk": 0.006446014627494723 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 9929, |
| "hard_count": 9929, |
| "accuracy": 0.9417866854668144, |
| "expected_accuracy": 0.9417866854668144, |
| "nll": 0.1586976512072927, |
| "brier": 0.08958742899221635, |
| "expected_brier": 0.08958742899221635, |
| "ece_bins": 15, |
| "multiclass_ece": 0.03298841702250886, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 9929, |
| "coverage": 1.0, |
| "accuracy": 0.9417866854668144, |
| "expected_accuracy": 0.9417866854668144, |
| "expected_risk": 0.05821331453318557 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 9661, |
| "coverage": 0.973008359351395, |
| "accuracy": 0.9533174619604596, |
| "expected_accuracy": 0.9533174619604596, |
| "expected_risk": 0.04668253803954037 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 9448, |
| "coverage": 0.9515560479403766, |
| "accuracy": 0.9613674851820491, |
| "expected_accuracy": 0.9613674851820491, |
| "expected_risk": 0.038632514817950936 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 9151, |
| "coverage": 0.9216436700574075, |
| "accuracy": 0.973117692055513, |
| "expected_accuracy": 0.973117692055513, |
| "expected_risk": 0.026882307944486983 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 8928, |
| "coverage": 0.899184207875919, |
| "accuracy": 0.9816308243727598, |
| "expected_accuracy": 0.9816308243727598, |
| "expected_risk": 0.018369175627240164 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 8067, |
| "coverage": 0.8124685265384228, |
| "accuracy": 0.9935539853725053, |
| "expected_accuracy": 0.9935539853725053, |
| "expected_risk": 0.006446014627494723 |
| } |
| ] |
| } |
| }, |
| "score": { |
| "count": 2553, |
| "successful": 2553, |
| "failed": 0, |
| "inference_coverage": 1.0, |
| "hard_count": 2524, |
| "accuracy_all_rows_failures_incorrect": 0.8997622820919176, |
| "expected_accuracy_all_rows_failures_zero": 0.8944379161770466, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2448, |
| "total": 2553, |
| "coverage": 0.9588719153936546 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2328, |
| "total": 2553, |
| "coverage": 0.9118683901292597 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2252, |
| "total": 2553, |
| "coverage": 0.8820994907951429 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2144, |
| "total": 2553, |
| "coverage": 0.8397963180571876 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2045, |
| "total": 2553, |
| "coverage": 0.8010184097140619 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1741, |
| "total": 2553, |
| "coverage": 0.681942812377595 |
| } |
| ], |
| "probability_metrics_all_rows": { |
| "count": 2553, |
| "hard_count": 2524, |
| "accuracy": 0.8997622820919176, |
| "expected_accuracy": 0.8944379161770466, |
| "nll": 0.40706831123282244, |
| "brier": 0.16323629063231726, |
| "expected_brier": 0.16969927535617155, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05413140129609939, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2448, |
| "coverage": 0.9588719153936546, |
| "accuracy": 0.9197860962566845, |
| "expected_accuracy": 0.9168709150326797, |
| "expected_risk": 0.0831290849673203 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2328, |
| "coverage": 0.9118683901292597, |
| "accuracy": 0.9286941580756014, |
| "expected_accuracy": 0.9286941580756014, |
| "expected_risk": 0.0713058419243986 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2252, |
| "coverage": 0.8820994907951429, |
| "accuracy": 0.933392539964476, |
| "expected_accuracy": 0.933392539964476, |
| "expected_risk": 0.06660746003552398 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2144, |
| "coverage": 0.8397963180571876, |
| "accuracy": 0.9421641791044776, |
| "expected_accuracy": 0.9421641791044776, |
| "expected_risk": 0.057835820895522416 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2045, |
| "coverage": 0.8010184097140619, |
| "accuracy": 0.9486552567237164, |
| "expected_accuracy": 0.9486552567237164, |
| "expected_risk": 0.05134474327628358 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1741, |
| "coverage": 0.681942812377595, |
| "accuracy": 0.9620907524411257, |
| "expected_accuracy": 0.9620907524411257, |
| "expected_risk": 0.03790924755887426 |
| } |
| ] |
| }, |
| "probability_metrics_successful_rows_only": { |
| "count": 2553, |
| "hard_count": 2524, |
| "accuracy": 0.8997622820919176, |
| "expected_accuracy": 0.8944379161770466, |
| "nll": 0.40706831123282244, |
| "brier": 0.16323629063231726, |
| "expected_brier": 0.16969927535617155, |
| "ece_bins": 15, |
| "multiclass_ece": 0.05413140129609939, |
| "coverage": [ |
| { |
| "threshold": 0.5, |
| "selected": 2448, |
| "coverage": 0.9588719153936546, |
| "accuracy": 0.9197860962566845, |
| "expected_accuracy": 0.9168709150326797, |
| "expected_risk": 0.0831290849673203 |
| }, |
| { |
| "threshold": 0.7, |
| "selected": 2328, |
| "coverage": 0.9118683901292597, |
| "accuracy": 0.9286941580756014, |
| "expected_accuracy": 0.9286941580756014, |
| "expected_risk": 0.0713058419243986 |
| }, |
| { |
| "threshold": 0.8, |
| "selected": 2252, |
| "coverage": 0.8820994907951429, |
| "accuracy": 0.933392539964476, |
| "expected_accuracy": 0.933392539964476, |
| "expected_risk": 0.06660746003552398 |
| }, |
| { |
| "threshold": 0.9, |
| "selected": 2144, |
| "coverage": 0.8397963180571876, |
| "accuracy": 0.9421641791044776, |
| "expected_accuracy": 0.9421641791044776, |
| "expected_risk": 0.057835820895522416 |
| }, |
| { |
| "threshold": 0.95, |
| "selected": 2045, |
| "coverage": 0.8010184097140619, |
| "accuracy": 0.9486552567237164, |
| "expected_accuracy": 0.9486552567237164, |
| "expected_risk": 0.05134474327628358 |
| }, |
| { |
| "threshold": 0.99, |
| "selected": 1741, |
| "coverage": 0.681942812377595, |
| "accuracy": 0.9620907524411257, |
| "expected_accuracy": 0.9620907524411257, |
| "expected_risk": 0.03790924755887426 |
| } |
| ] |
| } |
| } |
| } |
| } |
| }, |
| "full_data_baseline_evaluated": false, |
| "full_data_training_gain": null, |
| "limitations": [ |
| "No full-data baseline exists in this run. Earlier 512-row-per-split baseline results cannot establish full-data training gain.", |
| "Per-group metrics were recomputed and compared for every group; compact report retains source/kind metrics and group counts/hashes. Original complete summary remains in ignored evidence.", |
| "Snapshot/checkpoint hashes bind this evaluation to the packaged inference files; the trainer's historical reload lacked a contemporaneous weight digest.", |
| "Synthetic held-out decision accuracy is not end-to-end game, browser or flight performance. Four-replica wall time is not single-GPU latency." |
| ], |
| "schema_version": 1, |
| "status": "passed", |
| "source_audit_sha256": "d490dd1636acebaf393fe181dfd0f65e66eb54edb7247b4abffd74a3d0891100", |
| "source_audit_path": "reports/full-data-eval-n1-v1/9b-audit.json", |
| "scope": "Existing completed full release-v2 held-out evaluation, bound to the unchanged five packaged inference files. No new model inference was run for this Hugging Face upload." |
| } |
|
|
