{ "model": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "checkpoint": { "model": "Qwen/Qwen3.5-9B", "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "checkpoint_sha256": "a691105dd5751bfcf72d276a6ce075d45054db77ae3272e87b5f07b2bd7e2193", "temperature": 1.8969118766347646, "calibration": { "temperature": 1.8969118766347646, "split": "calibration", "n": 512, "ids_sha256": "6ff7db59c1ae2485179b80316b3cb4f0161df77704692dcab48871a295ab94f1" }, "max_length": 4096, "training_completion": { "steps": 20204, "trained_rows_consumed": 80816, "checkpoint_reload_max_error": 0.0, "run_sha256": "e5a119cc2ec1f78615375824352f71a68bcc78eb30d28e14bcdbcd442075b42b", "summary_sha256": "789ae2a85fc3dec77388b11c039f56e3110052b77c55e1eb371d5d5284e72ebc" } }, "package_bound_inference_files": { "checkpoint/adapter/adapter_config.json": { "sha256": "3b2d3b4c9e95a7ca0dfb6132a8a76af281363032eeffbbd08ba34fc76cb3b9be", "bytes": 1170 }, "checkpoint/adapter/adapter_model.safetensors": { "sha256": "f85650a8fb97c6d0ac3e948cdca2f30a0ca6ace8b8a43aebca1c152fe38aeb75", "bytes": 23613816 }, "checkpoint/head.pt": { "sha256": "229fe9800384e824135e59d1af59bda7346da031aea61640be5f456b9be6ce70", "bytes": 18189 }, "checkpoint/model.json": { "sha256": "a3a8a691a88b2af64ecb6952a7dd006977fd8c544df09afcbe1e4303b91a8daa", "bytes": 186 }, "checkpoint/temperature.json": { "sha256": "8cfe40a9f42cc2d0607a3381f2a4e0374cb3842506ac7263f7fff35facfca034", "bytes": 162 } }, "data_identity": { "manifest_sha256": "56105dc9fc89ef74919f5beb60bb6ae8c6e17bb95699dab59205f67d8b338d97", "split_sha256": { "train": "80a9ec19da9c4065b72f7a9e5ff8d67e62c1d1dd2572dfd296e46bfe2ea5a182", "calibration": "dd21b26912a6b26a09c3b00ae66a7985ee760e737b94ada4fcc6320406d4db63", "validation": "1dc97079d4fc034a1ba16315f801b0a4715b30c2a3874ae05aeb60dd1e40e8bb", "test": "fa8d62775b96c1514f238c2d8a8f8b66515dada67d490978376545296593e29b", "ood": "9152bff7f3d9c83cce3f1423cc6e0e6b33286b3092b3f544cb18dcc37b35a7ee" }, "counts": { "test": 10532, "ood": 15920 }, "ordered_ids_sha256": { "test": "98e1af223471c08d84e44c8864e51cefa1bc671687c00bfbf64b7e5a22e8bf52", "ood": "52fa6cc83c0a172516609257b6048aaece9a3fef75f3f5f694bc6213e5c30a42" } }, "coverage": { "expected_rows": 26452, "unique_ids": 26452, "shard_rows": [ 6613, 6613, 6613, 6613 ], "missing": 0, "duplicate": 0, "failed": 0, "merged_original_order_verified": true }, "metrics": { "test": { "count": 10532, "successful": 10532, "failed": 0, "inference_coverage": 1.0, "hard_count": 10046, "accuracy_all_rows_failures_incorrect": 0.9754130997411905, "expected_accuracy_all_rows_failures_zero": 0.9471531754526437, "coverage": [ { "threshold": 0.5, "selected": 9945, "total": 10532, "coverage": 0.9442650968477022 }, { "threshold": 0.7, "selected": 9636, "total": 10532, "coverage": 0.9149259399924041 }, { "threshold": 0.8, "selected": 9552, "total": 10532, "coverage": 0.906950246866692 }, { "threshold": 0.9, "selected": 9409, "total": 10532, "coverage": 0.893372578807444 }, { "threshold": 0.95, "selected": 9208, "total": 10532, "coverage": 0.8742878845423472 }, { "threshold": 0.99, "selected": 8674, "total": 10532, "coverage": 0.823585263957463 } ], "probability_metrics_all_rows": { "count": 10532, "hard_count": 10046, "accuracy": 0.9754130997411905, "expected_accuracy": 0.9471531754526437, "nll": 0.1309471525831089, "brier": 0.03911290032779779, "expected_brier": 0.0671638646396812, "ece_bins": 15, "multiclass_ece": 0.007706544094318023, "coverage": [ { "threshold": 0.5, "selected": 9945, "coverage": 0.9442650968477022, "accuracy": 0.9885468861846815, "expected_accuracy": 0.9797553209317914, "expected_risk": 0.020244679068208593 }, { "threshold": 0.7, "selected": 9636, "coverage": 0.9149259399924041, "accuracy": 0.9936570656129771, "expected_accuracy": 0.9926490936764909, "expected_risk": 0.007350906323509099 }, { "threshold": 0.8, "selected": 9552, "coverage": 0.906950246866692, "accuracy": 0.9952859836580766, "expected_accuracy": 0.9949748743718593, "expected_risk": 0.005025125628140725 }, { "threshold": 0.9, "selected": 9409, "coverage": 0.893372578807444, "accuracy": 0.9977680943777234, "expected_accuracy": 0.9977680943777234, "expected_risk": 0.0022319056222765976 }, { "threshold": 0.95, "selected": 9208, "coverage": 0.8742878845423472, "accuracy": 0.9993483927019983, "expected_accuracy": 0.9993483927019983, "expected_risk": 0.0006516072980017329 }, { "threshold": 0.99, "selected": 8674, "coverage": 0.823585263957463, "accuracy": 0.9997694258704174, "expected_accuracy": 0.9997694258704174, "expected_risk": 0.0002305741295826058 } ] }, "probability_metrics_successful_rows_only": { "count": 10532, "hard_count": 10046, "accuracy": 0.9754130997411905, "expected_accuracy": 0.9471531754526437, "nll": 0.1309471525831089, "brier": 0.03911290032779779, "expected_brier": 0.0671638646396812, "ece_bins": 15, "multiclass_ece": 0.007706544094318023, "coverage": [ { "threshold": 0.5, "selected": 9945, "coverage": 0.9442650968477022, "accuracy": 0.9885468861846815, "expected_accuracy": 0.9797553209317914, "expected_risk": 0.020244679068208593 }, { "threshold": 0.7, "selected": 9636, "coverage": 0.9149259399924041, "accuracy": 0.9936570656129771, "expected_accuracy": 0.9926490936764909, "expected_risk": 0.007350906323509099 }, { "threshold": 0.8, "selected": 9552, "coverage": 0.906950246866692, "accuracy": 0.9952859836580766, "expected_accuracy": 0.9949748743718593, "expected_risk": 0.005025125628140725 }, { "threshold": 0.9, "selected": 9409, "coverage": 0.893372578807444, "accuracy": 0.9977680943777234, "expected_accuracy": 0.9977680943777234, "expected_risk": 0.0022319056222765976 }, { "threshold": 0.95, "selected": 9208, "coverage": 0.8742878845423472, "accuracy": 0.9993483927019983, "expected_accuracy": 0.9993483927019983, "expected_risk": 0.0006516072980017329 }, { "threshold": 0.99, "selected": 8674, "coverage": 0.823585263957463, "accuracy": 0.9997694258704174, "expected_accuracy": 0.9997694258704174, "expected_risk": 0.0002305741295826058 } ] }, "by_source": { "customer-control-v1": { "count": 606, "successful": 606, "failed": 0, "inference_coverage": 1.0, "hard_count": 574, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 0.9722222222222222, "coverage": [ { "threshold": 0.5, "selected": 596, "total": 606, "coverage": 0.9834983498349835 }, { "threshold": 0.7, "selected": 574, "total": 606, "coverage": 0.9471947194719472 }, { "threshold": 0.8, "selected": 574, "total": 606, "coverage": 0.9471947194719472 }, { "threshold": 0.9, "selected": 574, "total": 606, "coverage": 0.9471947194719472 }, { "threshold": 0.95, "selected": 574, "total": 606, "coverage": 0.9471947194719472 }, { "threshold": 0.99, "selected": 564, "total": 606, "coverage": 0.9306930693069307 } ], "probability_metrics_all_rows": { "count": 606, "hard_count": 574, "accuracy": 1.0, "expected_accuracy": 0.9722222222222222, "nll": 0.04199953432734906, "brier": 6.843414604449945e-05, "expected_brier": 0.027846211923822282, "ece_bins": 15, "multiclass_ece": 0.0019262574800517217, "coverage": [ { "threshold": 0.5, "selected": 596, "coverage": 0.9834983498349835, "accuracy": 1.0, "expected_accuracy": 0.9815436241610739, "expected_risk": 0.01845637583892612 }, { "threshold": 0.7, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 564, "coverage": 0.9306930693069307, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 606, "hard_count": 574, "accuracy": 1.0, "expected_accuracy": 0.9722222222222222, "nll": 0.04199953432734906, "brier": 6.843414604449945e-05, "expected_brier": 0.027846211923822282, "ece_bins": 15, "multiclass_ece": 0.0019262574800517217, "coverage": [ { "threshold": 0.5, "selected": 596, "coverage": 0.9834983498349835, "accuracy": 1.0, "expected_accuracy": 0.9815436241610739, "expected_risk": 0.01845637583892612 }, { "threshold": 0.7, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 574, "coverage": 0.9471947194719472, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 564, "coverage": 0.9306930693069307, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "painting-geometry-v1": { "count": 3072, "successful": 3072, "failed": 0, "inference_coverage": 1.0, "hard_count": 3072, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 1.0, "coverage": [ { "threshold": 0.5, "selected": 3072, "total": 3072, "coverage": 1.0 }, { "threshold": 0.7, "selected": 3071, "total": 3072, "coverage": 0.9996744791666666 }, { "threshold": 0.8, "selected": 3071, "total": 3072, "coverage": 0.9996744791666666 }, { "threshold": 0.9, "selected": 3071, "total": 3072, "coverage": 0.9996744791666666 }, { "threshold": 0.95, "selected": 3068, "total": 3072, "coverage": 0.9986979166666666 }, { "threshold": 0.99, "selected": 3031, "total": 3072, "coverage": 0.9866536458333334 } ], "probability_metrics_all_rows": { "count": 3072, "hard_count": 3072, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0013689140515597267, "brier": 7.957005332993052e-05, "expected_brier": 7.957005332993308e-05, "ece_bins": 15, "multiclass_ece": 0.0013415313830535288, "coverage": [ { "threshold": 0.5, "selected": 3072, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3068, "coverage": 0.9986979166666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3031, "coverage": 0.9866536458333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 3072, "hard_count": 3072, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0013689140515597267, "brier": 7.957005332993052e-05, "expected_brier": 7.957005332993308e-05, "ece_bins": 15, "multiclass_ece": 0.0013415313830535288, "coverage": [ { "threshold": 0.5, "selected": 3072, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 3071, "coverage": 0.9996744791666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3068, "coverage": 0.9986979166666666, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 3031, "coverage": 0.9866536458333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "reasoning-control-v1": { "count": 681, "successful": 681, "failed": 0, "inference_coverage": 1.0, "hard_count": 681, "accuracy_all_rows_failures_incorrect": 0.8722466960352423, "expected_accuracy_all_rows_failures_zero": 0.8722466960352423, "coverage": [ { "threshold": 0.5, "selected": 627, "total": 681, "coverage": 0.920704845814978 }, { "threshold": 0.7, "selected": 576, "total": 681, "coverage": 0.8458149779735683 }, { "threshold": 0.8, "selected": 554, "total": 681, "coverage": 0.8135095447870778 }, { "threshold": 0.9, "selected": 512, "total": 681, "coverage": 0.7518355359765051 }, { "threshold": 0.95, "selected": 489, "total": 681, "coverage": 0.7180616740088106 }, { "threshold": 0.99, "selected": 419, "total": 681, "coverage": 0.6152716593245228 } ], "probability_metrics_all_rows": { "count": 681, "hard_count": 681, "accuracy": 0.8722466960352423, "expected_accuracy": 0.8722466960352423, "nll": 0.30288131001112933, "brier": 0.1731250583140691, "expected_brier": 0.1731250583140691, "ece_bins": 15, "multiclass_ece": 0.0369016945577258, "coverage": [ { "threshold": 0.5, "selected": 627, "coverage": 0.920704845814978, "accuracy": 0.9138755980861244, "expected_accuracy": 0.9138755980861244, "expected_risk": 0.0861244019138756 }, { "threshold": 0.7, "selected": 576, "coverage": 0.8458149779735683, "accuracy": 0.9461805555555556, "expected_accuracy": 0.9461805555555556, "expected_risk": 0.05381944444444442 }, { "threshold": 0.8, "selected": 554, "coverage": 0.8135095447870778, "accuracy": 0.9620938628158845, "expected_accuracy": 0.9620938628158845, "expected_risk": 0.03790613718411551 }, { "threshold": 0.9, "selected": 512, "coverage": 0.7518355359765051, "accuracy": 0.986328125, "expected_accuracy": 0.986328125, "expected_risk": 0.013671875 }, { "threshold": 0.95, "selected": 489, "coverage": 0.7180616740088106, "accuracy": 0.9959100204498977, "expected_accuracy": 0.9959100204498977, "expected_risk": 0.004089979550102263 }, { "threshold": 0.99, "selected": 419, "coverage": 0.6152716593245228, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 681, "hard_count": 681, "accuracy": 0.8722466960352423, "expected_accuracy": 0.8722466960352423, "nll": 0.30288131001112933, "brier": 0.1731250583140691, "expected_brier": 0.1731250583140691, "ece_bins": 15, "multiclass_ece": 0.0369016945577258, "coverage": [ { "threshold": 0.5, "selected": 627, "coverage": 0.920704845814978, "accuracy": 0.9138755980861244, "expected_accuracy": 0.9138755980861244, "expected_risk": 0.0861244019138756 }, { "threshold": 0.7, "selected": 576, "coverage": 0.8458149779735683, "accuracy": 0.9461805555555556, "expected_accuracy": 0.9461805555555556, "expected_risk": 0.05381944444444442 }, { "threshold": 0.8, "selected": 554, "coverage": 0.8135095447870778, "accuracy": 0.9620938628158845, "expected_accuracy": 0.9620938628158845, "expected_risk": 0.03790613718411551 }, { "threshold": 0.9, "selected": 512, "coverage": 0.7518355359765051, "accuracy": 0.986328125, "expected_accuracy": 0.986328125, "expected_risk": 0.013671875 }, { "threshold": 0.95, "selected": 489, "coverage": 0.7180616740088106, "accuracy": 0.9959100204498977, "expected_accuracy": 0.9959100204498977, "expected_risk": 0.004089979550102263 }, { "threshold": 0.99, "selected": 419, "coverage": 0.6152716593245228, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "snake-v1": { "count": 1392, "successful": 1392, "failed": 0, "inference_coverage": 1.0, "hard_count": 1279, "accuracy_all_rows_failures_incorrect": 0.9976544175136826, "expected_accuracy_all_rows_failures_zero": 0.9567768199233716, "coverage": [ { "threshold": 0.5, "selected": 1340, "total": 1392, "coverage": 0.9626436781609196 }, { "threshold": 0.7, "selected": 1268, "total": 1392, "coverage": 0.9109195402298851 }, { "threshold": 0.8, "selected": 1254, "total": 1392, "coverage": 0.9008620689655172 }, { "threshold": 0.9, "selected": 1201, "total": 1392, "coverage": 0.8627873563218391 }, { "threshold": 0.95, "selected": 1152, "total": 1392, "coverage": 0.8275862068965517 }, { "threshold": 0.99, "selected": 1076, "total": 1392, "coverage": 0.7729885057471264 } ], "probability_metrics_all_rows": { "count": 1392, "hard_count": 1279, "accuracy": 0.9976544175136826, "expected_accuracy": 0.9567768199233716, "nll": 0.08536951007564594, "brier": 0.01017585055261676, "expected_brier": 0.05124385821545201, "ece_bins": 15, "multiclass_ece": 0.014056450259188788, "coverage": [ { "threshold": 0.5, "selected": 1340, "coverage": 0.9626436781609196, "accuracy": 0.9976525821596244, "expected_accuracy": 0.9743781094527364, "expected_risk": 0.025621890547263604 }, { "threshold": 0.7, "selected": 1268, "coverage": 0.9109195402298851, "accuracy": 0.9984164687252574, "expected_accuracy": 0.9963196635120924, "expected_risk": 0.003680336487907554 }, { "threshold": 0.8, "selected": 1254, "coverage": 0.9008620689655172, "accuracy": 0.9984012789768185, "expected_accuracy": 0.9972089314194578, "expected_risk": 0.002791068580542233 }, { "threshold": 0.9, "selected": 1201, "coverage": 0.8627873563218391, "accuracy": 0.9991673605328892, "expected_accuracy": 0.9991673605328892, "expected_risk": 0.0008326394671107629 }, { "threshold": 0.95, "selected": 1152, "coverage": 0.8275862068965517, "accuracy": 0.9991319444444444, "expected_accuracy": 0.9991319444444444, "expected_risk": 0.0008680555555555802 }, { "threshold": 0.99, "selected": 1076, "coverage": 0.7729885057471264, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 1392, "hard_count": 1279, "accuracy": 0.9976544175136826, "expected_accuracy": 0.9567768199233716, "nll": 0.08536951007564594, "brier": 0.01017585055261676, "expected_brier": 0.05124385821545201, "ece_bins": 15, "multiclass_ece": 0.014056450259188788, "coverage": [ { "threshold": 0.5, "selected": 1340, "coverage": 0.9626436781609196, "accuracy": 0.9976525821596244, "expected_accuracy": 0.9743781094527364, "expected_risk": 0.025621890547263604 }, { "threshold": 0.7, "selected": 1268, "coverage": 0.9109195402298851, "accuracy": 0.9984164687252574, "expected_accuracy": 0.9963196635120924, "expected_risk": 0.003680336487907554 }, { "threshold": 0.8, "selected": 1254, "coverage": 0.9008620689655172, "accuracy": 0.9984012789768185, "expected_accuracy": 0.9972089314194578, "expected_risk": 0.002791068580542233 }, { "threshold": 0.9, "selected": 1201, "coverage": 0.8627873563218391, "accuracy": 0.9991673605328892, "expected_accuracy": 0.9991673605328892, "expected_risk": 0.0008326394671107629 }, { "threshold": 0.95, "selected": 1152, "coverage": 0.8275862068965517, "accuracy": 0.9991319444444444, "expected_accuracy": 0.9991319444444444, "expected_risk": 0.0008680555555555802 }, { "threshold": 0.99, "selected": 1076, "coverage": 0.7729885057471264, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "tic_tac_toe-v1": { "count": 472, "successful": 472, "failed": 0, "inference_coverage": 1.0, "hard_count": 256, "accuracy_all_rows_failures_incorrect": 0.64453125, "expected_accuracy_all_rows_failures_zero": 0.5206315577078289, "coverage": [ { "threshold": 0.5, "selected": 174, "total": 472, "coverage": 0.3686440677966102 }, { "threshold": 0.7, "selected": 34, "total": 472, "coverage": 0.07203389830508475 }, { "threshold": 0.8, "selected": 7, "total": 472, "coverage": 0.014830508474576272 }, { "threshold": 0.9, "selected": 0, "total": 472, "coverage": 0.0 }, { "threshold": 0.95, "selected": 0, "total": 472, "coverage": 0.0 }, { "threshold": 0.99, "selected": 0, "total": 472, "coverage": 0.0 } ], "probability_metrics_all_rows": { "count": 472, "hard_count": 256, "accuracy": 0.64453125, "expected_accuracy": 0.5206315577078289, "nll": 1.0984003514512244, "brier": 0.3480926885371523, "expected_brier": 0.6180684754620918, "ece_bins": 15, "multiclass_ece": 0.06619460047533682, "coverage": [ { "threshold": 0.5, "selected": 174, "coverage": 0.3686440677966102, "accuracy": 0.8152173913043478, "expected_accuracy": 0.632183908045977, "expected_risk": 0.367816091954023 }, { "threshold": 0.7, "selected": 34, "coverage": 0.07203389830508475, "accuracy": 1.0, "expected_accuracy": 0.7892156862745098, "expected_risk": 0.21078431372549022 }, { "threshold": 0.8, "selected": 7, "coverage": 0.014830508474576272, "accuracy": 1.0, "expected_accuracy": 0.7857142857142857, "expected_risk": 0.2142857142857143 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "probability_metrics_successful_rows_only": { "count": 472, "hard_count": 256, "accuracy": 0.64453125, "expected_accuracy": 0.5206315577078289, "nll": 1.0984003514512244, "brier": 0.3480926885371523, "expected_brier": 0.6180684754620918, "ece_bins": 15, "multiclass_ece": 0.06619460047533682, "coverage": [ { "threshold": 0.5, "selected": 174, "coverage": 0.3686440677966102, "accuracy": 0.8152173913043478, "expected_accuracy": 0.632183908045977, "expected_risk": 0.367816091954023 }, { "threshold": 0.7, "selected": 34, "coverage": 0.07203389830508475, "accuracy": 1.0, "expected_accuracy": 0.7892156862745098, "expected_risk": 0.21078431372549022 }, { "threshold": 0.8, "selected": 7, "coverage": 0.014830508474576272, "accuracy": 1.0, "expected_accuracy": 0.7857142857142857, "expected_risk": 0.2142857142857143 }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "tile_platformer-v1": { "count": 202, "successful": 202, "failed": 0, "inference_coverage": 1.0, "hard_count": 202, "accuracy_all_rows_failures_incorrect": 0.9752475247524752, "expected_accuracy_all_rows_failures_zero": 0.9752475247524752, "coverage": [ { "threshold": 0.5, "selected": 202, "total": 202, "coverage": 1.0 }, { "threshold": 0.7, "selected": 202, "total": 202, "coverage": 1.0 }, { "threshold": 0.8, "selected": 202, "total": 202, "coverage": 1.0 }, { "threshold": 0.9, "selected": 202, "total": 202, "coverage": 1.0 }, { "threshold": 0.95, "selected": 164, "total": 202, "coverage": 0.8118811881188119 }, { "threshold": 0.99, "selected": 0, "total": 202, "coverage": 0.0 } ], "probability_metrics_all_rows": { "count": 202, "hard_count": 202, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "nll": 0.13968696905117425, "brier": 0.047840610560813734, "expected_brier": 0.047840610560813734, "ece_bins": 15, "multiclass_ece": 0.035747153706735625, "coverage": [ { "threshold": 0.5, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.7, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.8, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.9, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.95, "selected": 164, "coverage": 0.8118811881188119, "accuracy": 0.9939024390243902, "expected_accuracy": 0.9939024390243902, "expected_risk": 0.0060975609756097615 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "probability_metrics_successful_rows_only": { "count": 202, "hard_count": 202, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "nll": 0.13968696905117425, "brier": 0.047840610560813734, "expected_brier": 0.047840610560813734, "ece_bins": 15, "multiclass_ece": 0.035747153706735625, "coverage": [ { "threshold": 0.5, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.7, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.8, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.9, "selected": 202, "coverage": 1.0, "accuracy": 0.9752475247524752, "expected_accuracy": 0.9752475247524752, "expected_risk": 0.024752475247524774 }, { "threshold": 0.95, "selected": 164, "coverage": 0.8118811881188119, "accuracy": 0.9939024390243902, "expected_accuracy": 0.9939024390243902, "expected_risk": 0.0060975609756097615 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "trex_runner-v1": { "count": 4, "successful": 4, "failed": 0, "inference_coverage": 1.0, "hard_count": 4, "accuracy_all_rows_failures_incorrect": 0.75, "expected_accuracy_all_rows_failures_zero": 0.75, "coverage": [ { "threshold": 0.5, "selected": 4, "total": 4, "coverage": 1.0 }, { "threshold": 0.7, "selected": 4, "total": 4, "coverage": 1.0 }, { "threshold": 0.8, "selected": 4, "total": 4, "coverage": 1.0 }, { "threshold": 0.9, "selected": 3, "total": 4, "coverage": 0.75 }, { "threshold": 0.95, "selected": 3, "total": 4, "coverage": 0.75 }, { "threshold": 0.99, "selected": 0, "total": 4, "coverage": 0.0 } ], "probability_metrics_all_rows": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.7668877262100418, "brier": 0.4059017097317489, "expected_brier": 0.40590170973174894, "ece_bins": 15, "multiclass_ece": 0.2272462205045263, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "probability_metrics_successful_rows_only": { "count": 4, "hard_count": 4, "accuracy": 0.75, "expected_accuracy": 0.75, "nll": 0.7668877262100418, "brier": 0.4059017097317489, "expected_brier": 0.40590170973174894, "ece_bins": 15, "multiclass_ece": 0.2272462205045263, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.8, "selected": 4, "coverage": 1.0, "accuracy": 0.75, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.9, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 3, "coverage": 0.75, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "vizdoom-basic-v1": { "count": 858, "successful": 858, "failed": 0, "inference_coverage": 1.0, "hard_count": 858, "accuracy_all_rows_failures_incorrect": 0.9766899766899767, "expected_accuracy_all_rows_failures_zero": 0.9766899766899767, "coverage": [ { "threshold": 0.5, "selected": 858, "total": 858, "coverage": 1.0 }, { "threshold": 0.7, "selected": 842, "total": 858, "coverage": 0.9813519813519813 }, { "threshold": 0.8, "selected": 823, "total": 858, "coverage": 0.9592074592074592 }, { "threshold": 0.9, "selected": 791, "total": 858, "coverage": 0.921911421911422 }, { "threshold": 0.95, "selected": 718, "total": 858, "coverage": 0.8368298368298368 }, { "threshold": 0.99, "selected": 564, "total": 858, "coverage": 0.6573426573426573 } ], "probability_metrics_all_rows": { "count": 858, "hard_count": 858, "accuracy": 0.9766899766899767, "expected_accuracy": 0.9766899766899767, "nll": 0.06761322089895994, "brier": 0.034824071109677196, "expected_brier": 0.034824071109677196, "ece_bins": 15, "multiclass_ece": 0.012718603010498264, "coverage": [ { "threshold": 0.5, "selected": 858, "coverage": 1.0, "accuracy": 0.9766899766899767, "expected_accuracy": 0.9766899766899767, "expected_risk": 0.023310023310023298 }, { "threshold": 0.7, "selected": 842, "coverage": 0.9813519813519813, "accuracy": 0.9857482185273159, "expected_accuracy": 0.9857482185273159, "expected_risk": 0.014251781472684133 }, { "threshold": 0.8, "selected": 823, "coverage": 0.9592074592074592, "accuracy": 0.9902794653705954, "expected_accuracy": 0.9902794653705954, "expected_risk": 0.009720534629404587 }, { "threshold": 0.9, "selected": 791, "coverage": 0.921911421911422, "accuracy": 0.9974715549936789, "expected_accuracy": 0.9974715549936789, "expected_risk": 0.0025284450063211006 }, { "threshold": 0.95, "selected": 718, "coverage": 0.8368298368298368, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 564, "coverage": 0.6573426573426573, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 858, "hard_count": 858, "accuracy": 0.9766899766899767, "expected_accuracy": 0.9766899766899767, "nll": 0.06761322089895994, "brier": 0.034824071109677196, "expected_brier": 0.034824071109677196, "ece_bins": 15, "multiclass_ece": 0.012718603010498264, "coverage": [ { "threshold": 0.5, "selected": 858, "coverage": 1.0, "accuracy": 0.9766899766899767, "expected_accuracy": 0.9766899766899767, "expected_risk": 0.023310023310023298 }, { "threshold": 0.7, "selected": 842, "coverage": 0.9813519813519813, "accuracy": 0.9857482185273159, "expected_accuracy": 0.9857482185273159, "expected_risk": 0.014251781472684133 }, { "threshold": 0.8, "selected": 823, "coverage": 0.9592074592074592, "accuracy": 0.9902794653705954, "expected_accuracy": 0.9902794653705954, "expected_risk": 0.009720534629404587 }, { "threshold": 0.9, "selected": 791, "coverage": 0.921911421911422, "accuracy": 0.9974715549936789, "expected_accuracy": 0.9974715549936789, "expected_risk": 0.0025284450063211006 }, { "threshold": 0.95, "selected": 718, "coverage": 0.8368298368298368, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 564, "coverage": 0.6573426573426573, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "wikispeedia-v1": { "count": 176, "successful": 176, "failed": 0, "inference_coverage": 1.0, "hard_count": 51, "accuracy_all_rows_failures_incorrect": 0.45098039215686275, "expected_accuracy_all_rows_failures_zero": 0.2709042535747081, "coverage": [ { "threshold": 0.5, "selected": 3, "total": 176, "coverage": 0.017045454545454544 }, { "threshold": 0.7, "selected": 0, "total": 176, "coverage": 0.0 }, { "threshold": 0.8, "selected": 0, "total": 176, "coverage": 0.0 }, { "threshold": 0.9, "selected": 0, "total": 176, "coverage": 0.0 }, { "threshold": 0.95, "selected": 0, "total": 176, "coverage": 0.0 }, { "threshold": 0.99, "selected": 0, "total": 176, "coverage": 0.0 } ], "probability_metrics_all_rows": { "count": 176, "hard_count": 51, "accuracy": 0.45098039215686275, "expected_accuracy": 0.2709042535747081, "nll": 2.1680481914874337, "brier": 0.31628613125660077, "expected_brier": 0.8504008170758321, "ece_bins": 15, "multiclass_ece": 0.0965942356793526, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 0.017045454545454544, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "probability_metrics_successful_rows_only": { "count": 176, "hard_count": 51, "accuracy": 0.45098039215686275, "expected_accuracy": 0.2709042535747081, "nll": 2.1680481914874337, "brier": 0.31628613125660077, "expected_brier": 0.8504008170758321, "ece_bins": 15, "multiclass_ece": 0.0965942356793526, "coverage": [ { "threshold": 0.5, "selected": 3, "coverage": 0.017045454545454544, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "workflow-controls-v1/agent_trace_observability": { "count": 357, "successful": 357, "failed": 0, "inference_coverage": 1.0, "hard_count": 357, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 1.0, "coverage": [ { "threshold": 0.5, "selected": 357, "total": 357, "coverage": 1.0 }, { "threshold": 0.7, "selected": 357, "total": 357, "coverage": 1.0 }, { "threshold": 0.8, "selected": 357, "total": 357, "coverage": 1.0 }, { "threshold": 0.9, "selected": 357, "total": 357, "coverage": 1.0 }, { "threshold": 0.95, "selected": 357, "total": 357, "coverage": 1.0 }, { "threshold": 0.99, "selected": 357, "total": 357, "coverage": 1.0 } ], "probability_metrics_all_rows": { "count": 357, "hard_count": 357, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00044687482748441643, "brier": 2.946417322061725e-06, "expected_brier": 2.946417322060909e-06, "ece_bins": 15, "multiclass_ece": 0.0004461355302124166, "coverage": [ { "threshold": 0.5, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 357, "hard_count": 357, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.00044687482748441643, "brier": 2.946417322061725e-06, "expected_brier": 2.946417322060909e-06, "ece_bins": 15, "multiclass_ece": 0.0004461355302124166, "coverage": [ { "threshold": 0.5, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 357, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "workflow-controls-v1/customer_service": { "count": 456, "successful": 456, "failed": 0, "inference_coverage": 1.0, "hard_count": 456, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 1.0, "coverage": [ { "threshold": 0.5, "selected": 456, "total": 456, "coverage": 1.0 }, { "threshold": 0.7, "selected": 456, "total": 456, "coverage": 1.0 }, { "threshold": 0.8, "selected": 456, "total": 456, "coverage": 1.0 }, { "threshold": 0.9, "selected": 456, "total": 456, "coverage": 1.0 }, { "threshold": 0.95, "selected": 456, "total": 456, "coverage": 1.0 }, { "threshold": 0.99, "selected": 455, "total": 456, "coverage": 0.9978070175438597 } ], "probability_metrics_all_rows": { "count": 456, "hard_count": 456, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004573076748327324, "brier": 8.826622033700542e-06, "expected_brier": 8.826622033703088e-06, "ece_bins": 15, "multiclass_ece": 0.0004550441009002813, "coverage": [ { "threshold": 0.5, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 455, "coverage": 0.9978070175438597, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 456, "hard_count": 456, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.0004573076748327324, "brier": 8.826622033700542e-06, "expected_brier": 8.826622033703088e-06, "ece_bins": 15, "multiclass_ece": 0.0004550441009002813, "coverage": [ { "threshold": 0.5, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 456, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 455, "coverage": 0.9978070175438597, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "workflow-controls-v1/invoice_processing": { "count": 930, "successful": 930, "failed": 0, "inference_coverage": 1.0, "hard_count": 930, "accuracy_all_rows_failures_incorrect": 0.9967741935483871, "expected_accuracy_all_rows_failures_zero": 0.9967741935483871, "coverage": [ { "threshold": 0.5, "selected": 930, "total": 930, "coverage": 1.0 }, { "threshold": 0.7, "selected": 927, "total": 930, "coverage": 0.9967741935483871 }, { "threshold": 0.8, "selected": 925, "total": 930, "coverage": 0.9946236559139785 }, { "threshold": 0.9, "selected": 924, "total": 930, "coverage": 0.9935483870967742 }, { "threshold": 0.95, "selected": 924, "total": 930, "coverage": 0.9935483870967742 }, { "threshold": 0.99, "selected": 917, "total": 930, "coverage": 0.986021505376344 } ], "probability_metrics_all_rows": { "count": 930, "hard_count": 930, "accuracy": 0.9967741935483871, "expected_accuracy": 0.9967741935483871, "nll": 0.005458157001187219, "brier": 0.003648645648216325, "expected_brier": 0.0036486456482163225, "ece_bins": 15, "multiclass_ece": 0.0037258424516831987, "coverage": [ { "threshold": 0.5, "selected": 930, "coverage": 1.0, "accuracy": 0.9967741935483871, "expected_accuracy": 0.9967741935483871, "expected_risk": 0.003225806451612856 }, { "threshold": 0.7, "selected": 927, "coverage": 0.9967741935483871, "accuracy": 0.9978425026968716, "expected_accuracy": 0.9978425026968716, "expected_risk": 0.0021574973031284195 }, { "threshold": 0.8, "selected": 925, "coverage": 0.9946236559139785, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 924, "coverage": 0.9935483870967742, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 924, "coverage": 0.9935483870967742, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 917, "coverage": 0.986021505376344, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 930, "hard_count": 930, "accuracy": 0.9967741935483871, "expected_accuracy": 0.9967741935483871, "nll": 0.005458157001187219, "brier": 0.003648645648216325, "expected_brier": 0.0036486456482163225, "ece_bins": 15, "multiclass_ece": 0.0037258424516831987, "coverage": [ { "threshold": 0.5, "selected": 930, "coverage": 1.0, "accuracy": 0.9967741935483871, "expected_accuracy": 0.9967741935483871, "expected_risk": 0.003225806451612856 }, { "threshold": 0.7, "selected": 927, "coverage": 0.9967741935483871, "accuracy": 0.9978425026968716, "expected_accuracy": 0.9978425026968716, "expected_risk": 0.0021574973031284195 }, { "threshold": 0.8, "selected": 925, "coverage": 0.9946236559139785, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 924, "coverage": 0.9935483870967742, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 924, "coverage": 0.9935483870967742, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 917, "coverage": 0.986021505376344, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "workflow-controls-v1/security_incidents": { "count": 1326, "successful": 1326, "failed": 0, "inference_coverage": 1.0, "hard_count": 1326, "accuracy_all_rows_failures_incorrect": 0.9932126696832579, "expected_accuracy_all_rows_failures_zero": 0.9932126696832579, "coverage": [ { "threshold": 0.5, "selected": 1326, "total": 1326, "coverage": 1.0 }, { "threshold": 0.7, "selected": 1325, "total": 1326, "coverage": 0.9992458521870287 }, { "threshold": 0.8, "selected": 1325, "total": 1326, "coverage": 0.9992458521870287 }, { "threshold": 0.9, "selected": 1318, "total": 1326, "coverage": 0.9939668174962293 }, { "threshold": 0.95, "selected": 1303, "total": 1326, "coverage": 0.9826546003016591 }, { "threshold": 0.99, "selected": 1291, "total": 1326, "coverage": 0.973604826546003 } ], "probability_metrics_all_rows": { "count": 1326, "hard_count": 1326, "accuracy": 0.9932126696832579, "expected_accuracy": 0.9932126696832579, "nll": 0.022337704900579736, "brier": 0.011355443994287867, "expected_brier": 0.011355443994287865, "ece_bins": 15, "multiclass_ece": 0.004366133269192267, "coverage": [ { "threshold": 0.5, "selected": 1326, "coverage": 1.0, "accuracy": 0.9932126696832579, "expected_accuracy": 0.9932126696832579, "expected_risk": 0.006787330316742057 }, { "threshold": 0.7, "selected": 1325, "coverage": 0.9992458521870287, "accuracy": 0.9939622641509434, "expected_accuracy": 0.9939622641509434, "expected_risk": 0.006037735849056647 }, { "threshold": 0.8, "selected": 1325, "coverage": 0.9992458521870287, "accuracy": 0.9939622641509434, "expected_accuracy": 0.9939622641509434, "expected_risk": 0.006037735849056647 }, { "threshold": 0.9, "selected": 1318, "coverage": 0.9939668174962293, "accuracy": 0.9954476479514416, "expected_accuracy": 0.9954476479514416, "expected_risk": 0.004552352048558417 }, { "threshold": 0.95, "selected": 1303, "coverage": 0.9826546003016591, "accuracy": 0.9984650805832693, "expected_accuracy": 0.9984650805832693, "expected_risk": 0.0015349194167306734 }, { "threshold": 0.99, "selected": 1291, "coverage": 0.973604826546003, "accuracy": 0.9984508133230054, "expected_accuracy": 0.9984508133230054, "expected_risk": 0.0015491866769945517 } ] }, "probability_metrics_successful_rows_only": { "count": 1326, "hard_count": 1326, "accuracy": 0.9932126696832579, "expected_accuracy": 0.9932126696832579, "nll": 0.022337704900579736, "brier": 0.011355443994287867, "expected_brier": 0.011355443994287865, "ece_bins": 15, "multiclass_ece": 0.004366133269192267, "coverage": [ { "threshold": 0.5, "selected": 1326, "coverage": 1.0, "accuracy": 0.9932126696832579, "expected_accuracy": 0.9932126696832579, "expected_risk": 0.006787330316742057 }, { "threshold": 0.7, "selected": 1325, "coverage": 0.9992458521870287, "accuracy": 0.9939622641509434, "expected_accuracy": 0.9939622641509434, "expected_risk": 0.006037735849056647 }, { "threshold": 0.8, "selected": 1325, "coverage": 0.9992458521870287, "accuracy": 0.9939622641509434, "expected_accuracy": 0.9939622641509434, "expected_risk": 0.006037735849056647 }, { "threshold": 0.9, "selected": 1318, "coverage": 0.9939668174962293, "accuracy": 0.9954476479514416, "expected_accuracy": 0.9954476479514416, "expected_risk": 0.004552352048558417 }, { "threshold": 0.95, "selected": 1303, "coverage": 0.9826546003016591, "accuracy": 0.9984650805832693, "expected_accuracy": 0.9984650805832693, "expected_risk": 0.0015349194167306734 }, { "threshold": 0.99, "selected": 1291, "coverage": 0.973604826546003, "accuracy": 0.9984508133230054, "expected_accuracy": 0.9984508133230054, "expected_risk": 0.0015491866769945517 } ] } } }, "by_kind": { "choice": { "count": 2584, "successful": 2584, "failed": 0, "inference_coverage": 1.0, "hard_count": 2119, "accuracy_all_rows_failures_incorrect": 0.9188296366210477, "expected_accuracy_all_rows_failures_zero": 0.8180149292571892, "coverage": [ { "threshold": 0.5, "selected": 2050, "total": 2584, "coverage": 0.7933436532507739 }, { "threshold": 0.7, "selected": 1783, "total": 2584, "coverage": 0.690015479876161 }, { "threshold": 0.8, "selected": 1715, "total": 2584, "coverage": 0.6636996904024768 }, { "threshold": 0.9, "selected": 1620, "total": 2584, "coverage": 0.6269349845201239 }, { "threshold": 0.95, "selected": 1458, "total": 2584, "coverage": 0.5642414860681114 }, { "threshold": 0.99, "selected": 1056, "total": 2584, "coverage": 0.4086687306501548 } ], "probability_metrics_all_rows": { "count": 2584, "hard_count": 2119, "accuracy": 0.9188296366210477, "expected_accuracy": 0.8180149292571892, "nll": 0.45266508836146735, "brier": 0.11881345254759784, "expected_brier": 0.228759049606198, "ece_bins": 15, "multiclass_ece": 0.027110901433906452, "coverage": [ { "threshold": 0.5, "selected": 2050, "coverage": 0.7933436532507739, "accuracy": 0.9651898734177216, "expected_accuracy": 0.9271544715447155, "expected_risk": 0.07284552845528447 }, { "threshold": 0.7, "selected": 1783, "coverage": 0.690015479876161, "accuracy": 0.985827664399093, "expected_accuracy": 0.9804636380631894, "expected_risk": 0.01953636193681063 }, { "threshold": 0.8, "selected": 1715, "coverage": 0.6636996904024768, "accuracy": 0.9900526623756583, "expected_accuracy": 0.9883381924198251, "expected_risk": 0.011661807580174877 }, { "threshold": 0.9, "selected": 1620, "coverage": 0.6269349845201239, "accuracy": 0.9950617283950617, "expected_accuracy": 0.9950617283950617, "expected_risk": 0.004938271604938316 }, { "threshold": 0.95, "selected": 1458, "coverage": 0.5642414860681114, "accuracy": 0.9986282578875172, "expected_accuracy": 0.9986282578875172, "expected_risk": 0.00137174211248281 }, { "threshold": 0.99, "selected": 1056, "coverage": 0.4086687306501548, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 2584, "hard_count": 2119, "accuracy": 0.9188296366210477, "expected_accuracy": 0.8180149292571892, "nll": 0.45266508836146735, "brier": 0.11881345254759784, "expected_brier": 0.228759049606198, "ece_bins": 15, "multiclass_ece": 0.027110901433906452, "coverage": [ { "threshold": 0.5, "selected": 2050, "coverage": 0.7933436532507739, "accuracy": 0.9651898734177216, "expected_accuracy": 0.9271544715447155, "expected_risk": 0.07284552845528447 }, { "threshold": 0.7, "selected": 1783, "coverage": 0.690015479876161, "accuracy": 0.985827664399093, "expected_accuracy": 0.9804636380631894, "expected_risk": 0.01953636193681063 }, { "threshold": 0.8, "selected": 1715, "coverage": 0.6636996904024768, "accuracy": 0.9900526623756583, "expected_accuracy": 0.9883381924198251, "expected_risk": 0.011661807580174877 }, { "threshold": 0.9, "selected": 1620, "coverage": 0.6269349845201239, "accuracy": 0.9950617283950617, "expected_accuracy": 0.9950617283950617, "expected_risk": 0.004938271604938316 }, { "threshold": 0.95, "selected": 1458, "coverage": 0.5642414860681114, "accuracy": 0.9986282578875172, "expected_accuracy": 0.9986282578875172, "expected_risk": 0.00137174211248281 }, { "threshold": 0.99, "selected": 1056, "coverage": 0.4086687306501548, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "noul": { "count": 6364, "successful": 6364, "failed": 0, "inference_coverage": 1.0, "hard_count": 6364, "accuracy_all_rows_failures_incorrect": 0.9951288497800126, "expected_accuracy_all_rows_failures_zero": 0.9951288497800126, "coverage": [ { "threshold": 0.5, "selected": 6364, "total": 6364, "coverage": 1.0 }, { "threshold": 0.7, "selected": 6347, "total": 6364, "coverage": 0.9973287240729101 }, { "threshold": 0.8, "selected": 6338, "total": 6364, "coverage": 0.9959145191703331 }, { "threshold": 0.9, "selected": 6301, "total": 6364, "coverage": 0.990100565681961 }, { "threshold": 0.95, "selected": 6274, "total": 6364, "coverage": 0.98585795097423 }, { "threshold": 0.99, "selected": 6176, "total": 6364, "coverage": 0.9704588309239472 } ], "probability_metrics_all_rows": { "count": 6364, "hard_count": 6364, "accuracy": 0.9951288497800126, "expected_accuracy": 0.9951288497800126, "nll": 0.014164294737203246, "brier": 0.007765035196910086, "expected_brier": 0.007765035196910086, "ece_bins": 15, "multiclass_ece": 0.002491467968884566, "coverage": [ { "threshold": 0.5, "selected": 6364, "coverage": 1.0, "accuracy": 0.9951288497800126, "expected_accuracy": 0.9951288497800126, "expected_risk": 0.004871150219987408 }, { "threshold": 0.7, "selected": 6347, "coverage": 0.9973287240729101, "accuracy": 0.996061131243107, "expected_accuracy": 0.996061131243107, "expected_risk": 0.003938868756892977 }, { "threshold": 0.8, "selected": 6338, "coverage": 0.9959145191703331, "accuracy": 0.9965288734616599, "expected_accuracy": 0.9965288734616599, "expected_risk": 0.0034711265383401413 }, { "threshold": 0.9, "selected": 6301, "coverage": 0.990100565681961, "accuracy": 0.9980955403904143, "expected_accuracy": 0.9980955403904143, "expected_risk": 0.0019044596095857402 }, { "threshold": 0.95, "selected": 6274, "coverage": 0.98585795097423, "accuracy": 0.9993624481989162, "expected_accuracy": 0.9993624481989162, "expected_risk": 0.0006375518010838244 }, { "threshold": 0.99, "selected": 6176, "coverage": 0.9704588309239472, "accuracy": 0.9996761658031088, "expected_accuracy": 0.9996761658031088, "expected_risk": 0.00032383419689119286 } ] }, "probability_metrics_successful_rows_only": { "count": 6364, "hard_count": 6364, "accuracy": 0.9951288497800126, "expected_accuracy": 0.9951288497800126, "nll": 0.014164294737203246, "brier": 0.007765035196910086, "expected_brier": 0.007765035196910086, "ece_bins": 15, "multiclass_ece": 0.002491467968884566, "coverage": [ { "threshold": 0.5, "selected": 6364, "coverage": 1.0, "accuracy": 0.9951288497800126, "expected_accuracy": 0.9951288497800126, "expected_risk": 0.004871150219987408 }, { "threshold": 0.7, "selected": 6347, "coverage": 0.9973287240729101, "accuracy": 0.996061131243107, "expected_accuracy": 0.996061131243107, "expected_risk": 0.003938868756892977 }, { "threshold": 0.8, "selected": 6338, "coverage": 0.9959145191703331, "accuracy": 0.9965288734616599, "expected_accuracy": 0.9965288734616599, "expected_risk": 0.0034711265383401413 }, { "threshold": 0.9, "selected": 6301, "coverage": 0.990100565681961, "accuracy": 0.9980955403904143, "expected_accuracy": 0.9980955403904143, "expected_risk": 0.0019044596095857402 }, { "threshold": 0.95, "selected": 6274, "coverage": 0.98585795097423, "accuracy": 0.9993624481989162, "expected_accuracy": 0.9993624481989162, "expected_risk": 0.0006375518010838244 }, { "threshold": 0.99, "selected": 6176, "coverage": 0.9704588309239472, "accuracy": 0.9996761658031088, "expected_accuracy": 0.9996761658031088, "expected_risk": 0.00032383419689119286 } ] } }, "score": { "count": 1584, "successful": 1584, "failed": 0, "inference_coverage": 1.0, "hard_count": 1563, "accuracy_all_rows_failures_incorrect": 0.9718490083173384, "expected_accuracy_all_rows_failures_zero": 0.9650673400673401, "coverage": [ { "threshold": 0.5, "selected": 1531, "total": 1584, "coverage": 0.9665404040404041 }, { "threshold": 0.7, "selected": 1506, "total": 1584, "coverage": 0.9507575757575758 }, { "threshold": 0.8, "selected": 1499, "total": 1584, "coverage": 0.9463383838383839 }, { "threshold": 0.9, "selected": 1488, "total": 1584, "coverage": 0.9393939393939394 }, { "threshold": 0.95, "selected": 1476, "total": 1584, "coverage": 0.9318181818181818 }, { "threshold": 0.99, "selected": 1442, "total": 1584, "coverage": 0.9103535353535354 } ], "probability_metrics_all_rows": { "count": 1584, "hard_count": 1563, "accuracy": 0.9718490083173384, "expected_accuracy": 0.9650673400673401, "nll": 0.07532023419931186, "brier": 0.03504193237136221, "expected_brier": 0.042196814526244374, "ece_bins": 15, "multiclass_ece": 0.010182639374983222, "coverage": [ { "threshold": 0.5, "selected": 1531, "coverage": 0.9665404040404041, "accuracy": 0.990125082290981, "expected_accuracy": 0.9862834748530372, "expected_risk": 0.013716525146962755 }, { "threshold": 0.7, "selected": 1506, "coverage": 0.9507575757575758, "accuracy": 0.9926958831341302, "expected_accuracy": 0.9926958831341302, "expected_risk": 0.007304116865869847 }, { "threshold": 0.8, "selected": 1499, "coverage": 0.9463383838383839, "accuracy": 0.9959973315543695, "expected_accuracy": 0.9959973315543695, "expected_risk": 0.004002668445630464 }, { "threshold": 0.9, "selected": 1488, "coverage": 0.9393939393939394, "accuracy": 0.9993279569892473, "expected_accuracy": 0.9993279569892473, "expected_risk": 0.0006720430107527431 }, { "threshold": 0.95, "selected": 1476, "coverage": 0.9318181818181818, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1442, "coverage": 0.9103535353535354, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 1584, "hard_count": 1563, "accuracy": 0.9718490083173384, "expected_accuracy": 0.9650673400673401, "nll": 0.07532023419931186, "brier": 0.03504193237136221, "expected_brier": 0.042196814526244374, "ece_bins": 15, "multiclass_ece": 0.010182639374983222, "coverage": [ { "threshold": 0.5, "selected": 1531, "coverage": 0.9665404040404041, "accuracy": 0.990125082290981, "expected_accuracy": 0.9862834748530372, "expected_risk": 0.013716525146962755 }, { "threshold": 0.7, "selected": 1506, "coverage": 0.9507575757575758, "accuracy": 0.9926958831341302, "expected_accuracy": 0.9926958831341302, "expected_risk": 0.007304116865869847 }, { "threshold": 0.8, "selected": 1499, "coverage": 0.9463383838383839, "accuracy": 0.9959973315543695, "expected_accuracy": 0.9959973315543695, "expected_risk": 0.004002668445630464 }, { "threshold": 0.9, "selected": 1488, "coverage": 0.9393939393939394, "accuracy": 0.9993279569892473, "expected_accuracy": 0.9993279569892473, "expected_risk": 0.0006720430107527431 }, { "threshold": 0.95, "selected": 1476, "coverage": 0.9318181818181818, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 1442, "coverage": 0.9103535353535354, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } } } }, "ood": { "count": 15920, "successful": 15920, "failed": 0, "inference_coverage": 1.0, "hard_count": 15446, "accuracy_all_rows_failures_incorrect": 0.9196555742587078, "expected_accuracy_all_rows_failures_zero": 0.9040437668138673, "coverage": [ { "threshold": 0.5, "selected": 15349, "total": 15920, "coverage": 0.9641331658291458 }, { "threshold": 0.7, "selected": 14632, "total": 15920, "coverage": 0.9190954773869346 }, { "threshold": 0.8, "selected": 14183, "total": 15920, "coverage": 0.8908919597989949 }, { "threshold": 0.9, "selected": 13442, "total": 15920, "coverage": 0.8443467336683417 }, { "threshold": 0.95, "selected": 12751, "total": 15920, "coverage": 0.8009422110552764 }, { "threshold": 0.99, "selected": 10671, "total": 15920, "coverage": 0.6702889447236181 } ], "probability_metrics_all_rows": { "count": 15920, "hard_count": 15446, "accuracy": 0.9196555742587078, "expected_accuracy": 0.9040437668138673, "nll": 0.2994407995801986, "brier": 0.1266465263060649, "expected_brier": 0.1439772659530734, "ece_bins": 15, "multiclass_ece": 0.037398104632606045, "coverage": [ { "threshold": 0.5, "selected": 15349, "coverage": 0.9641331658291458, "accuracy": 0.9263081777544044, "expected_accuracy": 0.9224922361934546, "expected_risk": 0.07750776380654545 }, { "threshold": 0.7, "selected": 14632, "coverage": 0.9190954773869346, "accuracy": 0.939354574046219, "expected_accuracy": 0.9391630216876253, "expected_risk": 0.06083697831237467 }, { "threshold": 0.8, "selected": 14183, "coverage": 0.8908919597989949, "accuracy": 0.9484557890283458, "expected_accuracy": 0.9484241697807234, "expected_risk": 0.051575830219276586 }, { "threshold": 0.9, "selected": 13442, "coverage": 0.8443467336683417, "accuracy": 0.9607172085410312, "expected_accuracy": 0.9606829340871894, "expected_risk": 0.03931706591281059 }, { "threshold": 0.95, "selected": 12751, "coverage": 0.8009422110552764, "accuracy": 0.9714509803921568, "expected_accuracy": 0.971414006744569, "expected_risk": 0.028585993255430986 }, { "threshold": 0.99, "selected": 10671, "coverage": 0.6702889447236181, "accuracy": 0.9867866179364633, "expected_accuracy": 0.9867866179364633, "expected_risk": 0.013213382063536727 } ] }, "probability_metrics_successful_rows_only": { "count": 15920, "hard_count": 15446, "accuracy": 0.9196555742587078, "expected_accuracy": 0.9040437668138673, "nll": 0.2994407995801986, "brier": 0.1266465263060649, "expected_brier": 0.1439772659530734, "ece_bins": 15, "multiclass_ece": 0.037398104632606045, "coverage": [ { "threshold": 0.5, "selected": 15349, "coverage": 0.9641331658291458, "accuracy": 0.9263081777544044, "expected_accuracy": 0.9224922361934546, "expected_risk": 0.07750776380654545 }, { "threshold": 0.7, "selected": 14632, "coverage": 0.9190954773869346, "accuracy": 0.939354574046219, "expected_accuracy": 0.9391630216876253, "expected_risk": 0.06083697831237467 }, { "threshold": 0.8, "selected": 14183, "coverage": 0.8908919597989949, "accuracy": 0.9484557890283458, "expected_accuracy": 0.9484241697807234, "expected_risk": 0.051575830219276586 }, { "threshold": 0.9, "selected": 13442, "coverage": 0.8443467336683417, "accuracy": 0.9607172085410312, "expected_accuracy": 0.9606829340871894, "expected_risk": 0.03931706591281059 }, { "threshold": 0.95, "selected": 12751, "coverage": 0.8009422110552764, "accuracy": 0.9714509803921568, "expected_accuracy": 0.971414006744569, "expected_risk": 0.028585993255430986 }, { "threshold": 0.99, "selected": 10671, "coverage": 0.6702889447236181, "accuracy": 0.9867866179364633, "expected_accuracy": 0.9867866179364633, "expected_risk": 0.013213382063536727 } ] }, "by_source": { "customer-control-v1": { "count": 600, "successful": 600, "failed": 0, "inference_coverage": 1.0, "hard_count": 560, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 0.9633333333333334, "coverage": [ { "threshold": 0.5, "selected": 586, "total": 600, "coverage": 0.9766666666666667 }, { "threshold": 0.7, "selected": 559, "total": 600, "coverage": 0.9316666666666666 }, { "threshold": 0.8, "selected": 555, "total": 600, "coverage": 0.925 }, { "threshold": 0.9, "selected": 548, "total": 600, "coverage": 0.9133333333333333 }, { "threshold": 0.95, "selected": 540, "total": 600, "coverage": 0.9 }, { "threshold": 0.99, "selected": 512, "total": 600, "coverage": 0.8533333333333334 } ], "probability_metrics_all_rows": { "count": 600, "hard_count": 560, "accuracy": 1.0, "expected_accuracy": 0.9633333333333334, "nll": 0.06367998388489432, "brier": 0.002432497344975918, "expected_brier": 0.03909916401164259, "ece_bins": 15, "multiclass_ece": 0.007595238345599102, "coverage": [ { "threshold": 0.5, "selected": 586, "coverage": 0.9766666666666667, "accuracy": 1.0, "expected_accuracy": 0.976962457337884, "expected_risk": 0.023037542662115995 }, { "threshold": 0.7, "selected": 559, "coverage": 0.9316666666666666, "accuracy": 1.0, "expected_accuracy": 0.9991055456171736, "expected_risk": 0.0008944543828264262 }, { "threshold": 0.8, "selected": 555, "coverage": 0.925, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 548, "coverage": 0.9133333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 540, "coverage": 0.9, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 512, "coverage": 0.8533333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 600, "hard_count": 560, "accuracy": 1.0, "expected_accuracy": 0.9633333333333334, "nll": 0.06367998388489432, "brier": 0.002432497344975918, "expected_brier": 0.03909916401164259, "ece_bins": 15, "multiclass_ece": 0.007595238345599102, "coverage": [ { "threshold": 0.5, "selected": 586, "coverage": 0.9766666666666667, "accuracy": 1.0, "expected_accuracy": 0.976962457337884, "expected_risk": 0.023037542662115995 }, { "threshold": 0.7, "selected": 559, "coverage": 0.9316666666666666, "accuracy": 1.0, "expected_accuracy": 0.9991055456171736, "expected_risk": 0.0008944543828264262 }, { "threshold": 0.8, "selected": 555, "coverage": 0.925, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 548, "coverage": 0.9133333333333333, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 540, "coverage": 0.9, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 512, "coverage": 0.8533333333333334, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "painting-geometry-v1": { "count": 6912, "successful": 6912, "failed": 0, "inference_coverage": 1.0, "hard_count": 6912, "accuracy_all_rows_failures_incorrect": 0.8629918981481481, "expected_accuracy_all_rows_failures_zero": 0.8629918981481481, "coverage": [ { "threshold": 0.5, "selected": 6842, "total": 6912, "coverage": 0.9898726851851852 }, { "threshold": 0.7, "selected": 6420, "total": 6912, "coverage": 0.9288194444444444 }, { "threshold": 0.8, "selected": 6102, "total": 6912, "coverage": 0.8828125 }, { "threshold": 0.9, "selected": 5666, "total": 6912, "coverage": 0.8197337962962963 }, { "threshold": 0.95, "selected": 5246, "total": 6912, "coverage": 0.7589699074074074 }, { "threshold": 0.99, "selected": 3677, "total": 6912, "coverage": 0.5319733796296297 } ], "probability_metrics_all_rows": { "count": 6912, "hard_count": 6912, "accuracy": 0.8629918981481481, "expected_accuracy": 0.8629918981481481, "nll": 0.4601822966946938, "brier": 0.22129407318457964, "expected_brier": 0.22129407318457964, "ece_bins": 15, "multiclass_ece": 0.0793900158186277, "coverage": [ { "threshold": 0.5, "selected": 6842, "coverage": 0.9898726851851852, "accuracy": 0.8661210172464192, "expected_accuracy": 0.8661210172464192, "expected_risk": 0.1338789827535808 }, { "threshold": 0.7, "selected": 6420, "coverage": 0.9288194444444444, "accuracy": 0.8848909657320873, "expected_accuracy": 0.8848909657320873, "expected_risk": 0.11510903426791275 }, { "threshold": 0.8, "selected": 6102, "coverage": 0.8828125, "accuracy": 0.8982300884955752, "expected_accuracy": 0.8982300884955752, "expected_risk": 0.10176991150442483 }, { "threshold": 0.9, "selected": 5666, "coverage": 0.8197337962962963, "accuracy": 0.9200494175785386, "expected_accuracy": 0.9200494175785386, "expected_risk": 0.07995058242146136 }, { "threshold": 0.95, "selected": 5246, "coverage": 0.7589699074074074, "accuracy": 0.9412886008387342, "expected_accuracy": 0.9412886008387342, "expected_risk": 0.058711399161265776 }, { "threshold": 0.99, "selected": 3677, "coverage": 0.5319733796296297, "accuracy": 0.9709001903725863, "expected_accuracy": 0.9709001903725863, "expected_risk": 0.029099809627413653 } ] }, "probability_metrics_successful_rows_only": { "count": 6912, "hard_count": 6912, "accuracy": 0.8629918981481481, "expected_accuracy": 0.8629918981481481, "nll": 0.4601822966946938, "brier": 0.22129407318457964, "expected_brier": 0.22129407318457964, "ece_bins": 15, "multiclass_ece": 0.0793900158186277, "coverage": [ { "threshold": 0.5, "selected": 6842, "coverage": 0.9898726851851852, "accuracy": 0.8661210172464192, "expected_accuracy": 0.8661210172464192, "expected_risk": 0.1338789827535808 }, { "threshold": 0.7, "selected": 6420, "coverage": 0.9288194444444444, "accuracy": 0.8848909657320873, "expected_accuracy": 0.8848909657320873, "expected_risk": 0.11510903426791275 }, { "threshold": 0.8, "selected": 6102, "coverage": 0.8828125, "accuracy": 0.8982300884955752, "expected_accuracy": 0.8982300884955752, "expected_risk": 0.10176991150442483 }, { "threshold": 0.9, "selected": 5666, "coverage": 0.8197337962962963, "accuracy": 0.9200494175785386, "expected_accuracy": 0.9200494175785386, "expected_risk": 0.07995058242146136 }, { "threshold": 0.95, "selected": 5246, "coverage": 0.7589699074074074, "accuracy": 0.9412886008387342, "expected_accuracy": 0.9412886008387342, "expected_risk": 0.058711399161265776 }, { "threshold": 0.99, "selected": 3677, "coverage": 0.5319733796296297, "accuracy": 0.9709001903725863, "expected_accuracy": 0.9709001903725863, "expected_risk": 0.029099809627413653 } ] } }, "reasoning-control-v1": { "count": 750, "successful": 750, "failed": 0, "inference_coverage": 1.0, "hard_count": 750, "accuracy_all_rows_failures_incorrect": 0.736, "expected_accuracy_all_rows_failures_zero": 0.736, "coverage": [ { "threshold": 0.5, "selected": 644, "total": 750, "coverage": 0.8586666666666667 }, { "threshold": 0.7, "selected": 542, "total": 750, "coverage": 0.7226666666666667 }, { "threshold": 0.8, "selected": 492, "total": 750, "coverage": 0.656 }, { "threshold": 0.9, "selected": 414, "total": 750, "coverage": 0.552 }, { "threshold": 0.95, "selected": 373, "total": 750, "coverage": 0.49733333333333335 }, { "threshold": 0.99, "selected": 245, "total": 750, "coverage": 0.32666666666666666 } ], "probability_metrics_all_rows": { "count": 750, "hard_count": 750, "accuracy": 0.736, "expected_accuracy": 0.736, "nll": 0.6801700551809211, "brier": 0.3601332438976597, "expected_brier": 0.3601332438976597, "ece_bins": 15, "multiclass_ece": 0.0941763492512841, "coverage": [ { "threshold": 0.5, "selected": 644, "coverage": 0.8586666666666667, "accuracy": 0.8012422360248447, "expected_accuracy": 0.8012422360248447, "expected_risk": 0.19875776397515532 }, { "threshold": 0.7, "selected": 542, "coverage": 0.7226666666666667, "accuracy": 0.8431734317343174, "expected_accuracy": 0.8431734317343174, "expected_risk": 0.15682656826568264 }, { "threshold": 0.8, "selected": 492, "coverage": 0.656, "accuracy": 0.8760162601626016, "expected_accuracy": 0.8760162601626016, "expected_risk": 0.12398373983739841 }, { "threshold": 0.9, "selected": 414, "coverage": 0.552, "accuracy": 0.9106280193236715, "expected_accuracy": 0.9106280193236715, "expected_risk": 0.08937198067632846 }, { "threshold": 0.95, "selected": 373, "coverage": 0.49733333333333335, "accuracy": 0.9276139410187667, "expected_accuracy": 0.9276139410187667, "expected_risk": 0.07238605898123329 }, { "threshold": 0.99, "selected": 245, "coverage": 0.32666666666666666, "accuracy": 0.9591836734693877, "expected_accuracy": 0.9591836734693877, "expected_risk": 0.04081632653061229 } ] }, "probability_metrics_successful_rows_only": { "count": 750, "hard_count": 750, "accuracy": 0.736, "expected_accuracy": 0.736, "nll": 0.6801700551809211, "brier": 0.3601332438976597, "expected_brier": 0.3601332438976597, "ece_bins": 15, "multiclass_ece": 0.0941763492512841, "coverage": [ { "threshold": 0.5, "selected": 644, "coverage": 0.8586666666666667, "accuracy": 0.8012422360248447, "expected_accuracy": 0.8012422360248447, "expected_risk": 0.19875776397515532 }, { "threshold": 0.7, "selected": 542, "coverage": 0.7226666666666667, "accuracy": 0.8431734317343174, "expected_accuracy": 0.8431734317343174, "expected_risk": 0.15682656826568264 }, { "threshold": 0.8, "selected": 492, "coverage": 0.656, "accuracy": 0.8760162601626016, "expected_accuracy": 0.8760162601626016, "expected_risk": 0.12398373983739841 }, { "threshold": 0.9, "selected": 414, "coverage": 0.552, "accuracy": 0.9106280193236715, "expected_accuracy": 0.9106280193236715, "expected_risk": 0.08937198067632846 }, { "threshold": 0.95, "selected": 373, "coverage": 0.49733333333333335, "accuracy": 0.9276139410187667, "expected_accuracy": 0.9276139410187667, "expected_risk": 0.07238605898123329 }, { "threshold": 0.99, "selected": 245, "coverage": 0.32666666666666666, "accuracy": 0.9591836734693877, "expected_accuracy": 0.9591836734693877, "expected_risk": 0.04081632653061229 } ] } }, "snake-v1": { "count": 3464, "successful": 3464, "failed": 0, "inference_coverage": 1.0, "hard_count": 3192, "accuracy_all_rows_failures_incorrect": 0.9899749373433584, "expected_accuracy_all_rows_failures_zero": 0.9514049268668205, "coverage": [ { "threshold": 0.5, "selected": 3299, "total": 3464, "coverage": 0.9523672055427251 }, { "threshold": 0.7, "selected": 3151, "total": 3464, "coverage": 0.9096420323325635 }, { "threshold": 0.8, "selected": 3094, "total": 3464, "coverage": 0.8931870669745958 }, { "threshold": 0.9, "selected": 2929, "total": 3464, "coverage": 0.8455542725173211 }, { "threshold": 0.95, "selected": 2779, "total": 3464, "coverage": 0.8022517321016166 }, { "threshold": 0.99, "selected": 2592, "total": 3464, "coverage": 0.7482678983833718 } ], "probability_metrics_all_rows": { "count": 3464, "hard_count": 3192, "accuracy": 0.9899749373433584, "expected_accuracy": 0.9514049268668205, "nll": 0.10915249045149827, "brier": 0.020795628523113556, "expected_brier": 0.06015282636760932, "ece_bins": 15, "multiclass_ece": 0.013368579791205319, "coverage": [ { "threshold": 0.5, "selected": 3299, "coverage": 0.9523672055427251, "accuracy": 0.9899717956753369, "expected_accuracy": 0.9738809740325352, "expected_risk": 0.02611902596746485 }, { "threshold": 0.7, "selected": 3151, "coverage": 0.9096420323325635, "accuracy": 0.993006993006993, "expected_accuracy": 0.9921717973130224, "expected_risk": 0.007828202686977592 }, { "threshold": 0.8, "selected": 3094, "coverage": 0.8931870669745958, "accuracy": 0.9938570966698997, "expected_accuracy": 0.9936974789915967, "expected_risk": 0.006302521008403339 }, { "threshold": 0.9, "selected": 2929, "coverage": 0.8455542725173211, "accuracy": 0.994535519125683, "expected_accuracy": 0.9943666780471151, "expected_risk": 0.0056333219528849066 }, { "threshold": 0.95, "selected": 2779, "coverage": 0.8022517321016166, "accuracy": 0.9953203743700504, "expected_accuracy": 0.9951421374595179, "expected_risk": 0.00485786254048215 }, { "threshold": 0.99, "selected": 2592, "coverage": 0.7482678983833718, "accuracy": 0.9965277777777778, "expected_accuracy": 0.9965277777777778, "expected_risk": 0.00347222222222221 } ] }, "probability_metrics_successful_rows_only": { "count": 3464, "hard_count": 3192, "accuracy": 0.9899749373433584, "expected_accuracy": 0.9514049268668205, "nll": 0.10915249045149827, "brier": 0.020795628523113556, "expected_brier": 0.06015282636760932, "ece_bins": 15, "multiclass_ece": 0.013368579791205319, "coverage": [ { "threshold": 0.5, "selected": 3299, "coverage": 0.9523672055427251, "accuracy": 0.9899717956753369, "expected_accuracy": 0.9738809740325352, "expected_risk": 0.02611902596746485 }, { "threshold": 0.7, "selected": 3151, "coverage": 0.9096420323325635, "accuracy": 0.993006993006993, "expected_accuracy": 0.9921717973130224, "expected_risk": 0.007828202686977592 }, { "threshold": 0.8, "selected": 3094, "coverage": 0.8931870669745958, "accuracy": 0.9938570966698997, "expected_accuracy": 0.9936974789915967, "expected_risk": 0.006302521008403339 }, { "threshold": 0.9, "selected": 2929, "coverage": 0.8455542725173211, "accuracy": 0.994535519125683, "expected_accuracy": 0.9943666780471151, "expected_risk": 0.0056333219528849066 }, { "threshold": 0.95, "selected": 2779, "coverage": 0.8022517321016166, "accuracy": 0.9953203743700504, "expected_accuracy": 0.9951421374595179, "expected_risk": 0.00485786254048215 }, { "threshold": 0.99, "selected": 2592, "coverage": 0.7482678983833718, "accuracy": 0.9965277777777778, "expected_accuracy": 0.9965277777777778, "expected_risk": 0.00347222222222221 } ] } }, "vizdoom-basic-v1": { "count": 825, "successful": 825, "failed": 0, "inference_coverage": 1.0, "hard_count": 825, "accuracy_all_rows_failures_incorrect": 0.9745454545454545, "expected_accuracy_all_rows_failures_zero": 0.9745454545454545, "coverage": [ { "threshold": 0.5, "selected": 824, "total": 825, "coverage": 0.9987878787878788 }, { "threshold": 0.7, "selected": 812, "total": 825, "coverage": 0.9842424242424243 }, { "threshold": 0.8, "selected": 794, "total": 825, "coverage": 0.9624242424242424 }, { "threshold": 0.9, "selected": 748, "total": 825, "coverage": 0.9066666666666666 }, { "threshold": 0.95, "selected": 688, "total": 825, "coverage": 0.833939393939394 }, { "threshold": 0.99, "selected": 534, "total": 825, "coverage": 0.6472727272727272 } ], "probability_metrics_all_rows": { "count": 825, "hard_count": 825, "accuracy": 0.9745454545454545, "expected_accuracy": 0.9745454545454545, "nll": 0.07717523550004791, "brier": 0.0398977614679421, "expected_brier": 0.0398977614679421, "ece_bins": 15, "multiclass_ece": 0.021055164766154082, "coverage": [ { "threshold": 0.5, "selected": 824, "coverage": 0.9987878787878788, "accuracy": 0.9745145631067961, "expected_accuracy": 0.9745145631067961, "expected_risk": 0.025485436893203928 }, { "threshold": 0.7, "selected": 812, "coverage": 0.9842424242424243, "accuracy": 0.9778325123152709, "expected_accuracy": 0.9778325123152709, "expected_risk": 0.022167487684729092 }, { "threshold": 0.8, "selected": 794, "coverage": 0.9624242424242424, "accuracy": 0.9911838790931989, "expected_accuracy": 0.9911838790931989, "expected_risk": 0.008816120906801062 }, { "threshold": 0.9, "selected": 748, "coverage": 0.9066666666666666, "accuracy": 0.9973262032085561, "expected_accuracy": 0.9973262032085561, "expected_risk": 0.002673796791443861 }, { "threshold": 0.95, "selected": 688, "coverage": 0.833939393939394, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 534, "coverage": 0.6472727272727272, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 825, "hard_count": 825, "accuracy": 0.9745454545454545, "expected_accuracy": 0.9745454545454545, "nll": 0.07717523550004791, "brier": 0.0398977614679421, "expected_brier": 0.0398977614679421, "ece_bins": 15, "multiclass_ece": 0.021055164766154082, "coverage": [ { "threshold": 0.5, "selected": 824, "coverage": 0.9987878787878788, "accuracy": 0.9745145631067961, "expected_accuracy": 0.9745145631067961, "expected_risk": 0.025485436893203928 }, { "threshold": 0.7, "selected": 812, "coverage": 0.9842424242424243, "accuracy": 0.9778325123152709, "expected_accuracy": 0.9778325123152709, "expected_risk": 0.022167487684729092 }, { "threshold": 0.8, "selected": 794, "coverage": 0.9624242424242424, "accuracy": 0.9911838790931989, "expected_accuracy": 0.9911838790931989, "expected_risk": 0.008816120906801062 }, { "threshold": 0.9, "selected": 748, "coverage": 0.9066666666666666, "accuracy": 0.9973262032085561, "expected_accuracy": 0.9973262032085561, "expected_risk": 0.002673796791443861 }, { "threshold": 0.95, "selected": 688, "coverage": 0.833939393939394, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 534, "coverage": 0.6472727272727272, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "wikispeedia-v1": { "count": 219, "successful": 219, "failed": 0, "inference_coverage": 1.0, "hard_count": 57, "accuracy_all_rows_failures_incorrect": 0.6666666666666666, "expected_accuracy_all_rows_failures_zero": 0.3274433836077672, "coverage": [ { "threshold": 0.5, "selected": 4, "total": 219, "coverage": 0.0182648401826484 }, { "threshold": 0.7, "selected": 0, "total": 219, "coverage": 0.0 }, { "threshold": 0.8, "selected": 0, "total": 219, "coverage": 0.0 }, { "threshold": 0.9, "selected": 0, "total": 219, "coverage": 0.0 }, { "threshold": 0.95, "selected": 0, "total": 219, "coverage": 0.0 }, { "threshold": 0.99, "selected": 0, "total": 219, "coverage": 0.0 } ], "probability_metrics_all_rows": { "count": 219, "hard_count": 57, "accuracy": 0.6666666666666666, "expected_accuracy": 0.3274433836077672, "nll": 2.111795103417709, "brier": 0.29905520762617627, "expected_brier": 0.8359138461971435, "ece_bins": 15, "multiclass_ece": 0.14396741883440964, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 0.0182648401826484, "accuracy": 1.0, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] }, "probability_metrics_successful_rows_only": { "count": 219, "hard_count": 57, "accuracy": 0.6666666666666666, "expected_accuracy": 0.3274433836077672, "nll": 2.111795103417709, "brier": 0.29905520762617627, "expected_brier": 0.8359138461971435, "ece_bins": 15, "multiclass_ece": 0.14396741883440964, "coverage": [ { "threshold": 0.5, "selected": 4, "coverage": 0.0182648401826484, "accuracy": 1.0, "expected_accuracy": 0.75, "expected_risk": 0.25 }, { "threshold": 0.7, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.8, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.9, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.95, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null }, { "threshold": 0.99, "selected": 0, "coverage": 0.0, "accuracy": null, "expected_accuracy": null, "expected_risk": null } ] } }, "workflow-controls-v1/agent_trace_observability": { "count": 525, "successful": 525, "failed": 0, "inference_coverage": 1.0, "hard_count": 525, "accuracy_all_rows_failures_incorrect": 1.0, "expected_accuracy_all_rows_failures_zero": 1.0, "coverage": [ { "threshold": 0.5, "selected": 525, "total": 525, "coverage": 1.0 }, { "threshold": 0.7, "selected": 525, "total": 525, "coverage": 1.0 }, { "threshold": 0.8, "selected": 525, "total": 525, "coverage": 1.0 }, { "threshold": 0.9, "selected": 525, "total": 525, "coverage": 1.0 }, { "threshold": 0.95, "selected": 525, "total": 525, "coverage": 1.0 }, { "threshold": 0.99, "selected": 524, "total": 525, "coverage": 0.9980952380952381 } ], "probability_metrics_all_rows": { "count": 525, "hard_count": 525, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000545160068456591, "brier": 4.895507169664063e-06, "expected_brier": 4.8955071696642534e-06, "ece_bins": 15, "multiclass_ece": 0.0005439303275837422, "coverage": [ { "threshold": 0.5, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 524, "coverage": 0.9980952380952381, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 525, "hard_count": 525, "accuracy": 1.0, "expected_accuracy": 1.0, "nll": 0.000545160068456591, "brier": 4.895507169664063e-06, "expected_brier": 4.8955071696642534e-06, "ece_bins": 15, "multiclass_ece": 0.0005439303275837422, "coverage": [ { "threshold": 0.5, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.7, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.8, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.9, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.95, "selected": 525, "coverage": 1.0, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 524, "coverage": 0.9980952380952381, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "workflow-controls-v1/customer_service": { "count": 600, "successful": 600, "failed": 0, "inference_coverage": 1.0, "hard_count": 600, "accuracy_all_rows_failures_incorrect": 0.9783333333333334, "expected_accuracy_all_rows_failures_zero": 0.9783333333333334, "coverage": [ { "threshold": 0.5, "selected": 600, "total": 600, "coverage": 1.0 }, { "threshold": 0.7, "selected": 600, "total": 600, "coverage": 1.0 }, { "threshold": 0.8, "selected": 600, "total": 600, "coverage": 1.0 }, { "threshold": 0.9, "selected": 600, "total": 600, "coverage": 1.0 }, { "threshold": 0.95, "selected": 600, "total": 600, "coverage": 1.0 }, { "threshold": 0.99, "selected": 597, "total": 600, "coverage": 0.995 } ], "probability_metrics_all_rows": { "count": 600, "hard_count": 600, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "nll": 0.16283998438208533, "brier": 0.04328014604238859, "expected_brier": 0.04328014604238859, "ece_bins": 15, "multiclass_ece": 0.021130186926374108, "coverage": [ { "threshold": 0.5, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.7, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.8, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.9, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.95, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.99, "selected": 597, "coverage": 0.995, "accuracy": 0.9782244556113903, "expected_accuracy": 0.9782244556113903, "expected_risk": 0.021775544388609736 } ] }, "probability_metrics_successful_rows_only": { "count": 600, "hard_count": 600, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "nll": 0.16283998438208533, "brier": 0.04328014604238859, "expected_brier": 0.04328014604238859, "ece_bins": 15, "multiclass_ece": 0.021130186926374108, "coverage": [ { "threshold": 0.5, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.7, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.8, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.9, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.95, "selected": 600, "coverage": 1.0, "accuracy": 0.9783333333333334, "expected_accuracy": 0.9783333333333334, "expected_risk": 0.021666666666666612 }, { "threshold": 0.99, "selected": 597, "coverage": 0.995, "accuracy": 0.9782244556113903, "expected_accuracy": 0.9782244556113903, "expected_risk": 0.021775544388609736 } ] } }, "workflow-controls-v1/invoice_processing": { "count": 750, "successful": 750, "failed": 0, "inference_coverage": 1.0, "hard_count": 750, "accuracy_all_rows_failures_incorrect": 0.9973333333333333, "expected_accuracy_all_rows_failures_zero": 0.9973333333333333, "coverage": [ { "threshold": 0.5, "selected": 750, "total": 750, "coverage": 1.0 }, { "threshold": 0.7, "selected": 748, "total": 750, "coverage": 0.9973333333333333 }, { "threshold": 0.8, "selected": 746, "total": 750, "coverage": 0.9946666666666667 }, { "threshold": 0.9, "selected": 746, "total": 750, "coverage": 0.9946666666666667 }, { "threshold": 0.95, "selected": 744, "total": 750, "coverage": 0.992 }, { "threshold": 0.99, "selected": 743, "total": 750, "coverage": 0.9906666666666667 } ], "probability_metrics_all_rows": { "count": 750, "hard_count": 750, "accuracy": 0.9973333333333333, "expected_accuracy": 0.9973333333333333, "nll": 0.006416542163341697, "brier": 0.003983146029347859, "expected_brier": 0.003983146029347861, "ece_bins": 15, "multiclass_ece": 0.002603556053682448, "coverage": [ { "threshold": 0.5, "selected": 750, "coverage": 1.0, "accuracy": 0.9973333333333333, "expected_accuracy": 0.9973333333333333, "expected_risk": 0.002666666666666706 }, { "threshold": 0.7, "selected": 748, "coverage": 0.9973333333333333, "accuracy": 0.9986631016042781, "expected_accuracy": 0.9986631016042781, "expected_risk": 0.0013368983957219305 }, { "threshold": 0.8, "selected": 746, "coverage": 0.9946666666666667, "accuracy": 0.9986595174262735, "expected_accuracy": 0.9986595174262735, "expected_risk": 0.0013404825737265424 }, { "threshold": 0.9, "selected": 746, "coverage": 0.9946666666666667, "accuracy": 0.9986595174262735, "expected_accuracy": 0.9986595174262735, "expected_risk": 0.0013404825737265424 }, { "threshold": 0.95, "selected": 744, "coverage": 0.992, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 743, "coverage": 0.9906666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] }, "probability_metrics_successful_rows_only": { "count": 750, "hard_count": 750, "accuracy": 0.9973333333333333, "expected_accuracy": 0.9973333333333333, "nll": 0.006416542163341697, "brier": 0.003983146029347859, "expected_brier": 0.003983146029347861, "ece_bins": 15, "multiclass_ece": 0.002603556053682448, "coverage": [ { "threshold": 0.5, "selected": 750, "coverage": 1.0, "accuracy": 0.9973333333333333, "expected_accuracy": 0.9973333333333333, "expected_risk": 0.002666666666666706 }, { "threshold": 0.7, "selected": 748, "coverage": 0.9973333333333333, "accuracy": 0.9986631016042781, "expected_accuracy": 0.9986631016042781, "expected_risk": 0.0013368983957219305 }, { "threshold": 0.8, "selected": 746, "coverage": 0.9946666666666667, "accuracy": 0.9986595174262735, "expected_accuracy": 0.9986595174262735, "expected_risk": 0.0013404825737265424 }, { "threshold": 0.9, "selected": 746, "coverage": 0.9946666666666667, "accuracy": 0.9986595174262735, "expected_accuracy": 0.9986595174262735, "expected_risk": 0.0013404825737265424 }, { "threshold": 0.95, "selected": 744, "coverage": 0.992, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 }, { "threshold": 0.99, "selected": 743, "coverage": 0.9906666666666667, "accuracy": 1.0, "expected_accuracy": 1.0, "expected_risk": 0.0 } ] } }, "workflow-controls-v1/security_incidents": { "count": 1275, "successful": 1275, "failed": 0, "inference_coverage": 1.0, "hard_count": 1275, "accuracy_all_rows_failures_incorrect": 0.9929411764705882, "expected_accuracy_all_rows_failures_zero": 0.9929411764705882, "coverage": [ { "threshold": 0.5, "selected": 1275, "total": 1275, "coverage": 1.0 }, { "threshold": 0.7, "selected": 1275, "total": 1275, "coverage": 1.0 }, { "threshold": 0.8, "selected": 1275, "total": 1275, "coverage": 1.0 }, { "threshold": 0.9, "selected": 1266, "total": 1275, "coverage": 0.9929411764705882 }, { "threshold": 0.95, "selected": 1256, "total": 1275, "coverage": 0.9850980392156863 }, { "threshold": 0.99, "selected": 1247, "total": 1275, "coverage": 0.9780392156862745 } ], "probability_metrics_all_rows": { "count": 1275, "hard_count": 1275, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "nll": 0.02425288839114142, "brier": 0.012286914067882482, "expected_brier": 0.012286914067882486, "ece_bins": 15, "multiclass_ece": 0.004988858941316103, "coverage": [ { "threshold": 0.5, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.7, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.8, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.9, "selected": 1266, "coverage": 0.9929411764705882, "accuracy": 0.995260663507109, "expected_accuracy": 0.995260663507109, "expected_risk": 0.004739336492891044 }, { "threshold": 0.95, "selected": 1256, "coverage": 0.9850980392156863, "accuracy": 0.9976114649681529, "expected_accuracy": 0.9976114649681529, "expected_risk": 0.0023885350318471055 }, { "threshold": 0.99, "selected": 1247, "coverage": 0.9780392156862745, "accuracy": 0.9983961507618284, "expected_accuracy": 0.9983961507618284, "expected_risk": 0.001603849238171584 } ] }, "probability_metrics_successful_rows_only": { "count": 1275, "hard_count": 1275, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "nll": 0.02425288839114142, "brier": 0.012286914067882482, "expected_brier": 0.012286914067882486, "ece_bins": 15, "multiclass_ece": 0.004988858941316103, "coverage": [ { "threshold": 0.5, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.7, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.8, "selected": 1275, "coverage": 1.0, "accuracy": 0.9929411764705882, "expected_accuracy": 0.9929411764705882, "expected_risk": 0.007058823529411784 }, { "threshold": 0.9, "selected": 1266, "coverage": 0.9929411764705882, "accuracy": 0.995260663507109, "expected_accuracy": 0.995260663507109, "expected_risk": 0.004739336492891044 }, { "threshold": 0.95, "selected": 1256, "coverage": 0.9850980392156863, "accuracy": 0.9976114649681529, "expected_accuracy": 0.9976114649681529, "expected_risk": 0.0023885350318471055 }, { "threshold": 0.99, "selected": 1247, "coverage": 0.9780392156862745, "accuracy": 0.9983961507618284, "expected_accuracy": 0.9983961507618284, "expected_risk": 0.001603849238171584 } ] } } }, "by_kind": { "choice": { "count": 3438, "successful": 3438, "failed": 0, "inference_coverage": 1.0, "hard_count": 2993, "accuracy_all_rows_failures_incorrect": 0.863013698630137, "expected_accuracy_all_rows_failures_zero": 0.8021747433614798, "coverage": [ { "threshold": 0.5, "selected": 2972, "total": 3438, "coverage": 0.8644560791157649 }, { "threshold": 0.7, "selected": 2643, "total": 3438, "coverage": 0.768760907504363 }, { "threshold": 0.8, "selected": 2483, "total": 3438, "coverage": 0.7222222222222222 }, { "threshold": 0.9, "selected": 2147, "total": 3438, "coverage": 0.6244909831297266 }, { "threshold": 0.95, "selected": 1778, "total": 3438, "coverage": 0.5171611401977894 }, { "threshold": 0.99, "selected": 863, "total": 3438, "coverage": 0.2510180337405468 } ], "probability_metrics_all_rows": { "count": 3438, "hard_count": 2993, "accuracy": 0.863013698630137, "expected_accuracy": 0.8021747433614798, "nll": 0.6259869551780559, "brier": 0.20650286979189394, "expected_brier": 0.28195527676698856, "ece_bins": 15, "multiclass_ece": 0.05203723428051684, "coverage": [ { "threshold": 0.5, "selected": 2972, "coverage": 0.8644560791157649, "accuracy": 0.8779803646563815, "expected_accuracy": 0.8626626289816062, "expected_risk": 0.13733737101839383 }, { "threshold": 0.7, "selected": 2643, "coverage": 0.768760907504363, "accuracy": 0.8976109215017065, "expected_accuracy": 0.8966452263841594, "expected_risk": 0.10335477361584056 }, { "threshold": 0.8, "selected": 2483, "coverage": 0.7222222222222222, "accuracy": 0.9129734085414988, "expected_accuracy": 0.9128070881997583, "expected_risk": 0.08719291180024169 }, { "threshold": 0.9, "selected": 2147, "coverage": 0.6244909831297266, "accuracy": 0.9263746505125815, "expected_accuracy": 0.9261760596180717, "expected_risk": 0.0738239403819283 }, { "threshold": 0.95, "selected": 1778, "coverage": 0.5171611401977894, "accuracy": 0.9465391108610017, "expected_accuracy": 0.9462879640044994, "expected_risk": 0.05371203599550056 }, { "threshold": 0.99, "selected": 863, "coverage": 0.2510180337405468, "accuracy": 0.9733487833140209, "expected_accuracy": 0.9733487833140209, "expected_risk": 0.02665121668597914 } ] }, "probability_metrics_successful_rows_only": { "count": 3438, "hard_count": 2993, "accuracy": 0.863013698630137, "expected_accuracy": 0.8021747433614798, "nll": 0.6259869551780559, "brier": 0.20650286979189394, "expected_brier": 0.28195527676698856, "ece_bins": 15, "multiclass_ece": 0.05203723428051684, "coverage": [ { "threshold": 0.5, "selected": 2972, "coverage": 0.8644560791157649, "accuracy": 0.8779803646563815, "expected_accuracy": 0.8626626289816062, "expected_risk": 0.13733737101839383 }, { "threshold": 0.7, "selected": 2643, "coverage": 0.768760907504363, "accuracy": 0.8976109215017065, "expected_accuracy": 0.8966452263841594, "expected_risk": 0.10335477361584056 }, { "threshold": 0.8, "selected": 2483, "coverage": 0.7222222222222222, "accuracy": 0.9129734085414988, "expected_accuracy": 0.9128070881997583, "expected_risk": 0.08719291180024169 }, { "threshold": 0.9, "selected": 2147, "coverage": 0.6244909831297266, "accuracy": 0.9263746505125815, "expected_accuracy": 0.9261760596180717, "expected_risk": 0.0738239403819283 }, { "threshold": 0.95, "selected": 1778, "coverage": 0.5171611401977894, "accuracy": 0.9465391108610017, "expected_accuracy": 0.9462879640044994, "expected_risk": 0.05371203599550056 }, { "threshold": 0.99, "selected": 863, "coverage": 0.2510180337405468, "accuracy": 0.9733487833140209, "expected_accuracy": 0.9733487833140209, "expected_risk": 0.02665121668597914 } ] } }, "noul": { "count": 9929, "successful": 9929, "failed": 0, "inference_coverage": 1.0, "hard_count": 9929, "accuracy_all_rows_failures_incorrect": 0.9417866854668144, "expected_accuracy_all_rows_failures_zero": 0.9417866854668144, "coverage": [ { "threshold": 0.5, "selected": 9929, "total": 9929, "coverage": 1.0 }, { "threshold": 0.7, "selected": 9661, "total": 9929, "coverage": 0.973008359351395 }, { "threshold": 0.8, "selected": 9448, "total": 9929, "coverage": 0.9515560479403766 }, { "threshold": 0.9, "selected": 9151, "total": 9929, "coverage": 0.9216436700574075 }, { "threshold": 0.95, "selected": 8928, "total": 9929, "coverage": 0.899184207875919 }, { "threshold": 0.99, "selected": 8067, "total": 9929, "coverage": 0.8124685265384228 } ], "probability_metrics_all_rows": { "count": 9929, "hard_count": 9929, "accuracy": 0.9417866854668144, "expected_accuracy": 0.9417866854668144, "nll": 0.1586976512072927, "brier": 0.08958742899221635, "expected_brier": 0.08958742899221635, "ece_bins": 15, "multiclass_ece": 0.03298841702250886, "coverage": [ { "threshold": 0.5, "selected": 9929, "coverage": 1.0, "accuracy": 0.9417866854668144, "expected_accuracy": 0.9417866854668144, "expected_risk": 0.05821331453318557 }, { "threshold": 0.7, "selected": 9661, "coverage": 0.973008359351395, "accuracy": 0.9533174619604596, "expected_accuracy": 0.9533174619604596, "expected_risk": 0.04668253803954037 }, { "threshold": 0.8, "selected": 9448, "coverage": 0.9515560479403766, "accuracy": 0.9613674851820491, "expected_accuracy": 0.9613674851820491, "expected_risk": 0.038632514817950936 }, { "threshold": 0.9, "selected": 9151, "coverage": 0.9216436700574075, "accuracy": 0.973117692055513, "expected_accuracy": 0.973117692055513, "expected_risk": 0.026882307944486983 }, { "threshold": 0.95, "selected": 8928, "coverage": 0.899184207875919, "accuracy": 0.9816308243727598, "expected_accuracy": 0.9816308243727598, "expected_risk": 0.018369175627240164 }, { "threshold": 0.99, "selected": 8067, "coverage": 0.8124685265384228, "accuracy": 0.9935539853725053, "expected_accuracy": 0.9935539853725053, "expected_risk": 0.006446014627494723 } ] }, "probability_metrics_successful_rows_only": { "count": 9929, "hard_count": 9929, "accuracy": 0.9417866854668144, "expected_accuracy": 0.9417866854668144, "nll": 0.1586976512072927, "brier": 0.08958742899221635, "expected_brier": 0.08958742899221635, "ece_bins": 15, "multiclass_ece": 0.03298841702250886, "coverage": [ { "threshold": 0.5, "selected": 9929, "coverage": 1.0, "accuracy": 0.9417866854668144, "expected_accuracy": 0.9417866854668144, "expected_risk": 0.05821331453318557 }, { "threshold": 0.7, "selected": 9661, "coverage": 0.973008359351395, "accuracy": 0.9533174619604596, "expected_accuracy": 0.9533174619604596, "expected_risk": 0.04668253803954037 }, { "threshold": 0.8, "selected": 9448, "coverage": 0.9515560479403766, "accuracy": 0.9613674851820491, "expected_accuracy": 0.9613674851820491, "expected_risk": 0.038632514817950936 }, { "threshold": 0.9, "selected": 9151, "coverage": 0.9216436700574075, "accuracy": 0.973117692055513, "expected_accuracy": 0.973117692055513, "expected_risk": 0.026882307944486983 }, { "threshold": 0.95, "selected": 8928, "coverage": 0.899184207875919, "accuracy": 0.9816308243727598, "expected_accuracy": 0.9816308243727598, "expected_risk": 0.018369175627240164 }, { "threshold": 0.99, "selected": 8067, "coverage": 0.8124685265384228, "accuracy": 0.9935539853725053, "expected_accuracy": 0.9935539853725053, "expected_risk": 0.006446014627494723 } ] } }, "score": { "count": 2553, "successful": 2553, "failed": 0, "inference_coverage": 1.0, "hard_count": 2524, "accuracy_all_rows_failures_incorrect": 0.8997622820919176, "expected_accuracy_all_rows_failures_zero": 0.8944379161770466, "coverage": [ { "threshold": 0.5, "selected": 2448, "total": 2553, "coverage": 0.9588719153936546 }, { "threshold": 0.7, "selected": 2328, "total": 2553, "coverage": 0.9118683901292597 }, { "threshold": 0.8, "selected": 2252, "total": 2553, "coverage": 0.8820994907951429 }, { "threshold": 0.9, "selected": 2144, "total": 2553, "coverage": 0.8397963180571876 }, { "threshold": 0.95, "selected": 2045, "total": 2553, "coverage": 0.8010184097140619 }, { "threshold": 0.99, "selected": 1741, "total": 2553, "coverage": 0.681942812377595 } ], "probability_metrics_all_rows": { "count": 2553, "hard_count": 2524, "accuracy": 0.8997622820919176, "expected_accuracy": 0.8944379161770466, "nll": 0.40706831123282244, "brier": 0.16323629063231726, "expected_brier": 0.16969927535617155, "ece_bins": 15, "multiclass_ece": 0.05413140129609939, "coverage": [ { "threshold": 0.5, "selected": 2448, "coverage": 0.9588719153936546, "accuracy": 0.9197860962566845, "expected_accuracy": 0.9168709150326797, "expected_risk": 0.0831290849673203 }, { "threshold": 0.7, "selected": 2328, "coverage": 0.9118683901292597, "accuracy": 0.9286941580756014, "expected_accuracy": 0.9286941580756014, "expected_risk": 0.0713058419243986 }, { "threshold": 0.8, "selected": 2252, "coverage": 0.8820994907951429, "accuracy": 0.933392539964476, "expected_accuracy": 0.933392539964476, "expected_risk": 0.06660746003552398 }, { "threshold": 0.9, "selected": 2144, "coverage": 0.8397963180571876, "accuracy": 0.9421641791044776, "expected_accuracy": 0.9421641791044776, "expected_risk": 0.057835820895522416 }, { "threshold": 0.95, "selected": 2045, "coverage": 0.8010184097140619, "accuracy": 0.9486552567237164, "expected_accuracy": 0.9486552567237164, "expected_risk": 0.05134474327628358 }, { "threshold": 0.99, "selected": 1741, "coverage": 0.681942812377595, "accuracy": 0.9620907524411257, "expected_accuracy": 0.9620907524411257, "expected_risk": 0.03790924755887426 } ] }, "probability_metrics_successful_rows_only": { "count": 2553, "hard_count": 2524, "accuracy": 0.8997622820919176, "expected_accuracy": 0.8944379161770466, "nll": 0.40706831123282244, "brier": 0.16323629063231726, "expected_brier": 0.16969927535617155, "ece_bins": 15, "multiclass_ece": 0.05413140129609939, "coverage": [ { "threshold": 0.5, "selected": 2448, "coverage": 0.9588719153936546, "accuracy": 0.9197860962566845, "expected_accuracy": 0.9168709150326797, "expected_risk": 0.0831290849673203 }, { "threshold": 0.7, "selected": 2328, "coverage": 0.9118683901292597, "accuracy": 0.9286941580756014, "expected_accuracy": 0.9286941580756014, "expected_risk": 0.0713058419243986 }, { "threshold": 0.8, "selected": 2252, "coverage": 0.8820994907951429, "accuracy": 0.933392539964476, "expected_accuracy": 0.933392539964476, "expected_risk": 0.06660746003552398 }, { "threshold": 0.9, "selected": 2144, "coverage": 0.8397963180571876, "accuracy": 0.9421641791044776, "expected_accuracy": 0.9421641791044776, "expected_risk": 0.057835820895522416 }, { "threshold": 0.95, "selected": 2045, "coverage": 0.8010184097140619, "accuracy": 0.9486552567237164, "expected_accuracy": 0.9486552567237164, "expected_risk": 0.05134474327628358 }, { "threshold": 0.99, "selected": 1741, "coverage": 0.681942812377595, "accuracy": 0.9620907524411257, "expected_accuracy": 0.9620907524411257, "expected_risk": 0.03790924755887426 } ] } } } } }, "full_data_baseline_evaluated": false, "full_data_training_gain": null, "limitations": [ "No full-data baseline exists in this run. Earlier 512-row-per-split baseline results cannot establish full-data training gain.", "Per-group metrics were recomputed and compared for every group; compact report retains source/kind metrics and group counts/hashes. Original complete summary remains in ignored evidence.", "Snapshot/checkpoint hashes bind this evaluation to the packaged inference files; the trainer's historical reload lacked a contemporaneous weight digest.", "Synthetic held-out decision accuracy is not end-to-end game, browser or flight performance. Four-replica wall time is not single-GPU latency." ], "schema_version": 1, "status": "passed", "source_audit_sha256": "d490dd1636acebaf393fe181dfd0f65e66eb54edb7247b4abffd74a3d0891100", "source_audit_path": "reports/full-data-eval-n1-v1/9b-audit.json", "scope": "Existing completed full release-v2 held-out evaluation, bound to the unchanged five packaged inference files. No new model inference was run for this Hugging Face upload." }