{ "source_report": { "path": "d1-visual-adaptation-balanced-v3/results/report.json", "bytes": 52706054, "sha256": "1a1d2000b775789f8a29ecfa0fe90716cd11daafb4a032938e223012f84302b2" }, "synthetic_only": true, "real_site_generalization_verified": false, "labels": "AI-authored, independently AI-reviewed DOM/pixels; not human-expert labels", "models": { "A": { "suites": { "final_test": { "summary": { "questions": 144, "labelled_questions": 144, "unlabeled_questions": 0, "categorical_correct": 48, "categorical_total": 126, "categorical_accuracy": 0.38095238095238093, "score_count": 18, "score_mae": 0.7362379016349981, "score_rmse": 0.950402944149273, "score_normalized_mae": 0.2454126338783327, "score_argmax_class_match_diagnostic": 6, "mean_ground_truth_probability": 0.35284003049853, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 59, "tp": 21, "fp": 38, "fn": 3, "tn": 10, "precision": 0.3559322033898305, "recall": 0.875, "f1": 0.5060240963855421, "false_completion_count": 38, "false_completion_denominator": 48, "false_completion_rate": 0.7916666666666666, "false_completion_known_negative_count": 18, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 20, "false_completion_unknown_denominator": 24, "unknown_correct": 4, "unknown_total": 24, "unknown_accuracy": 0.16666666666666666 }, "uncertainty": { "correct": 6, "total": 28, "accuracy": 0.21428571428571427, "predicted_unknown": 18, "false_abstention_count": 12, "false_abstention_known_population": 56, "false_abstention_rate": 0.21428571428571427, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 10, "categorical_total": 24, "categorical_accuracy": 0.4166666666666667, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.3880929280227671, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 12, "tp": 4, "fp": 8, "fn": 0, "tn": 0, "precision": 0.3333333333333333, "recall": 1.0, "f1": 0.5, "false_completion_count": 8, "false_completion_denominator": 8, "false_completion_rate": 1.0, "false_completion_known_negative_count": 4, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 4, "false_completion_unknown_denominator": 4, "unknown_correct": 0, "unknown_total": 4, "unknown_accuracy": 0.0 }, "uncertainty": { "correct": 0, "total": 4, "accuracy": 0.0, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 7, "categorical_total": 18, "categorical_accuracy": 0.3888888888888889, "score_count": 6, "score_mae": 0.6117651269702643, "score_rmse": 0.8903388123553005, "score_normalized_mae": 0.2039217089900881, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.3949909739506741, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 12, "tp": 4, "fp": 8, "fn": 0, "tn": 0, "precision": 0.3333333333333333, "recall": 1.0, "f1": 0.5, "false_completion_count": 8, "false_completion_denominator": 8, "false_completion_rate": 1.0, "false_completion_known_negative_count": 4, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 4, "false_completion_unknown_denominator": 4, "unknown_correct": 0, "unknown_total": 4, "unknown_accuracy": 0.0 }, "uncertainty": { "correct": 0, "total": 4, "accuracy": 0.0, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 7, "categorical_total": 18, "categorical_accuracy": 0.3888888888888889, "score_count": 6, "score_mae": 0.9169420438508192, "score_rmse": 1.1117730731421402, "score_normalized_mae": 0.3056473479502731, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.33344664486745995, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 7, "tp": 3, "fp": 4, "fn": 1, "tn": 4, "precision": 0.42857142857142855, "recall": 0.75, "f1": 0.5454545454545454, "false_completion_count": 4, "false_completion_denominator": 8, "false_completion_rate": 0.5, "false_completion_known_negative_count": 1, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 3, "false_completion_unknown_denominator": 4, "unknown_correct": 1, "unknown_total": 4, "unknown_accuracy": 0.25 }, "uncertainty": { "correct": 1, "total": 4, "accuracy": 0.25, "predicted_unknown": 5, "false_abstention_count": 4, "false_abstention_known_population": 8, "false_abstention_rate": 0.5, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 24, "categorical_accuracy": 0.375, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.3354403299999831, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 11, "tp": 4, "fp": 7, "fn": 0, "tn": 1, "precision": 0.36363636363636365, "recall": 1.0, "f1": 0.5333333333333333, "false_completion_count": 7, "false_completion_denominator": 8, "false_completion_rate": 0.875, "false_completion_known_negative_count": 4, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 3, "false_completion_unknown_denominator": 4, "unknown_correct": 1, "unknown_total": 4, "unknown_accuracy": 0.25 }, "uncertainty": { "correct": 1, "total": 4, "accuracy": 0.25, "predicted_unknown": 1, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 24, "categorical_accuracy": 0.375, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.33362971385698376, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 6, "tp": 2, "fp": 4, "fn": 2, "tn": 4, "precision": 0.3333333333333333, "recall": 0.5, "f1": 0.4, "false_completion_count": 4, "false_completion_denominator": 8, "false_completion_rate": 0.5, "false_completion_known_negative_count": 2, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 2, "false_completion_unknown_denominator": 4, "unknown_correct": 2, "unknown_total": 4, "unknown_accuracy": 0.5 }, "uncertainty": { "correct": 4, "total": 8, "accuracy": 0.5, "predicted_unknown": 11, "false_abstention_count": 7, "false_abstention_known_population": 16, "false_abstention_rate": 0.4375, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 18, "categorical_accuracy": 0.3333333333333333, "score_count": 6, "score_mae": 0.6800065340839107, "score_rmse": 0.8252603842380003, "score_normalized_mae": 0.22666884469463688, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.33143959229331205, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 11, "tp": 4, "fp": 7, "fn": 0, "tn": 1, "precision": 0.36363636363636365, "recall": 1.0, "f1": 0.5333333333333333, "false_completion_count": 7, "false_completion_denominator": 8, "false_completion_rate": 0.875, "false_completion_known_negative_count": 3, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 4, "false_completion_unknown_denominator": 4, "unknown_correct": 0, "unknown_total": 4, "unknown_accuracy": 0.0 }, "uncertainty": { "correct": 0, "total": 4, "accuracy": 0.0, "predicted_unknown": 1, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 108, "labelled_questions": 108, "unlabeled_questions": 0, "categorical_correct": 39, "categorical_total": 108, "categorical_accuracy": 0.3611111111111111, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.3321999806037099, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 59, "tp": 21, "fp": 38, "fn": 3, "tn": 10, "precision": 0.3559322033898305, "recall": 0.875, "f1": 0.5060240963855421, "false_completion_count": 38, "false_completion_denominator": 48, "false_completion_rate": 0.7916666666666666, "false_completion_known_negative_count": 18, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 20, "false_completion_unknown_denominator": 24, "unknown_correct": 4, "unknown_total": 24, "unknown_accuracy": 0.16666666666666666 }, "uncertainty": { "correct": 6, "total": 28, "accuracy": 0.21428571428571427, "predicted_unknown": 18, "false_abstention_count": 12, "false_abstention_known_population": 56, "false_abstention_rate": 0.21428571428571427, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 18, "categorical_accuracy": 0.5, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.4690424129512394, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 18, "score_mae": 0.7362379016349981, "score_rmse": 0.950402944149273, "score_normalized_mae": 0.2454126338783327, "score_argmax_class_match_diagnostic": 6, "mean_ground_truth_probability": 0.3604779474147411, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "old_browser": { "summary": { "questions": 120, "labelled_questions": 120, "unlabeled_questions": 0, "categorical_correct": 43, "categorical_total": 90, "categorical_accuracy": 0.4777777777777778, "score_count": 30, "score_mae": 0.9737614470393585, "score_rmse": 1.2148143494095016, "score_normalized_mae": 0.29860634894107785, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.4105311758448655, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 14, "unknown_total": 14, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 17, "total": 18, "accuracy": 0.9444444444444444, "predicted_unknown": 33, "false_abstention_count": 16, "false_abstention_known_population": 20, "false_abstention_rate": 0.8, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 3, "categorical_total": 15, "categorical_accuracy": 0.2, "score_count": 5, "score_mae": 0.7003059403970837, "score_rmse": 0.7552460660779523, "score_normalized_mae": 0.23343531346569457, "score_argmax_class_match_diagnostic": 1, "mean_ground_truth_probability": 0.22589204203104601, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 12, "categorical_total": 15, "categorical_accuracy": 0.8, "score_count": 5, "score_mae": 0.8907763667404651, "score_rmse": 1.0215205298647336, "score_normalized_mae": 0.29692545558015504, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5972411710792584, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 1, "uncertain_population": 2, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 1, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 2, "unknown_correct": 2, "unknown_total": 2, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 2, "total": 2, "accuracy": 1.0, "predicted_unknown": 3, "false_abstention_count": 1, "false_abstention_known_population": 2, "false_abstention_rate": 0.5, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 15, "categorical_accuracy": 0.6, "score_count": 5, "score_mae": 1.1691360032418743, "score_rmse": 1.4191829080758471, "score_normalized_mae": 0.23382720064837487, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.5117919081822038, "completion": { "claims": 10, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 7, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 9, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 9, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 7, "unknown_correct": 7, "unknown_total": 7, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 7, "total": 7, "accuracy": 1.0, "predicted_unknown": 10, "false_abstention_count": 3, "false_abstention_known_population": 3, "false_abstention_rate": 1.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 4, "categorical_total": 15, "categorical_accuracy": 0.26666666666666666, "score_count": 5, "score_mae": 1.7031731545833737, "score_rmse": 1.8763876404664517, "score_normalized_mae": 0.5677243848611245, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.2106524537270161, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 1, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 1, "unknown_correct": 1, "unknown_total": 1, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 1, "total": 1, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 3, "false_abstention_known_population": 3, "false_abstention_rate": 1.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 15, "categorical_accuracy": 0.4, "score_count": 5, "score_mae": 0.6278841726481914, "score_rmse": 0.832546248865394, "score_normalized_mae": 0.20929472421606382, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.3636034029354846, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 3, "total": 4, "accuracy": 0.75, "predicted_unknown": 7, "false_abstention_count": 4, "false_abstention_known_population": 6, "false_abstention_rate": 0.6666666666666666, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 15, "categorical_accuracy": 0.6, "score_count": 5, "score_mae": 0.751293044625163, "score_rmse": 1.0063292937981685, "score_normalized_mae": 0.2504310148750543, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.5540060771141839, "completion": { "claims": 10, "positive_population": 4, "known_negative_population": 2, "uncertain_population": 4, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 4, "tn": 6, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 6, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 9, "false_abstention_count": 5, "false_abstention_known_population": 6, "false_abstention_rate": 0.8333333333333334, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 60, "labelled_questions": 60, "unlabeled_questions": 0, "categorical_correct": 27, "categorical_total": 60, "categorical_accuracy": 0.45, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.40605031248833257, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 14, "unknown_total": 14, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 17, "total": 18, "accuracy": 0.9444444444444444, "predicted_unknown": 33, "false_abstention_count": 16, "false_abstention_known_population": 20, "false_abstention_rate": 0.8, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 16, "categorical_total": 30, "categorical_accuracy": 0.5333333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5415574484847941, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 30, "score_mae": 0.9737614470393585, "score_rmse": 1.2148143494095016, "score_normalized_mae": 0.29860634894107785, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.2884666299180026, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "original": { "summary": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "None": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "noul": { "questions": 26, "labelled_questions": 0, "unlabeled_questions": 26, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "choice": { "questions": 20, "labelled_questions": 0, "unlabeled_questions": 20, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 24, "labelled_questions": 0, "unlabeled_questions": 24, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } } } }, "head_only": { "suites": { "final_test": { "summary": { "questions": 144, "labelled_questions": 144, "unlabeled_questions": 0, "categorical_correct": 51, "categorical_total": 126, "categorical_accuracy": 0.40476190476190477, "score_count": 18, "score_mae": 0.7239452033407159, "score_rmse": 0.866835996431051, "score_normalized_mae": 0.24131506778023862, "score_argmax_class_match_diagnostic": 7, "mean_ground_truth_probability": 0.3545439867173425, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 27, "tp": 11, "fp": 16, "fn": 13, "tn": 32, "precision": 0.4074074074074074, "recall": 0.4583333333333333, "f1": 0.43137254901960786, "false_completion_count": 16, "false_completion_denominator": 48, "false_completion_rate": 0.3333333333333333, "false_completion_known_negative_count": 9, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 7, "false_completion_unknown_denominator": 24, "unknown_correct": 9, "unknown_total": 24, "unknown_accuracy": 0.375 }, "uncertainty": { "correct": 9, "total": 28, "accuracy": 0.32142857142857145, "predicted_unknown": 23, "false_abstention_count": 14, "false_abstention_known_population": 56, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 24, "categorical_accuracy": 0.4583333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.3987615406513214, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 1, "tp": 1, "fp": 0, "fn": 3, "tn": 8, "precision": 1.0, "recall": 0.25, "f1": 0.4, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 1, "unknown_total": 4, "unknown_accuracy": 0.25 }, "uncertainty": { "correct": 1, "total": 4, "accuracy": 0.25, "predicted_unknown": 3, "false_abstention_count": 2, "false_abstention_known_population": 8, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 18, "categorical_accuracy": 0.3333333333333333, "score_count": 6, "score_mae": 0.6133753197888533, "score_rmse": 0.8152134241952144, "score_normalized_mae": 0.20445843992961776, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.3781159318362673, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 5, "tp": 2, "fp": 3, "fn": 2, "tn": 5, "precision": 0.4, "recall": 0.5, "f1": 0.4444444444444444, "false_completion_count": 3, "false_completion_denominator": 8, "false_completion_rate": 0.375, "false_completion_known_negative_count": 2, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 4, "unknown_correct": 1, "unknown_total": 4, "unknown_accuracy": 0.25 }, "uncertainty": { "correct": 1, "total": 4, "accuracy": 0.25, "predicted_unknown": 3, "false_abstention_count": 2, "false_abstention_known_population": 8, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 8, "categorical_total": 18, "categorical_accuracy": 0.4444444444444444, "score_count": 6, "score_mae": 0.7439198593298594, "score_rmse": 0.8470936067619846, "score_normalized_mae": 0.24797328644328645, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.32211995497345924, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 5, "tp": 2, "fp": 3, "fn": 2, "tn": 5, "precision": 0.4, "recall": 0.5, "f1": 0.4444444444444444, "false_completion_count": 3, "false_completion_denominator": 8, "false_completion_rate": 0.375, "false_completion_known_negative_count": 1, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 2, "false_completion_unknown_denominator": 4, "unknown_correct": 2, "unknown_total": 4, "unknown_accuracy": 0.5 }, "uncertainty": { "correct": 2, "total": 4, "accuracy": 0.5, "predicted_unknown": 4, "false_abstention_count": 2, "false_abstention_known_population": 8, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 24, "categorical_accuracy": 0.4583333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.39347876484195393, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 5, "tp": 2, "fp": 3, "fn": 2, "tn": 5, "precision": 0.4, "recall": 0.5, "f1": 0.4444444444444444, "false_completion_count": 3, "false_completion_denominator": 8, "false_completion_rate": 0.375, "false_completion_known_negative_count": 2, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 4, "unknown_correct": 1, "unknown_total": 4, "unknown_accuracy": 0.25 }, "uncertainty": { "correct": 1, "total": 4, "accuracy": 0.25, "predicted_unknown": 2, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 24, "categorical_accuracy": 0.375, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.32114054285921156, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 2, "fp": 2, "fn": 2, "tn": 6, "precision": 0.5, "recall": 0.5, "f1": 0.5, "false_completion_count": 2, "false_completion_denominator": 8, "false_completion_rate": 0.25, "false_completion_known_negative_count": 1, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 4, "unknown_correct": 2, "unknown_total": 4, "unknown_accuracy": 0.5 }, "uncertainty": { "correct": 2, "total": 8, "accuracy": 0.25, "predicted_unknown": 6, "false_abstention_count": 4, "false_abstention_known_population": 16, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 18, "categorical_accuracy": 0.3333333333333333, "score_count": 6, "score_mae": 0.8145404309034348, "score_rmse": 0.9338487182197799, "score_normalized_mae": 0.2715134769678116, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.3136471851418416, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 7, "tp": 2, "fp": 5, "fn": 2, "tn": 3, "precision": 0.2857142857142857, "recall": 0.5, "f1": 0.36363636363636365, "false_completion_count": 5, "false_completion_denominator": 8, "false_completion_rate": 0.625, "false_completion_known_negative_count": 3, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 2, "false_completion_unknown_denominator": 4, "unknown_correct": 2, "unknown_total": 4, "unknown_accuracy": 0.5 }, "uncertainty": { "correct": 2, "total": 4, "accuracy": 0.5, "predicted_unknown": 5, "false_abstention_count": 3, "false_abstention_known_population": 8, "false_abstention_rate": 0.375, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 108, "labelled_questions": 108, "unlabeled_questions": 0, "categorical_correct": 41, "categorical_total": 108, "categorical_accuracy": 0.37962962962962965, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.3389208158650608, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 27, "tp": 11, "fp": 16, "fn": 13, "tn": 32, "precision": 0.4074074074074074, "recall": 0.4583333333333333, "f1": 0.43137254901960786, "false_completion_count": 16, "false_completion_denominator": 48, "false_completion_rate": 0.3333333333333333, "false_completion_known_negative_count": 9, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 7, "false_completion_unknown_denominator": 24, "unknown_correct": 9, "unknown_total": 24, "unknown_accuracy": 0.375 }, "uncertainty": { "correct": 9, "total": 28, "accuracy": 0.32142857142857145, "predicted_unknown": 23, "false_abstention_count": 14, "false_abstention_known_population": 56, "false_abstention_rate": 0.25, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 10, "categorical_total": 18, "categorical_accuracy": 0.5555555555555556, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5306646360291375, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 18, "score_mae": 0.7239452033407159, "score_rmse": 0.866835996431051, "score_normalized_mae": 0.24131506778023862, "score_argmax_class_match_diagnostic": 7, "mean_ground_truth_probability": 0.27216236251923775, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "old_browser": { "summary": { "questions": 120, "labelled_questions": 120, "unlabeled_questions": 0, "categorical_correct": 42, "categorical_total": 90, "categorical_accuracy": 0.4666666666666667, "score_count": 30, "score_mae": 0.9138870733479659, "score_rmse": 1.1200267774029187, "score_normalized_mae": 0.2706026091674963, "score_argmax_class_match_diagnostic": 4, "mean_ground_truth_probability": 0.3668409484749039, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 14, "unknown_total": 14, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 18, "total": 18, "accuracy": 1.0, "predicted_unknown": 36, "false_abstention_count": 18, "false_abstention_known_population": 20, "false_abstention_rate": 0.9, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 3, "categorical_total": 15, "categorical_accuracy": 0.2, "score_count": 5, "score_mae": 0.7914798796176911, "score_rmse": 0.9174221091488283, "score_normalized_mae": 0.26382662653923034, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.24752127900719642, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 15, "categorical_accuracy": 0.7333333333333333, "score_count": 5, "score_mae": 0.7026939123868943, "score_rmse": 0.8924087482746634, "score_normalized_mae": 0.23423130412896476, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.47676235213875773, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 1, "uncertain_population": 2, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 1, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 2, "unknown_correct": 2, "unknown_total": 2, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 2, "total": 2, "accuracy": 1.0, "predicted_unknown": 3, "false_abstention_count": 1, "false_abstention_known_population": 2, "false_abstention_rate": 0.5, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 15, "categorical_accuracy": 0.6, "score_count": 5, "score_mae": 1.5311886876821519, "score_rmse": 1.7186105812631278, "score_normalized_mae": 0.30623773753643035, "score_argmax_class_match_diagnostic": 1, "mean_ground_truth_probability": 0.4152519032359123, "completion": { "claims": 10, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 7, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 9, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 9, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 7, "unknown_correct": 7, "unknown_total": 7, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 7, "total": 7, "accuracy": 1.0, "predicted_unknown": 10, "false_abstention_count": 3, "false_abstention_known_population": 3, "false_abstention_rate": 1.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 4, "categorical_total": 15, "categorical_accuracy": 0.26666666666666666, "score_count": 5, "score_mae": 0.7496379613876343, "score_rmse": 0.9489342693126716, "score_normalized_mae": 0.24987932046254474, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.2622296527028084, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 1, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 1, "unknown_correct": 1, "unknown_total": 1, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 1, "total": 1, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 3, "false_abstention_known_population": 3, "false_abstention_rate": 1.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 15, "categorical_accuracy": 0.4, "score_count": 5, "score_mae": 0.7382024854421616, "score_rmse": 0.9150833843041797, "score_normalized_mae": 0.24606749514738718, "score_argmax_class_match_diagnostic": 1, "mean_ground_truth_probability": 0.3350163586437702, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 10, "false_abstention_count": 6, "false_abstention_known_population": 6, "false_abstention_rate": 1.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 9, "categorical_total": 15, "categorical_accuracy": 0.6, "score_count": 5, "score_mae": 0.9701195135712624, "score_rmse": 1.0941786940486526, "score_normalized_mae": 0.32337317119042075, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.46426414512097836, "completion": { "claims": 10, "positive_population": 4, "known_negative_population": 2, "uncertain_population": 4, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 4, "tn": 6, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 6, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 9, "false_abstention_count": 5, "false_abstention_known_population": 6, "false_abstention_rate": 0.8333333333333334, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 60, "labelled_questions": 60, "unlabeled_questions": 0, "categorical_correct": 26, "categorical_total": 60, "categorical_accuracy": 0.43333333333333335, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.34402327512701353, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 14, "unknown_total": 14, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 18, "total": 18, "accuracy": 1.0, "predicted_unknown": 36, "false_abstention_count": 18, "false_abstention_known_population": 20, "false_abstention_rate": 0.9, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 16, "categorical_total": 30, "categorical_accuracy": 0.5333333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5211126377185186, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 30, "score_mae": 0.9138870733479659, "score_rmse": 1.1200267774029187, "score_normalized_mae": 0.2706026091674963, "score_argmax_class_match_diagnostic": 4, "mean_ground_truth_probability": 0.25820460592707, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "original": { "summary": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "None": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "noul": { "questions": 26, "labelled_questions": 0, "unlabeled_questions": 26, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "choice": { "questions": 20, "labelled_questions": 0, "unlabeled_questions": 20, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 24, "labelled_questions": 0, "unlabeled_questions": 24, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } } } }, "projector_head": { "suites": { "final_test": { "summary": { "questions": 144, "labelled_questions": 144, "unlabeled_questions": 0, "categorical_correct": 93, "categorical_total": 126, "categorical_accuracy": 0.7380952380952381, "score_count": 18, "score_mae": 0.5747373577824617, "score_rmse": 0.8099766704878334, "score_normalized_mae": 0.19157911926082055, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.7069218720946311, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 23, "tp": 22, "fp": 1, "fn": 2, "tn": 47, "precision": 0.9565217391304348, "recall": 0.9166666666666666, "f1": 0.9361702127659575, "false_completion_count": 1, "false_completion_denominator": 48, "false_completion_rate": 0.020833333333333332, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 24, "unknown_correct": 23, "unknown_total": 24, "unknown_accuracy": 0.9583333333333334 }, "uncertainty": { "correct": 23, "total": 28, "accuracy": 0.8214285714285714, "predicted_unknown": 28, "false_abstention_count": 5, "false_abstention_known_population": 56, "false_abstention_rate": 0.08928571428571429, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 18, "categorical_total": 24, "categorical_accuracy": 0.75, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7584458074561553, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 17, "categorical_total": 18, "categorical_accuracy": 0.9444444444444444, "score_count": 6, "score_mae": 0.2564091903814874, "score_rmse": 0.4489962048732849, "score_normalized_mae": 0.08546973012716247, "score_argmax_class_match_diagnostic": 4, "mean_ground_truth_probability": 0.8521811933411906, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 5, "tp": 4, "fp": 1, "fn": 0, "tn": 7, "precision": 0.8, "recall": 1.0, "f1": 0.8888888888888888, "false_completion_count": 1, "false_completion_denominator": 8, "false_completion_rate": 0.125, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 4, "unknown_correct": 3, "unknown_total": 4, "unknown_accuracy": 0.75 }, "uncertainty": { "correct": 3, "total": 4, "accuracy": 0.75, "predicted_unknown": 3, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 12, "categorical_total": 18, "categorical_accuracy": 0.6666666666666666, "score_count": 6, "score_mae": 0.8481783106837065, "score_rmse": 1.0681764697590814, "score_normalized_mae": 0.28272610356123545, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.5993321870145237, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 3, "tp": 3, "fp": 0, "fn": 1, "tn": 8, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 5, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 19, "categorical_total": 24, "categorical_accuracy": 0.7916666666666666, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7843273461351551, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 14, "categorical_total": 24, "categorical_accuracy": 0.5833333333333334, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.614936593557862, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 8, "accuracy": 0.5, "predicted_unknown": 7, "false_abstention_count": 3, "false_abstention_known_population": 16, "false_abstention_rate": 0.1875, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 18, "categorical_accuracy": 0.7222222222222222, "score_count": 6, "score_mae": 0.6196245722821914, "score_rmse": 0.7909412479230565, "score_normalized_mae": 0.20654152409406382, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.6323081050628995, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 3, "tp": 3, "fp": 0, "fn": 1, "tn": 8, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 5, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 108, "labelled_questions": 108, "unlabeled_questions": 0, "categorical_correct": 77, "categorical_total": 108, "categorical_accuracy": 0.7129629629629629, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7321095337876151, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 23, "tp": 22, "fp": 1, "fn": 2, "tn": 47, "precision": 0.9565217391304348, "recall": 0.9166666666666666, "f1": 0.9361702127659575, "false_completion_count": 1, "false_completion_denominator": 48, "false_completion_rate": 0.020833333333333332, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 24, "unknown_correct": 23, "unknown_total": 24, "unknown_accuracy": 0.9583333333333334 }, "uncertainty": { "correct": 23, "total": 28, "accuracy": 0.8214285714285714, "predicted_unknown": 28, "false_abstention_count": 5, "false_abstention_known_population": 56, "false_abstention_rate": 0.08928571428571429, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 16, "categorical_total": 18, "categorical_accuracy": 0.8888888888888888, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.8326162501344276, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 18, "score_mae": 0.5747373577824617, "score_rmse": 0.8099766704878334, "score_normalized_mae": 0.19157911926082055, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.4301015238969304, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "old_browser": { "summary": { "questions": 120, "labelled_questions": 120, "unlabeled_questions": 0, "categorical_correct": 52, "categorical_total": 90, "categorical_accuracy": 0.5777777777777777, "score_count": 30, "score_mae": 0.6056049749167762, "score_rmse": 0.8229793308861133, "score_normalized_mae": 0.18521513006069146, "score_argmax_class_match_diagnostic": 14, "mean_ground_truth_probability": 0.534484488642563, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 13, "unknown_total": 14, "unknown_accuracy": 0.9285714285714286 }, "uncertainty": { "correct": 15, "total": 18, "accuracy": 0.8333333333333334, "predicted_unknown": 28, "false_abstention_count": 13, "false_abstention_known_population": 20, "false_abstention_rate": 0.65, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 4, "categorical_total": 15, "categorical_accuracy": 0.26666666666666666, "score_count": 5, "score_mae": 0.6115967812922343, "score_rmse": 0.7963200171515148, "score_normalized_mae": 0.2038655937640781, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.30909750328440516, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 15, "categorical_accuracy": 0.7333333333333333, "score_count": 5, "score_mae": 0.3560560927833876, "score_rmse": 0.5613013441971785, "score_normalized_mae": 0.11868536426112922, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.6941640635314797, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 1, "uncertain_population": 2, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 1, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 2, "unknown_correct": 2, "unknown_total": 2, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 2, "total": 2, "accuracy": 1.0, "predicted_unknown": 3, "false_abstention_count": 1, "false_abstention_known_population": 2, "false_abstention_rate": 0.5, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 15, "categorical_accuracy": 0.8666666666666667, "score_count": 5, "score_mae": 0.7493937710205286, "score_rmse": 1.1841181446260023, "score_normalized_mae": 0.14987875420410574, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.7735454666806618, "completion": { "claims": 10, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 7, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 9, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 9, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 7, "unknown_correct": 7, "unknown_total": 7, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 7, "total": 7, "accuracy": 1.0, "predicted_unknown": 8, "false_abstention_count": 1, "false_abstention_known_population": 3, "false_abstention_rate": 0.3333333333333333, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 15, "categorical_accuracy": 0.4, "score_count": 5, "score_mae": 0.5581772183526859, "score_rmse": 0.6506613036691996, "score_normalized_mae": 0.18605907278422862, "score_argmax_class_match_diagnostic": 1, "mean_ground_truth_probability": 0.3517056047501194, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 1, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 1, "unknown_correct": 0, "unknown_total": 1, "unknown_accuracy": 0.0 }, "uncertainty": { "correct": 0, "total": 1, "accuracy": 0.0, "predicted_unknown": 2, "false_abstention_count": 2, "false_abstention_known_population": 3, "false_abstention_rate": 0.6666666666666666, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 7, "categorical_total": 15, "categorical_accuracy": 0.4666666666666667, "score_count": 5, "score_mae": 0.7582705806996273, "score_rmse": 0.8587852678332104, "score_normalized_mae": 0.25275686023320915, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.41066865629455834, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 2, "total": 4, "accuracy": 0.5, "predicted_unknown": 7, "false_abstention_count": 5, "false_abstention_known_population": 6, "false_abstention_rate": 0.8333333333333334, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 15, "categorical_accuracy": 0.7333333333333333, "score_count": 5, "score_mae": 0.6001354053521937, "score_rmse": 0.7426823391655362, "score_normalized_mae": 0.20004513511739788, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.6677256373141539, "completion": { "claims": 10, "positive_population": 4, "known_negative_population": 2, "uncertain_population": 4, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 4, "tn": 6, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 6, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 8, "false_abstention_count": 4, "false_abstention_known_population": 6, "false_abstention_rate": 0.6666666666666666, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 60, "labelled_questions": 60, "unlabeled_questions": 0, "categorical_correct": 32, "categorical_total": 60, "categorical_accuracy": 0.5333333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5288121277530039, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 13, "unknown_total": 14, "unknown_accuracy": 0.9285714285714286 }, "uncertainty": { "correct": 15, "total": 18, "accuracy": 0.8333333333333334, "predicted_unknown": 28, "false_abstention_count": 13, "false_abstention_known_population": 20, "false_abstention_rate": 0.65, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 20, "categorical_total": 30, "categorical_accuracy": 0.6666666666666666, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.6226229309037498, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 30, "score_mae": 0.6056049749167762, "score_rmse": 0.8229793308861133, "score_normalized_mae": 0.18521513006069146, "score_argmax_class_match_diagnostic": 14, "mean_ground_truth_probability": 0.4576907681604947, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "original": { "summary": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "None": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "noul": { "questions": 26, "labelled_questions": 0, "unlabeled_questions": 26, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "choice": { "questions": 20, "labelled_questions": 0, "unlabeled_questions": 20, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 24, "labelled_questions": 0, "unlabeled_questions": 24, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } } } }, "projector_head_text_lora": { "suites": { "final_test": { "summary": { "questions": 144, "labelled_questions": 144, "unlabeled_questions": 0, "categorical_correct": 92, "categorical_total": 126, "categorical_accuracy": 0.7301587301587301, "score_count": 18, "score_mae": 0.5742192765255577, "score_rmse": 0.776629787968387, "score_normalized_mae": 0.19140642550851922, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.6952334629110392, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 23, "tp": 22, "fp": 1, "fn": 2, "tn": 47, "precision": 0.9565217391304348, "recall": 0.9166666666666666, "f1": 0.9361702127659575, "false_completion_count": 1, "false_completion_denominator": 48, "false_completion_rate": 0.020833333333333332, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 24, "unknown_correct": 23, "unknown_total": 24, "unknown_accuracy": 0.9583333333333334 }, "uncertainty": { "correct": 23, "total": 28, "accuracy": 0.8214285714285714, "predicted_unknown": 28, "false_abstention_count": 5, "false_abstention_known_population": 56, "false_abstention_rate": 0.08928571428571429, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 19, "categorical_total": 24, "categorical_accuracy": 0.7916666666666666, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7670989430043846, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 15, "categorical_total": 18, "categorical_accuracy": 0.8333333333333334, "score_count": 6, "score_mae": 0.34072416225073443, "score_rmse": 0.5044211620017911, "score_normalized_mae": 0.11357472075024481, "score_argmax_class_match_diagnostic": 4, "mean_ground_truth_probability": 0.7950331290097287, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 5, "tp": 4, "fp": 1, "fn": 0, "tn": 7, "precision": 0.8, "recall": 1.0, "f1": 0.8888888888888888, "false_completion_count": 1, "false_completion_denominator": 8, "false_completion_rate": 0.125, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 4, "unknown_correct": 3, "unknown_total": 4, "unknown_accuracy": 0.75 }, "uncertainty": { "correct": 3, "total": 4, "accuracy": 0.75, "predicted_unknown": 3, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 18, "categorical_accuracy": 0.7222222222222222, "score_count": 6, "score_mae": 0.7106363351122127, "score_rmse": 0.935490113574171, "score_normalized_mae": 0.23687877837073756, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.6052360907698168, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 3, "tp": 3, "fp": 0, "fn": 1, "tn": 8, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 5, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 18, "categorical_total": 24, "categorical_accuracy": 0.75, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.751891012711471, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 4, "false_abstention_count": 0, "false_abstention_known_population": 8, "false_abstention_rate": 0.0, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 14, "categorical_total": 24, "categorical_accuracy": 0.5833333333333334, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.6268679489676288, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 4, "tp": 4, "fp": 0, "fn": 0, "tn": 8, "precision": 1.0, "recall": 1.0, "f1": 1.0, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 8, "accuracy": 0.5, "predicted_unknown": 7, "false_abstention_count": 3, "false_abstention_known_population": 16, "false_abstention_rate": 0.1875, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 24, "labelled_questions": 24, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 18, "categorical_accuracy": 0.7222222222222222, "score_count": 6, "score_mae": 0.6712973322137259, "score_rmse": 0.8245477678153117, "score_normalized_mae": 0.2237657774045753, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.6252736530032053, "completion": { "claims": 12, "positive_population": 4, "known_negative_population": 4, "uncertain_population": 4, "predicted_completed": 3, "tp": 3, "fp": 0, "fn": 1, "tn": 8, "precision": 1.0, "recall": 0.75, "f1": 0.8571428571428571, "false_completion_count": 0, "false_completion_denominator": 8, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 4, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 5, "false_abstention_count": 1, "false_abstention_known_population": 8, "false_abstention_rate": 0.125, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 108, "labelled_questions": 108, "unlabeled_questions": 0, "categorical_correct": 79, "categorical_total": 108, "categorical_accuracy": 0.7314814814814815, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7343110651039089, "completion": { "claims": 72, "positive_population": 24, "known_negative_population": 24, "uncertain_population": 24, "predicted_completed": 23, "tp": 22, "fp": 1, "fn": 2, "tn": 47, "precision": 0.9565217391304348, "recall": 0.9166666666666666, "f1": 0.9361702127659575, "false_completion_count": 1, "false_completion_denominator": 48, "false_completion_rate": 0.020833333333333332, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 24, "false_completion_unknown_count": 1, "false_completion_unknown_denominator": 24, "unknown_correct": 23, "unknown_total": 24, "unknown_accuracy": 0.9583333333333334 }, "uncertainty": { "correct": 23, "total": 28, "accuracy": 0.8214285714285714, "predicted_unknown": 28, "false_abstention_count": 5, "false_abstention_known_population": 56, "false_abstention_rate": 0.08928571428571429, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 18, "categorical_accuracy": 0.7222222222222222, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.7327897010565922, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 18, "labelled_questions": 18, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 18, "score_mae": 0.5742192765255577, "score_rmse": 0.776629787968387, "score_normalized_mae": 0.19140642550851922, "score_argmax_class_match_diagnostic": 8, "mean_ground_truth_probability": 0.42321161160826803, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "old_browser": { "summary": { "questions": 120, "labelled_questions": 120, "unlabeled_questions": 0, "categorical_correct": 52, "categorical_total": 90, "categorical_accuracy": 0.5777777777777777, "score_count": 30, "score_mae": 0.5702236743444072, "score_rmse": 0.7683694704302426, "score_normalized_mae": 0.17645176749157868, "score_argmax_class_match_diagnostic": 13, "mean_ground_truth_probability": 0.5449975172530416, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 13, "unknown_total": 14, "unknown_accuracy": 0.9285714285714286 }, "uncertainty": { "correct": 15, "total": 18, "accuracy": 0.8333333333333334, "predicted_unknown": 28, "false_abstention_count": 13, "false_abstention_known_population": 20, "false_abstention_rate": 0.65, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "form_state": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 4, "categorical_total": 15, "categorical_accuracy": 0.26666666666666666, "score_count": 5, "score_mae": 0.5769111174213686, "score_rmse": 0.7758909622744338, "score_normalized_mae": 0.1923037058071229, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.3323688083056328, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "modal_overlay": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 15, "categorical_accuracy": 0.7333333333333333, "score_count": 5, "score_mae": 0.3393246058945806, "score_rmse": 0.5131055533685425, "score_normalized_mae": 0.1131082019648602, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.6826594555877818, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 1, "uncertain_population": 2, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 1, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 2, "unknown_correct": 2, "unknown_total": 2, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 2, "total": 2, "accuracy": 1.0, "predicted_unknown": 3, "false_abstention_count": 1, "false_abstention_known_population": 2, "false_abstention_rate": 0.5, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "loading": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 13, "categorical_total": 15, "categorical_accuracy": 0.8666666666666667, "score_count": 5, "score_mae": 0.6130255780450711, "score_rmse": 0.9600317894366309, "score_normalized_mae": 0.12260511560901421, "score_argmax_class_match_diagnostic": 3, "mean_ground_truth_probability": 0.7702819129626732, "completion": { "claims": 10, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 7, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 9, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 9, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 7, "unknown_correct": 7, "unknown_total": 7, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 7, "total": 7, "accuracy": 1.0, "predicted_unknown": 8, "false_abstention_count": 1, "false_abstention_known_population": 3, "false_abstention_rate": 0.3333333333333333, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "visible_errors": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 6, "categorical_total": 15, "categorical_accuracy": 0.4, "score_count": 5, "score_mae": 0.5330597519979046, "score_rmse": 0.6740108013904127, "score_normalized_mae": 0.17768658399930154, "score_argmax_class_match_diagnostic": 1, "mean_ground_truth_probability": 0.3802962523170349, "completion": { "claims": 4, "positive_population": 1, "known_negative_population": 2, "uncertain_population": 1, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 1, "tn": 3, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 3, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 1, "unknown_correct": 0, "unknown_total": 1, "unknown_accuracy": 0.0 }, "uncertainty": { "correct": 0, "total": 1, "accuracy": 0.0, "predicted_unknown": 2, "false_abstention_count": 2, "false_abstention_known_population": 3, "false_abstention_rate": 0.6666666666666666, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "target_selection": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 7, "categorical_total": 15, "categorical_accuracy": 0.4666666666666667, "score_count": 5, "score_mae": 0.7270940032965882, "score_rmse": 0.8232650869312949, "score_normalized_mae": 0.24236466776552942, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.43888114450855936, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 2, "total": 4, "accuracy": 0.5, "predicted_unknown": 7, "false_abstention_count": 5, "false_abstention_known_population": 6, "false_abstention_rate": 0.8333333333333334, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "completion_evidence": { "questions": 20, "labelled_questions": 20, "unlabeled_questions": 0, "categorical_correct": 11, "categorical_total": 15, "categorical_accuracy": 0.7333333333333333, "score_count": 5, "score_mae": 0.6319269894109312, "score_rmse": 0.7895243908954838, "score_normalized_mae": 0.21064232980364372, "score_argmax_class_match_diagnostic": 2, "mean_ground_truth_probability": 0.6654975298365675, "completion": { "claims": 10, "positive_population": 4, "known_negative_population": 2, "uncertain_population": 4, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 4, "tn": 6, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 6, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 2, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 4, "unknown_correct": 4, "unknown_total": 4, "unknown_accuracy": 1.0 }, "uncertainty": { "correct": 4, "total": 4, "accuracy": 1.0, "predicted_unknown": 8, "false_abstention_count": 4, "false_abstention_known_population": 6, "false_abstention_rate": 0.6666666666666666, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "choice": { "questions": 60, "labelled_questions": 60, "unlabeled_questions": 0, "categorical_correct": 32, "categorical_total": 60, "categorical_accuracy": 0.5333333333333333, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.5432760533689989, "completion": { "claims": 28, "positive_population": 7, "known_negative_population": 7, "uncertain_population": 14, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 7, "tn": 21, "precision": null, "recall": 0.0, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 21, "false_completion_rate": 0.0, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 7, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 14, "unknown_correct": 13, "unknown_total": 14, "unknown_accuracy": 0.9285714285714286 }, "uncertainty": { "correct": 15, "total": 18, "accuracy": 0.8333333333333334, "predicted_unknown": 28, "false_abstention_count": 13, "false_abstention_known_population": 20, "false_abstention_rate": 0.65, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "noul": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 20, "categorical_total": 30, "categorical_accuracy": 0.6666666666666666, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": 0.6197544339866227, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 30, "labelled_questions": 30, "unlabeled_questions": 0, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 30, "score_mae": 0.5702236743444072, "score_rmse": 0.7683694704302426, "score_normalized_mae": 0.17645176749157868, "score_argmax_class_match_diagnostic": 13, "mean_ground_truth_probability": 0.47368352828754595, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } }, "original": { "summary": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "by_category": { "None": { "questions": 70, "labelled_questions": 0, "unlabeled_questions": 70, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } }, "by_question_type": { "noul": { "questions": 26, "labelled_questions": 0, "unlabeled_questions": 26, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "choice": { "questions": 20, "labelled_questions": 0, "unlabeled_questions": 20, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } }, "score": { "questions": 24, "labelled_questions": 0, "unlabeled_questions": 24, "categorical_correct": 0, "categorical_total": 0, "categorical_accuracy": null, "score_count": 0, "score_mae": null, "score_rmse": null, "score_normalized_mae": null, "score_argmax_class_match_diagnostic": 0, "mean_ground_truth_probability": null, "completion": { "claims": 0, "positive_population": 0, "known_negative_population": 0, "uncertain_population": 0, "predicted_completed": 0, "tp": 0, "fp": 0, "fn": 0, "tn": 0, "precision": null, "recall": null, "f1": 0.0, "false_completion_count": 0, "false_completion_denominator": 0, "false_completion_rate": null, "false_completion_known_negative_count": 0, "false_completion_known_negative_denominator": 0, "false_completion_unknown_count": 0, "false_completion_unknown_denominator": 0, "unknown_correct": 0, "unknown_total": 0, "unknown_accuracy": null }, "uncertainty": { "correct": 0, "total": 0, "accuracy": null, "predicted_unknown": 0, "false_abstention_count": 0, "false_abstention_known_population": 0, "false_abstention_rate": null, "missing_outputs": 0, "scope": "Unknown is a semantic public option, not a missing/invalid model output." } } } } } } }, "limitations": [ "Finite authored synthetic layouts, low-level widgets, domain language and semantic colors are shared; not arbitrary real-web or independent external-browser quality.", "Zero matched auxiliary viewport/class MI is not all-nuisance independence: Ready-ID versus completion has empirical MI 0.62255625 bits in 16 sparse Ready-target examples; it is not a deterministic completion-class mapping.", "Completion family-plus-option-position retains joint association despite balanced marginal position/viewport: train=0.584962500721156 bits, validation=0.5849625007211562 bits, final_test=0.5849625007211562 bits. No causal attribution is established.", "Three binary families have three scenarios per viewport, hence 18 paired questions cannot have exactly 9/9 public labels; closest balanced counts are 8/10. Actual raw census is retained in the QC proof.", "All performed candidates and all paired outcomes are retained. The locked validation export choice is not reselected from held-out results.", "The original 70 questions are unlabeled drift comparisons, not invented accuracy or improvement labels.", "Undefined precision means no predicted completed output; zero false-completion counts alone do not prove reliable positive detection.", "Continuous score MAE/RMSE use expected level; score argmax-class match is explicitly diagnostic only.", "Current-run GPU logs/gradients are sparse runtime measurements, not historical or instantaneous telemetry. Independent tensor/gradient checks are attributed only to their explicitly bound audit artifact." ] }