| { |
| "best_epoch": 15, |
| "best_tau": 0.9, |
| "best_temp": 0.5, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "best_delta_full": 0.2109050750732422, |
| "tune": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 5657, |
| "n_failures": 384, |
| "failure_rate": 6.788050203287963, |
| "rerank_count": 25, |
| "rerank_rate": 0.44193035177656, |
| "q0_miou": 86.46936416625977, |
| "oracle_miou": 89.69137072563171, |
| "selected_miou": 86.5086019039154, |
| "delta_full": 0.03923773765563965, |
| "q0_miou_fail": 32.901766896247864, |
| "oracle_miou_fail": 80.36765456199646, |
| "selected_miou_fail": 34.72617566585541, |
| "delta_fail": 1.824408769607544, |
| "gap_closed": 3.8436208808626957, |
| "auc_fail": 0.7775509276819015, |
| "gate_acc": 92.29273200035095, |
| "gate_prec": 41.82389937106918, |
| "gate_rec": 34.63541666666667, |
| "gate_tp": 133, |
| "gate_fp": 185, |
| "gate_tn": 5088, |
| "gate_fn": 251, |
| "decision_acc": 93.4063971042633, |
| "decision_prec": 72.0, |
| "decision_rec": 4.6875, |
| "decision_fp_rate": 0.13275175421960933, |
| "decision_tp": 18, |
| "decision_fp": 7, |
| "decision_tn": 5266, |
| "decision_fn": 366, |
| "oracle_acc_on_reranked_failures": 44.44444477558136, |
| "true_gain_on_reranked_failures": 38.9208197593689, |
| "n_false_positive_reranks": 7, |
| "fp_rerank_harm": -68.36986541748047, |
| "oracle_accuracy_overall": 93.21194887161255 |
| }, |
| "splits": { |
| "testA_refcoco_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 5657, |
| "n_failures": 384, |
| "failure_rate": 6.788050203287963, |
| "rerank_count": 25, |
| "rerank_rate": 0.44193035177656, |
| "q0_miou": 86.46936416625977, |
| "oracle_miou": 89.69137072563171, |
| "selected_miou": 86.5086019039154, |
| "delta_full": 0.03923773765563965, |
| "q0_miou_fail": 32.901766896247864, |
| "oracle_miou_fail": 80.36765456199646, |
| "selected_miou_fail": 34.72617566585541, |
| "delta_fail": 1.824408769607544, |
| "gap_closed": 3.8436208808626957, |
| "auc_fail": 0.7775509276819015, |
| "gate_acc": 92.29273200035095, |
| "gate_prec": 41.82389937106918, |
| "gate_rec": 34.63541666666667, |
| "gate_tp": 133, |
| "gate_fp": 185, |
| "gate_tn": 5088, |
| "gate_fn": 251, |
| "decision_acc": 93.4063971042633, |
| "decision_prec": 72.0, |
| "decision_rec": 4.6875, |
| "decision_fp_rate": 0.13275175421960933, |
| "decision_tp": 18, |
| "decision_fp": 7, |
| "decision_tn": 5266, |
| "decision_fn": 366, |
| "oracle_acc_on_reranked_failures": 44.44444477558136, |
| "true_gain_on_reranked_failures": 38.9208197593689, |
| "n_false_positive_reranks": 7, |
| "fp_rerank_harm": -68.36986541748047, |
| "oracle_accuracy_overall": 93.21194887161255 |
| }, |
| "testA_refcocoplus_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 5726, |
| "n_failures": 574, |
| "failure_rate": 10.024449877750612, |
| "rerank_count": 34, |
| "rerank_rate": 0.5937827453719874, |
| "q0_miou": 83.376145362854, |
| "oracle_miou": 89.49434757232666, |
| "selected_miou": 83.3836317062378, |
| "delta_full": 0.0074863433837890625, |
| "q0_miou_fail": 19.878679513931274, |
| "oracle_miou_fail": 80.9114396572113, |
| "selected_miou_fail": 21.422217786312103, |
| "delta_fail": 1.5435382723808289, |
| "gap_closed": 2.5290323897481133, |
| "auc_fail": 0.8122211934879997, |
| "gate_acc": 89.62626457214355, |
| "gate_prec": 48.175182481751825, |
| "gate_rec": 45.99303135888502, |
| "gate_tp": 264, |
| "gate_fp": 284, |
| "gate_tn": 4868, |
| "gate_fn": 310, |
| "decision_acc": 90.15018939971924, |
| "decision_prec": 64.70588235294117, |
| "decision_rec": 3.8327526132404177, |
| "decision_fp_rate": 0.2329192546583851, |
| "decision_tp": 22, |
| "decision_fp": 12, |
| "decision_tn": 5140, |
| "decision_fn": 552, |
| "oracle_acc_on_reranked_failures": 54.54545617103577, |
| "true_gain_on_reranked_failures": 40.27233123779297, |
| "n_false_positive_reranks": 12, |
| "fp_rerank_harm": -70.25737762451172, |
| "oracle_accuracy_overall": 89.97554779052734 |
| }, |
| "testB_refcoco_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 5095, |
| "n_failures": 590, |
| "failure_rate": 11.579980372914623, |
| "rerank_count": 39, |
| "rerank_rate": 0.7654563297350344, |
| "q0_miou": 84.56805944442749, |
| "oracle_miou": 89.69974517822266, |
| "selected_miou": 84.47709083557129, |
| "delta_full": -0.09096860885620117, |
| "q0_miou_fail": 32.04871118068695, |
| "oracle_miou_fail": 76.36381387710571, |
| "selected_miou_fail": 32.97693431377411, |
| "delta_fail": 0.9282231330871582, |
| "gap_closed": 2.094597725398413, |
| "auc_fail": 0.8125561052690984, |
| "gate_acc": 87.4779224395752, |
| "gate_prec": 46.49122807017544, |
| "gate_rec": 53.898305084745765, |
| "gate_tp": 318, |
| "gate_fp": 366, |
| "gate_tn": 4139, |
| "gate_fn": 272, |
| "decision_acc": 88.635915517807, |
| "decision_prec": 64.1025641025641, |
| "decision_rec": 4.23728813559322, |
| "decision_fp_rate": 0.31076581576026635, |
| "decision_tp": 25, |
| "decision_fp": 14, |
| "decision_tn": 4491, |
| "decision_fn": 565, |
| "oracle_acc_on_reranked_failures": 31.999999284744263, |
| "true_gain_on_reranked_failures": 21.906165778636932, |
| "n_false_positive_reranks": 14, |
| "fp_rerank_harm": -72.22229838371277, |
| "oracle_accuracy_overall": 88.42002153396606 |
| }, |
| "testB_refcocoplus_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 4889, |
| "n_failures": 867, |
| "failure_rate": 17.73368787073021, |
| "rerank_count": 67, |
| "rerank_rate": 1.3704233994681938, |
| "q0_miou": 78.63370776176453, |
| "oracle_miou": 89.7470235824585, |
| "selected_miou": 78.84461283683777, |
| "delta_full": 0.2109050750732422, |
| "q0_miou_fail": 17.938430607318878, |
| "oracle_miou_fail": 80.60628771781921, |
| "selected_miou_fail": 20.129036903381348, |
| "delta_fail": 2.1906062960624695, |
| "gap_closed": 3.495581941153404, |
| "auc_fail": 0.8008149525934924, |
| "gate_acc": 81.3458800315857, |
| "gate_prec": 47.75224775224775, |
| "gate_rec": 55.13264129181085, |
| "gate_tp": 478, |
| "gate_fp": 523, |
| "gate_tn": 3499, |
| "gate_fn": 389, |
| "decision_acc": 82.98220634460449, |
| "decision_prec": 76.11940298507463, |
| "decision_rec": 5.88235294117647, |
| "decision_fp_rate": 0.39781203381402286, |
| "decision_tp": 51, |
| "decision_fp": 16, |
| "decision_tn": 4006, |
| "decision_fn": 816, |
| "oracle_acc_on_reranked_failures": 50.98039507865906, |
| "true_gain_on_reranked_failures": 37.24033236503601, |
| "n_false_positive_reranks": 16, |
| "fp_rerank_harm": -54.256606101989746, |
| "oracle_accuracy_overall": 82.26631283760071 |
| }, |
| "test_refcocog_umd": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 9602, |
| "n_failures": 1346, |
| "failure_rate": 14.01791293480525, |
| "rerank_count": 71, |
| "rerank_rate": 0.7394292855655072, |
| "q0_miou": 81.1055064201355, |
| "oracle_miou": 87.99707889556885, |
| "selected_miou": 81.12919926643372, |
| "delta_full": 0.023692846298217773, |
| "q0_miou_fail": 27.811765670776367, |
| "oracle_miou_fail": 76.97429060935974, |
| "selected_miou_fail": 28.97375226020813, |
| "delta_fail": 1.1619865894317627, |
| "gap_closed": 2.363561657753304, |
| "auc_fail": 0.7782235190112536, |
| "gate_acc": 85.66965460777283, |
| "gate_prec": 48.62637362637363, |
| "gate_rec": 39.45022288261516, |
| "gate_tp": 531, |
| "gate_fp": 561, |
| "gate_tn": 7695, |
| "gate_fn": 815, |
| "decision_acc": 86.28410696983337, |
| "decision_prec": 70.4225352112676, |
| "decision_rec": 3.7147102526002973, |
| "decision_fp_rate": 0.2543604651162791, |
| "decision_tp": 50, |
| "decision_fp": 21, |
| "decision_tn": 8235, |
| "decision_fn": 1296, |
| "oracle_acc_on_reranked_failures": 40.00000059604645, |
| "true_gain_on_reranked_failures": 31.280744075775146, |
| "n_false_positive_reranks": 21, |
| "fp_rerank_harm": -63.64555358886719, |
| "oracle_accuracy_overall": 85.98208427429199 |
| }, |
| "val_refcoco_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 10834, |
| "n_failures": 937, |
| "failure_rate": 8.648698541628209, |
| "rerank_count": 59, |
| "rerank_rate": 0.5445818718848071, |
| "q0_miou": 85.45660972595215, |
| "oracle_miou": 89.42838907241821, |
| "selected_miou": 85.45025587081909, |
| "delta_full": -0.006353855133056641, |
| "q0_miou_fail": 32.160285115242004, |
| "oracle_miou_fail": 78.08363437652588, |
| "selected_miou_fail": 33.016687631607056, |
| "delta_fail": 0.8564025163650513, |
| "gap_closed": 1.8648520418066497, |
| "auc_fail": 0.8085167837046013, |
| "gate_acc": 90.28059840202332, |
| "gate_prec": 43.722943722943725, |
| "gate_rec": 43.11632870864461, |
| "gate_tp": 404, |
| "gate_fp": 520, |
| "gate_tn": 9377, |
| "gate_fn": 533, |
| "decision_acc": 91.61897897720337, |
| "decision_prec": 74.57627118644068, |
| "decision_rec": 4.695837780149413, |
| "decision_fp_rate": 0.1515610791148833, |
| "decision_tp": 44, |
| "decision_fp": 15, |
| "decision_tn": 9882, |
| "decision_fn": 893, |
| "oracle_acc_on_reranked_failures": 34.090909361839294, |
| "true_gain_on_reranked_failures": 18.237465620040894, |
| "n_false_positive_reranks": 15, |
| "fp_rerank_harm": -58.08122754096985, |
| "oracle_accuracy_overall": 91.35130047798157 |
| }, |
| "val_refcocog_umd": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 4896, |
| "n_failures": 758, |
| "failure_rate": 15.48202614379085, |
| "rerank_count": 35, |
| "rerank_rate": 0.7148692810457516, |
| "q0_miou": 79.804128408432, |
| "oracle_miou": 87.27956414222717, |
| "selected_miou": 79.77062463760376, |
| "delta_full": -0.03350377082824707, |
| "q0_miou_fail": 26.25426948070526, |
| "oracle_miou_fail": 74.53891634941101, |
| "selected_miou_fail": 27.04944908618927, |
| "delta_fail": 0.7951796054840088, |
| "gap_closed": 1.6468580740504093, |
| "auc_fail": 0.7836166121065967, |
| "gate_acc": 84.84477400779724, |
| "gate_prec": 51.34680134680135, |
| "gate_rec": 40.23746701846966, |
| "gate_tp": 305, |
| "gate_fp": 289, |
| "gate_tn": 3849, |
| "gate_fn": 453, |
| "decision_acc": 84.82434749603271, |
| "decision_prec": 71.42857142857143, |
| "decision_rec": 3.2981530343007917, |
| "decision_fp_rate": 0.2416626389560174, |
| "decision_tp": 25, |
| "decision_fp": 10, |
| "decision_tn": 4128, |
| "decision_fn": 733, |
| "oracle_acc_on_reranked_failures": 43.99999976158142, |
| "true_gain_on_reranked_failures": 24.10985380411148, |
| "n_false_positive_reranks": 10, |
| "fp_rerank_harm": -76.68079137802124, |
| "oracle_accuracy_overall": 84.51797366142273 |
| }, |
| "val_refcocoplus_unc": { |
| "tau": 0.9, |
| "margin_pred": 0.0, |
| "margin_gap": 0.0, |
| "gate_temperature": 0.5, |
| "n_samples": 10758, |
| "n_failures": 1419, |
| "failure_rate": 13.190184049079754, |
| "rerank_count": 110, |
| "rerank_rate": 1.0224948875255624, |
| "q0_miou": 81.12329840660095, |
| "oracle_miou": 89.28661346435547, |
| "selected_miou": 81.11199140548706, |
| "delta_full": -0.011307001113891602, |
| "q0_miou_fail": 18.030162155628204, |
| "oracle_miou_fail": 79.91952896118164, |
| "selected_miou_fail": 19.942304491996765, |
| "delta_fail": 1.9121423363685608, |
| "gap_closed": 3.0896136688168236, |
| "auc_fail": 0.8240500085986754, |
| "gate_acc": 86.73545122146606, |
| "gate_prec": 49.72375690607735, |
| "gate_rec": 50.7399577167019, |
| "gate_tp": 720, |
| "gate_fp": 728, |
| "gate_tn": 8611, |
| "gate_fn": 699, |
| "decision_acc": 87.08868026733398, |
| "decision_prec": 63.63636363636363, |
| "decision_rec": 4.933051444679352, |
| "decision_fp_rate": 0.4283113823749866, |
| "decision_tp": 70, |
| "decision_fp": 40, |
| "decision_tn": 9299, |
| "decision_fn": 1349, |
| "oracle_acc_on_reranked_failures": 48.571428656578064, |
| "true_gain_on_reranked_failures": 38.76187205314636, |
| "n_false_positive_reranks": 40, |
| "fp_rerank_harm": -70.87111473083496, |
| "oracle_accuracy_overall": 86.80981397628784 |
| } |
| } |
| } |
