{
  "best_epoch": 15,
  "best_tau": 0.9,
  "best_temp": 0.5,
  "margin_pred": 0.0,
  "margin_gap": 0.0,
  "best_delta_full": 0.2109050750732422,
  "tune": {
    "tau": 0.9,
    "margin_pred": 0.0,
    "margin_gap": 0.0,
    "gate_temperature": 0.5,
    "n_samples": 5657,
    "n_failures": 384,
    "failure_rate": 6.788050203287963,
    "rerank_count": 25,
    "rerank_rate": 0.44193035177656,
    "q0_miou": 86.46936416625977,
    "oracle_miou": 89.69137072563171,
    "selected_miou": 86.5086019039154,
    "delta_full": 0.03923773765563965,
    "q0_miou_fail": 32.901766896247864,
    "oracle_miou_fail": 80.36765456199646,
    "selected_miou_fail": 34.72617566585541,
    "delta_fail": 1.824408769607544,
    "gap_closed": 3.8436208808626957,
    "auc_fail": 0.7775509276819015,
    "gate_acc": 92.29273200035095,
    "gate_prec": 41.82389937106918,
    "gate_rec": 34.63541666666667,
    "gate_tp": 133,
    "gate_fp": 185,
    "gate_tn": 5088,
    "gate_fn": 251,
    "decision_acc": 93.4063971042633,
    "decision_prec": 72.0,
    "decision_rec": 4.6875,
    "decision_fp_rate": 0.13275175421960933,
    "decision_tp": 18,
    "decision_fp": 7,
    "decision_tn": 5266,
    "decision_fn": 366,
    "oracle_acc_on_reranked_failures": 44.44444477558136,
    "true_gain_on_reranked_failures": 38.9208197593689,
    "n_false_positive_reranks": 7,
    "fp_rerank_harm": -68.36986541748047,
    "oracle_accuracy_overall": 93.21194887161255
  },
  "splits": {
    "testA_refcoco_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 5657,
      "n_failures": 384,
      "failure_rate": 6.788050203287963,
      "rerank_count": 25,
      "rerank_rate": 0.44193035177656,
      "q0_miou": 86.46936416625977,
      "oracle_miou": 89.69137072563171,
      "selected_miou": 86.5086019039154,
      "delta_full": 0.03923773765563965,
      "q0_miou_fail": 32.901766896247864,
      "oracle_miou_fail": 80.36765456199646,
      "selected_miou_fail": 34.72617566585541,
      "delta_fail": 1.824408769607544,
      "gap_closed": 3.8436208808626957,
      "auc_fail": 0.7775509276819015,
      "gate_acc": 92.29273200035095,
      "gate_prec": 41.82389937106918,
      "gate_rec": 34.63541666666667,
      "gate_tp": 133,
      "gate_fp": 185,
      "gate_tn": 5088,
      "gate_fn": 251,
      "decision_acc": 93.4063971042633,
      "decision_prec": 72.0,
      "decision_rec": 4.6875,
      "decision_fp_rate": 0.13275175421960933,
      "decision_tp": 18,
      "decision_fp": 7,
      "decision_tn": 5266,
      "decision_fn": 366,
      "oracle_acc_on_reranked_failures": 44.44444477558136,
      "true_gain_on_reranked_failures": 38.9208197593689,
      "n_false_positive_reranks": 7,
      "fp_rerank_harm": -68.36986541748047,
      "oracle_accuracy_overall": 93.21194887161255
    },
    "testA_refcocoplus_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 5726,
      "n_failures": 574,
      "failure_rate": 10.024449877750612,
      "rerank_count": 34,
      "rerank_rate": 0.5937827453719874,
      "q0_miou": 83.376145362854,
      "oracle_miou": 89.49434757232666,
      "selected_miou": 83.3836317062378,
      "delta_full": 0.0074863433837890625,
      "q0_miou_fail": 19.878679513931274,
      "oracle_miou_fail": 80.9114396572113,
      "selected_miou_fail": 21.422217786312103,
      "delta_fail": 1.5435382723808289,
      "gap_closed": 2.5290323897481133,
      "auc_fail": 0.8122211934879997,
      "gate_acc": 89.62626457214355,
      "gate_prec": 48.175182481751825,
      "gate_rec": 45.99303135888502,
      "gate_tp": 264,
      "gate_fp": 284,
      "gate_tn": 4868,
      "gate_fn": 310,
      "decision_acc": 90.15018939971924,
      "decision_prec": 64.70588235294117,
      "decision_rec": 3.8327526132404177,
      "decision_fp_rate": 0.2329192546583851,
      "decision_tp": 22,
      "decision_fp": 12,
      "decision_tn": 5140,
      "decision_fn": 552,
      "oracle_acc_on_reranked_failures": 54.54545617103577,
      "true_gain_on_reranked_failures": 40.27233123779297,
      "n_false_positive_reranks": 12,
      "fp_rerank_harm": -70.25737762451172,
      "oracle_accuracy_overall": 89.97554779052734
    },
    "testB_refcoco_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 5095,
      "n_failures": 590,
      "failure_rate": 11.579980372914623,
      "rerank_count": 39,
      "rerank_rate": 0.7654563297350344,
      "q0_miou": 84.56805944442749,
      "oracle_miou": 89.69974517822266,
      "selected_miou": 84.47709083557129,
      "delta_full": -0.09096860885620117,
      "q0_miou_fail": 32.04871118068695,
      "oracle_miou_fail": 76.36381387710571,
      "selected_miou_fail": 32.97693431377411,
      "delta_fail": 0.9282231330871582,
      "gap_closed": 2.094597725398413,
      "auc_fail": 0.8125561052690984,
      "gate_acc": 87.4779224395752,
      "gate_prec": 46.49122807017544,
      "gate_rec": 53.898305084745765,
      "gate_tp": 318,
      "gate_fp": 366,
      "gate_tn": 4139,
      "gate_fn": 272,
      "decision_acc": 88.635915517807,
      "decision_prec": 64.1025641025641,
      "decision_rec": 4.23728813559322,
      "decision_fp_rate": 0.31076581576026635,
      "decision_tp": 25,
      "decision_fp": 14,
      "decision_tn": 4491,
      "decision_fn": 565,
      "oracle_acc_on_reranked_failures": 31.999999284744263,
      "true_gain_on_reranked_failures": 21.906165778636932,
      "n_false_positive_reranks": 14,
      "fp_rerank_harm": -72.22229838371277,
      "oracle_accuracy_overall": 88.42002153396606
    },
    "testB_refcocoplus_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 4889,
      "n_failures": 867,
      "failure_rate": 17.73368787073021,
      "rerank_count": 67,
      "rerank_rate": 1.3704233994681938,
      "q0_miou": 78.63370776176453,
      "oracle_miou": 89.7470235824585,
      "selected_miou": 78.84461283683777,
      "delta_full": 0.2109050750732422,
      "q0_miou_fail": 17.938430607318878,
      "oracle_miou_fail": 80.60628771781921,
      "selected_miou_fail": 20.129036903381348,
      "delta_fail": 2.1906062960624695,
      "gap_closed": 3.495581941153404,
      "auc_fail": 0.8008149525934924,
      "gate_acc": 81.3458800315857,
      "gate_prec": 47.75224775224775,
      "gate_rec": 55.13264129181085,
      "gate_tp": 478,
      "gate_fp": 523,
      "gate_tn": 3499,
      "gate_fn": 389,
      "decision_acc": 82.98220634460449,
      "decision_prec": 76.11940298507463,
      "decision_rec": 5.88235294117647,
      "decision_fp_rate": 0.39781203381402286,
      "decision_tp": 51,
      "decision_fp": 16,
      "decision_tn": 4006,
      "decision_fn": 816,
      "oracle_acc_on_reranked_failures": 50.98039507865906,
      "true_gain_on_reranked_failures": 37.24033236503601,
      "n_false_positive_reranks": 16,
      "fp_rerank_harm": -54.256606101989746,
      "oracle_accuracy_overall": 82.26631283760071
    },
    "test_refcocog_umd": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 9602,
      "n_failures": 1346,
      "failure_rate": 14.01791293480525,
      "rerank_count": 71,
      "rerank_rate": 0.7394292855655072,
      "q0_miou": 81.1055064201355,
      "oracle_miou": 87.99707889556885,
      "selected_miou": 81.12919926643372,
      "delta_full": 0.023692846298217773,
      "q0_miou_fail": 27.811765670776367,
      "oracle_miou_fail": 76.97429060935974,
      "selected_miou_fail": 28.97375226020813,
      "delta_fail": 1.1619865894317627,
      "gap_closed": 2.363561657753304,
      "auc_fail": 0.7782235190112536,
      "gate_acc": 85.66965460777283,
      "gate_prec": 48.62637362637363,
      "gate_rec": 39.45022288261516,
      "gate_tp": 531,
      "gate_fp": 561,
      "gate_tn": 7695,
      "gate_fn": 815,
      "decision_acc": 86.28410696983337,
      "decision_prec": 70.4225352112676,
      "decision_rec": 3.7147102526002973,
      "decision_fp_rate": 0.2543604651162791,
      "decision_tp": 50,
      "decision_fp": 21,
      "decision_tn": 8235,
      "decision_fn": 1296,
      "oracle_acc_on_reranked_failures": 40.00000059604645,
      "true_gain_on_reranked_failures": 31.280744075775146,
      "n_false_positive_reranks": 21,
      "fp_rerank_harm": -63.64555358886719,
      "oracle_accuracy_overall": 85.98208427429199
    },
    "val_refcoco_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 10834,
      "n_failures": 937,
      "failure_rate": 8.648698541628209,
      "rerank_count": 59,
      "rerank_rate": 0.5445818718848071,
      "q0_miou": 85.45660972595215,
      "oracle_miou": 89.42838907241821,
      "selected_miou": 85.45025587081909,
      "delta_full": -0.006353855133056641,
      "q0_miou_fail": 32.160285115242004,
      "oracle_miou_fail": 78.08363437652588,
      "selected_miou_fail": 33.016687631607056,
      "delta_fail": 0.8564025163650513,
      "gap_closed": 1.8648520418066497,
      "auc_fail": 0.8085167837046013,
      "gate_acc": 90.28059840202332,
      "gate_prec": 43.722943722943725,
      "gate_rec": 43.11632870864461,
      "gate_tp": 404,
      "gate_fp": 520,
      "gate_tn": 9377,
      "gate_fn": 533,
      "decision_acc": 91.61897897720337,
      "decision_prec": 74.57627118644068,
      "decision_rec": 4.695837780149413,
      "decision_fp_rate": 0.1515610791148833,
      "decision_tp": 44,
      "decision_fp": 15,
      "decision_tn": 9882,
      "decision_fn": 893,
      "oracle_acc_on_reranked_failures": 34.090909361839294,
      "true_gain_on_reranked_failures": 18.237465620040894,
      "n_false_positive_reranks": 15,
      "fp_rerank_harm": -58.08122754096985,
      "oracle_accuracy_overall": 91.35130047798157
    },
    "val_refcocog_umd": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 4896,
      "n_failures": 758,
      "failure_rate": 15.48202614379085,
      "rerank_count": 35,
      "rerank_rate": 0.7148692810457516,
      "q0_miou": 79.804128408432,
      "oracle_miou": 87.27956414222717,
      "selected_miou": 79.77062463760376,
      "delta_full": -0.03350377082824707,
      "q0_miou_fail": 26.25426948070526,
      "oracle_miou_fail": 74.53891634941101,
      "selected_miou_fail": 27.04944908618927,
      "delta_fail": 0.7951796054840088,
      "gap_closed": 1.6468580740504093,
      "auc_fail": 0.7836166121065967,
      "gate_acc": 84.84477400779724,
      "gate_prec": 51.34680134680135,
      "gate_rec": 40.23746701846966,
      "gate_tp": 305,
      "gate_fp": 289,
      "gate_tn": 3849,
      "gate_fn": 453,
      "decision_acc": 84.82434749603271,
      "decision_prec": 71.42857142857143,
      "decision_rec": 3.2981530343007917,
      "decision_fp_rate": 0.2416626389560174,
      "decision_tp": 25,
      "decision_fp": 10,
      "decision_tn": 4128,
      "decision_fn": 733,
      "oracle_acc_on_reranked_failures": 43.99999976158142,
      "true_gain_on_reranked_failures": 24.10985380411148,
      "n_false_positive_reranks": 10,
      "fp_rerank_harm": -76.68079137802124,
      "oracle_accuracy_overall": 84.51797366142273
    },
    "val_refcocoplus_unc": {
      "tau": 0.9,
      "margin_pred": 0.0,
      "margin_gap": 0.0,
      "gate_temperature": 0.5,
      "n_samples": 10758,
      "n_failures": 1419,
      "failure_rate": 13.190184049079754,
      "rerank_count": 110,
      "rerank_rate": 1.0224948875255624,
      "q0_miou": 81.12329840660095,
      "oracle_miou": 89.28661346435547,
      "selected_miou": 81.11199140548706,
      "delta_full": -0.011307001113891602,
      "q0_miou_fail": 18.030162155628204,
      "oracle_miou_fail": 79.91952896118164,
      "selected_miou_fail": 19.942304491996765,
      "delta_fail": 1.9121423363685608,
      "gap_closed": 3.0896136688168236,
      "auc_fail": 0.8240500085986754,
      "gate_acc": 86.73545122146606,
      "gate_prec": 49.72375690607735,
      "gate_rec": 50.7399577167019,
      "gate_tp": 720,
      "gate_fp": 728,
      "gate_tn": 8611,
      "gate_fn": 699,
      "decision_acc": 87.08868026733398,
      "decision_prec": 63.63636363636363,
      "decision_rec": 4.933051444679352,
      "decision_fp_rate": 0.4283113823749866,
      "decision_tp": 70,
      "decision_fp": 40,
      "decision_tn": 9299,
      "decision_fn": 1349,
      "oracle_acc_on_reranked_failures": 48.571428656578064,
      "true_gain_on_reranked_failures": 38.76187205314636,
      "n_false_positive_reranks": 40,
      "fp_rerank_harm": -70.87111473083496,
      "oracle_accuracy_overall": 86.80981397628784
    }
  }
}