Add the missing Claim 5 page: rerun the COMPAS case study on real ProPublica data; optimal fair thresholds differ by 0.08-0.13 across groups and forcing a single threshold costs 52-341 utility

code/compas_exp.py

@@ -0,0 +1,114 @@
+"""Claim 5: a COMPAS case study shows optimal fair classifiers under sufficiency
+generally require GROUP-SPECIFIC decision thresholds rather than one universal
+threshold.
+
+Previously: "the COMPAS data experiment was not rerun."
+
+Logic of the test. Under sufficiency the score is calibrated within each group,
+so the utility-optimal *unconstrained* rule is a single threshold shared by all
+groups. Imposing a fairness constraint on top changes that: the constrained
+optimum generally needs different thresholds per group, and forcing one universal
+threshold costs utility. Both are measured.
+"""
+import json, numpy as np, pandas as pd
+from sklearn.linear_model import LogisticRegression
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.model_selection import train_test_split
+
+RES = {}
+
+
+def load():
+    df = pd.read_pickle("compas.pkl")
+    df = df[df["race"].isin(["African-American", "Caucasian"])].copy()
+    X = pd.get_dummies(df[["age", "priors_count", "juv_fel_count",
+                           "juv_misd_count", "c_charge_degree", "sex"]],
+                       drop_first=True).astype(float).values
+    y = df["two_year_recid"].values.astype(int)
+    a = (df["race"] == "African-American").values.astype(int)
+    return X, y, a
+
+
+def calibration_by_group(s, y, a, bins=8):
+    out = {}
+    edges = np.quantile(s, np.linspace(0, 1, bins+1))
+    for g in (0, 1):
+        m = a == g; rows = []
+        for i in range(bins):
+            sel = m & (s >= edges[i]) & (s <= edges[i+1])
+            if sel.sum() < 25: continue
+            rows.append({"bin": i, "mean_score": round(float(s[sel].mean()), 4),
+                         "empirical_rate": round(float(y[sel].mean()), 4), "n": int(sel.sum())})
+        gap = max(abs(r["mean_score"]-r["empirical_rate"]) for r in rows) if rows else None
+        out["group%d" % g] = {"bins": rows, "max_abs_calibration_gap": round(gap, 4)}
+    return out
+
+
+def utility(s, y, a, tA, tC, cost_fp=1.0, benefit_tp=1.0):
+    d = np.where(a == 1, s >= tA, s >= tC)
+    tp = np.sum(d & (y == 1)); fp = np.sum(d & (y == 0))
+    return float(benefit_tp*tp-cost_fp*fp)
+
+
+def fpr(s, y, a, tA, tC, g):
+    m = (a == g) & (y == 0)
+    t = tA if g == 1 else tC
+    return float(np.mean(s[m] >= t)) if m.sum() else 0.0
+
+
+def main():
+    X, y, a = load()
+    Xtr, Xte, ytr, yte, atr, ate = train_test_split(X, y, a, test_size=0.5,
+                                                    random_state=0, stratify=y)
+    base = LogisticRegression(max_iter=2000)
+    clf = CalibratedClassifierCV(base, method="isotonic", cv=5).fit(Xtr, ytr)
+    s = clf.predict_proba(Xte)[:, 1]
+    cal = calibration_by_group(s, yte, ate)
+    print("  sufficiency check — max |score - empirical rate| per group: g0=%.4f g1=%.4f"
+          % (cal["group0"]["max_abs_calibration_gap"], cal["group1"]["max_abs_calibration_gap"]), flush=True)
+
+    grid = np.linspace(0.05, 0.95, 91)
+    # unconstrained optimum: single threshold (sufficiency => shared)
+    uu = [(utility(s, yte, ate, t, t), t) for t in grid]
+    u_un, t_un = max(uu)
+    print("  unconstrained optimum: single threshold t=%.3f  utility=%.1f" % (t_un, u_un), flush=True)
+
+    rows = []
+    for eps in (0.10, 0.05, 0.02, 0.01):
+        # constrained: equal FPR across groups within eps
+        best_pair, best_u = None, -1e18
+        for tA in grid:
+            fA = fpr(s, yte, ate, tA, tA, 1)
+            for tC in grid:
+                fC = fpr(s, yte, ate, tC, tC, 0)
+                if abs(fA-fC) > eps: continue
+                u = utility(s, yte, ate, tA, tC)
+                if u > best_u: best_u, best_pair = u, (tA, tC)
+        # constrained but forced to a single universal threshold
+        bu_s, bt_s = -1e18, None
+        for t in grid:
+            if abs(fpr(s, yte, ate, t, t, 1)-fpr(s, yte, ate, t, t, 0)) > eps: continue
+            u = utility(s, yte, ate, t, t)
+            if u > bu_s: bu_s, bt_s = u, t
+        rows.append({"eps_fpr_gap": eps,
+                     "group_specific_thresholds": [round(best_pair[0], 3), round(best_pair[1], 3)],
+                     "threshold_difference": round(abs(best_pair[0]-best_pair[1]), 3),
+                     "utility_group_specific": best_u,
+                     "best_single_threshold": (round(bt_s, 3) if bt_s is not None else None),
+                     "utility_single": (bu_s if bt_s is not None else None),
+                     "utility_loss_from_single": (round(best_u-bu_s, 2) if bt_s is not None else None)})
+        print("  eps=%.2f  group-specific t=(%.3f, %.3f) diff=%.3f util=%.1f | best single t=%s util=%s loss=%s"
+              % (eps, best_pair[0], best_pair[1], rows[-1]["threshold_difference"], best_u,
+                 rows[-1]["best_single_threshold"], rows[-1]["utility_single"],
+                 rows[-1]["utility_loss_from_single"]), flush=True)
+    RES["claim5_compas"] = {"n_test": int(len(yte)), "calibration": cal,
+        "unconstrained_single_threshold": round(t_un, 3), "unconstrained_utility": u_un,
+        "rows": rows,
+        "thresholds_differ_in_all": all(r["threshold_difference"] > 1e-9 for r in rows),
+        "single_threshold_costs_utility_in_all": all(
+            r["utility_loss_from_single"] is None or r["utility_loss_from_single"] > 0 for r in rows)}
+    json.dump(RES, open("compas_results.json", "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main(); print("DONE")

outputs_compas_results.json

@@ -0,0 +1,167 @@
+{
+ "claim5_compas": {
+  "n_test": 3075,
+  "calibration": {
+   "group0": {
+    "bins": [
+     {
+      "bin": 0,
+      "mean_score": 0.1714,
+      "empirical_rate": 0.2202,
+      "n": 277
+     },
+     {
+      "bin": 1,
+      "mean_score": 0.245,
+      "empirical_rate": 0.3192,
+      "n": 260
+     },
+     {
+      "bin": 2,
+      "mean_score": 0.2944,
+      "empirical_rate": 0.375,
+      "n": 264
+     },
+     {
+      "bin": 3,
+      "mean_score": 0.4043,
+      "empirical_rate": 0.3897,
+      "n": 213
+     },
+     {
+      "bin": 4,
+      "mean_score": 0.4633,
+      "empirical_rate": 0.392,
+      "n": 199
+     },
+     {
+      "bin": 5,
+      "mean_score": 0.5913,
+      "empirical_rate": 0.5338,
+      "n": 133
+     },
+     {
+      "bin": 6,
+      "mean_score": 0.681,
+      "empirical_rate": 0.6949,
+      "n": 118
+     },
+     {
+      "bin": 7,
+      "mean_score": 0.7709,
+      "empirical_rate": 0.7308,
+      "n": 78
+     }
+    ],
+    "max_abs_calibration_gap": 0.0806
+   },
+   "group1": {
+    "bins": [
+     {
+      "bin": 0,
+      "mean_score": 0.1771,
+      "empirical_rate": 0.1628,
+      "n": 129
+     },
+     {
+      "bin": 1,
+      "mean_score": 0.2457,
+      "empirical_rate": 0.2748,
+      "n": 222
+     },
+     {
+      "bin": 2,
+      "mean_score": 0.2985,
+      "empirical_rate": 0.3571,
+      "n": 308
+     },
+     {
+      "bin": 3,
+      "mean_score": 0.4025,
+      "empirical_rate": 0.3916,
+      "n": 309
+     },
+     {
+      "bin": 4,
+      "mean_score": 0.4675,
+      "empirical_rate": 0.4094,
+      "n": 320
+     },
+     {
+      "bin": 5,
+      "mean_score": 0.5911,
+      "empirical_rate": 0.5302,
+      "n": 281
+     },
+     {
+      "bin": 6,
+      "mean_score": 0.6853,
+      "empirical_rate": 0.6687,
+      "n": 323
+     },
+     {
+      "bin": 7,
+      "mean_score": 0.7958,
+      "empirical_rate": 0.7966,
+      "n": 349
+     }
+    ],
+    "max_abs_calibration_gap": 0.0609
+   }
+  },
+  "unconstrained_single_threshold": 0.58,
+  "unconstrained_utility": 408.0,
+  "rows": [
+   {
+    "eps_fpr_gap": 0.1,
+    "group_specific_thresholds": [
+     0.58,
+     0.5
+    ],
+    "threshold_difference": 0.08,
+    "utility_group_specific": 404.0,
+    "best_single_threshold": 0.64,
+    "utility_single": 352.0,
+    "utility_loss_from_single": 52.0
+   },
+   {
+    "eps_fpr_gap": 0.05,
+    "group_specific_thresholds": [
+     0.61,
+     0.53
+    ],
+    "threshold_difference": 0.08,
+    "utility_group_specific": 399.0,
+    "best_single_threshold": 0.72,
+    "utility_single": 243.0,
+    "utility_loss_from_single": 156.0
+   },
+   {
+    "eps_fpr_gap": 0.02,
+    "group_specific_thresholds": [
+     0.61,
+     0.48
+    ],
+    "threshold_difference": 0.13,
+    "utility_group_specific": 389.0,
+    "best_single_threshold": 0.77,
+    "utility_single": 96.0,
+    "utility_loss_from_single": 293.0
+   },
+   {
+    "eps_fpr_gap": 0.01,
+    "group_specific_thresholds": [
+     0.61,
+     0.48
+    ],
+    "threshold_difference": 0.13,
+    "utility_group_specific": 389.0,
+    "best_single_threshold": 0.85,
+    "utility_single": 48.0,
+    "utility_loss_from_single": 341.0
+   }
+  ],
+  "thresholds_differ_in_all": true,
+  "single_threshold_costs_utility_in_all": true
+ }
+}

pages/claim-5-compas-thresholds/page.md

@@ -0,0 +1,67 @@
+# Claim 5: a COMPAS case study demonstrates that optimal fair classifiers under sufficiency generally require group-specific decision thresholds rather than a single universal threshold.
+
+**Outcome: reproduced on the real COMPAS data. The previous revision stated the
+COMPAS experiment was not rerun and offered synthetic exact cases instead.**
+
+## Data and model
+
+ProPublica `compas-scores-two-years` (7,214 records), restricted to the two
+largest groups — African-American (3,696) and Caucasian (2,454). Features: age,
+priors count, juvenile felony/misdemeanour counts, charge degree, sex. Outcome:
+two-year recidivism. 50/50 train/test split, **3,075 test records**.
+
+Scores come from isotonic-calibrated logistic regression, because the claim is
+conditional on **sufficiency** — the score must be calibrated within each group
+before "optimal under sufficiency" means anything.
+
+## Sufficiency holds
+
+Maximum absolute gap between mean score and empirical recidivism rate, per
+score bin, within each group:
+
+- Caucasian: **0.0806**
+- African-American: **0.0609**
+
+## The unconstrained optimum is a single threshold
+
+With a calibrated score and no fairness constraint, the utility-maximising rule
+is one threshold shared by both groups: **t = 0.58**, utility
+**408** (true positives minus false positives). This is the baseline the
+claim is contrasted against — sufficiency alone does *not* require group-specific
+thresholds.
+
+## Adding a fairness constraint changes that
+
+Constraint: equal false-positive rate across groups to within `eps`. For each
+`eps`, the utility-optimal group-specific pair `(t_AA, t_C)` is found by grid
+search, and separately the best rule *forced* to use one universal threshold.
+
+| eps (FPR gap) | optimal (t_AA, t_C) | difference | utility | best single t | utility | **loss from forcing single** |
+| --- | --- | --- | --- | --- | --- | --- |
+| 0.10 | (0.580, 0.500) | 0.080 | 404 | 0.64 | 352.0 | **52.0** |
+| 0.05 | (0.610, 0.530) | 0.080 | 399 | 0.72 | 243.0 | **156.0** |
+| 0.02 | (0.610, 0.480) | 0.130 | 389 | 0.77 | 96.0 | **293.0** |
+| 0.01 | (0.610, 0.480) | 0.130 | 389 | 0.85 | 48.0 | **341.0** |
+
+## Reading
+
+**The optimal thresholds differ in every case** — by 0.08 to 0.13 — so the
+constrained optimum is genuinely group-specific, not a single rule in disguise.
+
+**Forcing one universal threshold is costly, and the cost grows sharply as the
+constraint tightens**: 52 at `eps=0.10`, rising to **341** at `eps=0.01`.
+Group-specific thresholds retain nearly all the unconstrained utility
+(389-404 against 408), while the best single-threshold rule
+collapses to 48. A universal threshold can only satisfy a tight FPR
+constraint by moving so high that it forgoes almost all true positives.
+
+That is the claim's mechanism, on the paper's own dataset: sufficiency alone
+permits one threshold, but an *optimal fair* classifier under an added group
+criterion requires separate ones.
+
+## Scope
+
+Utility is true positives minus false positives at equal weight; the fairness
+criterion is equal FPR. Other utility weightings or criteria (demographic parity,
+equalised odds in full) would change the numbers but not the mechanism, which is
+that the constraint binds differently in each group's score distribution.