import numpy as np def ece(conf, correct, bins=15): conf = np.asarray(conf); correct = np.asarray(correct, dtype=float) edges = np.linspace(0, 1, bins + 1); e = 0.0 for lo, hi in zip(edges[:-1], edges[1:]): m = (conf > lo) & (conf <= hi) if m.any(): e += m.mean() * abs(conf[m].mean() - correct[m].mean()) return float(e) def aurc(conf, correct): """Area under risk-coverage curve (lower is better).""" order = np.argsort(-np.asarray(conf)); c = np.asarray(correct, dtype=float)[order] risk = np.cumsum(1 - c) / np.arange(1, len(c) + 1) return float(risk.mean()) def sel_acc(conf, correct, coverage): order = np.argsort(-np.asarray(conf)); c = np.asarray(correct, dtype=float)[order] n = max(1, int(round(coverage * len(c)))) return float(c[:n].mean()) def summarize(probs, golds, nopts): """probs [N,K] (masked entries 0), golds [N], nopts [N].""" probs = np.asarray(probs); golds = np.asarray(golds); nopts = np.asarray(nopts) pred = probs.argmax(1); conf = probs.max(1); correct = (pred == golds) p_gold = probs[np.arange(len(golds)), golds] nll = -np.log(np.clip(p_gold, 1e-12, 1)).mean() onehot = np.zeros_like(probs); onehot[np.arange(len(golds)), golds] = 1 brier = ((probs - onehot) ** 2).sum(1).mean() return dict(n=int(len(golds)), acc=float(correct.mean()), nll=float(nll), brier=float(brier), ece=ece(conf, correct), aurc=aurc(conf, correct), acc_at_80=sel_acc(conf, correct, 0.8), acc_at_50=sel_acc(conf, correct, 0.5), chance=float((1.0 / nopts).mean()), mean_conf=float(conf.mean()))