File size: 4,021 Bytes
7da570f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | #!/usr/bin/env python
import argparse
import json
import math
import statistics
from pathlib import Path
METHODS = ("sft", "random", "greedy_verifier", "oracle_grid")
METRICS = ("hypervolume", "mip", "preference_regret", "near_optimal", "unique")
DIRECTIONS = {"up_up": (1, 1), "up_down": (1, -1),
"down_up": (-1, 1), "down_down": (-1, -1)}
def ci(xs):
sd = statistics.stdev(xs)
return {"mean": statistics.fmean(xs), "sd": sd, "ci95": 2.776 * sd / math.sqrt(len(xs))}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--input", type=Path, required=True)
ap.add_argument("--output", type=Path, required=True)
ap.add_argument("--markdown", type=Path, required=True)
args = ap.parse_args()
runs = [json.loads(p.read_text()) for p in sorted(args.input.glob("seed_*.json"))]
if len(runs) != 5 or len({r["split_hash"] for r in runs}) != 1:
raise SystemExit("expected five seeds with one split")
summary = {m: {k: ci([r["results"][m][k] for r in runs]) for k in METRICS}
for m in METHODS}
paired, relative = {}, {}
for base in ("random", "greedy_verifier"):
paired[base] = {k: ci([r["results"]["sft"][k] - r["results"][base][k]
for r in runs]) for k in METRICS}
relative[base] = {k: ci([100 * (r["results"]["sft"][k] - r["results"][base][k])
/ r["results"][base][k] for r in runs])
for k in ("hypervolume", "mip")}
by_direction = {}
for name, direction in DIRECTIONS.items():
by_direction[name] = {m: {metric: ci([
statistics.fmean(q[metric] for q in r["results"][m]["tasks"]
if tuple(q["directions"]) == direction) for r in runs])
for metric in ("hypervolume", "mip")} for m in METHODS}
by_direction[name]["sft_minus_greedy"] = {metric: ci([
statistics.fmean(q[metric] for q in r["results"]["sft"]["tasks"]
if tuple(q["directions"]) == direction)
- statistics.fmean(q[metric] for q in r["results"]["greedy_verifier"]["tasks"]
if tuple(q["directions"]) == direction) for r in runs])
for metric in ("hypervolume", "mip")}
verifier_rmse = {p: ci([r["val_rmse"][p] for r in runs]) for p in ("Egc", "Egb")}
output = {"seeds": [r["seed"] for r in runs], "split_hash": runs[0]["split_hash"],
"data_sha256": runs[0]["data_sha256"], "n_eval_tasks": runs[0]["n_eval_tasks"],
"n_preference_decisions": runs[0]["n_preference_decisions"],
"verifier_rmse": verifier_rmse, "summary": summary, "by_direction": by_direction,
"sft_minus": paired, "sft_relative_percent": relative}
args.output.write_text(json.dumps(output, indent=2) + "\n")
lines = ["# PolyEdit multi-objective Hypervolume and MIP", "",
"Values are five-seed means with 95% t confidence intervals.", "",
"| Method | Hypervolume | MIP | Preference regret | Near-optimal | Unique |",
"|---|---:|---:|---:|---:|---:|"]
for method in METHODS:
row = summary[method]
cell = lambda k: f"{row[k]['mean']:.3f} ± {row[k]['ci95']:.3f}"
lines.append(f"| {method} | {cell('hypervolume')} | {cell('mip')} | "
f"{cell('preference_regret')} | {cell('near_optimal')} | {cell('unique')} |")
lines += ["", "## Direction breakdown", "",
"| Direction | Method | Hypervolume | MIP |", "|---|---|---:|---:|"]
for name in DIRECTIONS:
for method in METHODS:
row = by_direction[name][method]
lines.append(f"| {name} | {method} | {row['hypervolume']['mean']:.3f} ± "
f"{row['hypervolume']['ci95']:.3f} | {row['mip']['mean']:.3f} ± "
f"{row['mip']['ci95']:.3f} |")
args.markdown.write_text("\n".join(lines) + "\n")
if __name__ == "__main__":
main()
|