Phase 4: LDAM head fine-tune (test F1 0.962, has_next_action 0.907, has_quality_finding 0.917)

README.md

@@ -30,7 +30,7 @@ Zero Python runtime needed.
 | claim_present F1 | 0.962 |
 | false_emission_rate | 0.017 |
 | span_token_IoU | 0.969 |
-| predicate_accuracy_macro | 0.959 |
+| predicate_accuracy_macro | 0.966 |
 | subject_type_accuracy_macro | 0.997 |
 
 ## Contents

calibrator.json

@@ -17,8 +17,8 @@
   "y": [
     0.9090909090909091,
     0.9090909090909091,
-    0.9676258992805755,
-    0.9676258992805755,
+    0.9694244604316546,
+    0.9694244604316546,
     0.9803921568627451,
     0.9803921568627451,
     0.9900611620795107,

config.json

@@ -10,55 +10,22 @@
   "cls_token_id": 50281,
   "sep_token_id": 50282,
   "trained_args": {
-    "backbone": "answerdotai/ModernBERT-large",
+    "checkpoint": "models/phase3/modernbert-large-mh-span-v3/best.pt",
     "train_file": "data/bakeoff/splits_v3/span_slot.train.jsonl",
     "val_file": "data/bakeoff/splits_v3/span_slot.val.jsonl",
-    "out_dir": "models/phase3/modernbert-large-mh-span-v3",
-    "summary": "runs/phase3/modernbert-large-mh-span-v3/summary.json",
-    "max_length": 512,
-    "batch_size": 16,
-    "eval_batch_size": 32,
-    "lr": 3e-05,
-    "warmup_steps": 300,
-    "max_steps": 5000,
-    "eval_every": 1000,
-    "seed": 42,
-    "precision": "bf16",
-    "max_train_rows": null,
-    "subsample_abstain": null
-  },
-  "trained_steps": 5000,
-  "eval_at_save": {
-    "n": 13264,
-    "claim_precision": 0.9587293646823412,
-    "claim_recall": 0.9596895343014522,
-    "claim_accuracy": 0.975422195416164,
-    "false_emission_rate": 0.01779935275080906,
-    "tp": 3833,
-    "fp": 165,
-    "fn": 161,
-    "tn": 9105,
-    "predicate_accuracy": 0.9819729594391587,
-    "subject_type_accuracy": 0.9957436154231347,
-    "confidence_mae": 0.02157552532094326,
-    "span_token_iou": 0.9724878820809442,
-    "span_exact_match": 0.8061250316375601,
-    "span_seen": 3951,
-    "predicate_recall_by_class": {
-      "decided": 0.9444444444444444,
-      "blocked_by": 0.9538461538461539,
-      "has_next_action": 0.9215686274509803,
-      "has_status": 0.9615384615384616,
-      "has_goal": 0.9938775510204082,
-      "touched_file": 0.9916167664670659,
-      "ran_command": 0.9983319432860718,
-      "has_constraint": 0.9792243767313019,
-      "has_open_question": 0.9386363636363636,
-      "has_quality_finding": 0.9090909090909091,
-      "created_file": 0.9552238805970149
-    },
-    "claim_f1": 0.9592092092092092
+    "out_dir": "models/phase4/modernbert-large-ldam-head",
+    "summary": "runs/phase4/ldam-head-summary.json",
+    "max_steps": 500,
+    "drw_start": 400,
+    "batch_size": 32,
+    "lr": 0.0001,
+    "warmup_steps": 50,
+    "max_m": 0.5,
+    "cb_beta": 0.9999,
+    "seed": 42
   },
+  "trained_steps": 500,
+  "eval_at_save": {},
   "heads": {
     "claim_present": {
       "kind": "linear",

logit_adjustment.json

@@ -0,0 +1,50 @@
+{
+  "priors": {
+    "validated_by": 3.4233679093492176e-05,
+    "had_outcome": 3.4233679093492176e-05,
+    "failed_because": 3.4233679093492176e-05,
+    "worked_because": 3.4233679093492176e-05,
+    "decided": 0.004416144603060491,
+    "blocked_by": 0.01554209030844545,
+    "has_next_action": 0.013933107391051316,
+    "has_status": 0.013727705316490363,
+    "has_goal": 0.16336311663414468,
+    "touched_file": 0.18633391530587792,
+    "ran_command": 0.2814350758275992,
+    "logged_event": 3.4233679093492176e-05,
+    "has_constraint": 0.1833898189038376,
+    "has_open_question": 0.1111225223374756,
+    "has_quality_finding": 0.012221423436376708,
+    "reverted_file": 3.4233679093492176e-05,
+    "deleted_file": 0.0005135051864023827,
+    "created_file": 0.013693471637396872,
+    "committed": 3.4233679093492176e-05,
+    "deployed": 3.4233679093492176e-05,
+    "incident_observed": 3.4233679093492176e-05
+  },
+  "default_tau": 0.0,
+  "predicate_order": [
+    "validated_by",
+    "had_outcome",
+    "failed_because",
+    "worked_because",
+    "decided",
+    "blocked_by",
+    "has_next_action",
+    "has_status",
+    "has_goal",
+    "touched_file",
+    "ran_command",
+    "logged_event",
+    "has_constraint",
+    "has_open_question",
+    "has_quality_finding",
+    "reverted_file",
+    "deleted_file",
+    "created_file",
+    "committed",
+    "deployed",
+    "incident_observed"
+  ],
+  "note": "At inference: predicate_logit[y] -= tau * log(priors[y])."
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08131bfe019bcc175eb827b6d5f90aa0a7a6cc288547b4e4b5d0d9a8e21431f0
+oid sha256:d2bd8dd832a925ae0cdeaefa15fa41e4bc3a98bcb039f90db089ee61252a1e20
 size 789661668