Upload scripts/run_all_baselines.sh

scripts/run_all_baselines.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run all baseline evaluations for publication comparison.
+# Run this on your RTX 6000 Ada server.
+#
+# Prerequisites:
+#   - HF_TOKEN set
+#   - OPENAI_API_KEY set (for GPT-4o-mini)
+#   - .venv activated with dependencies from requirements.txt
+#
+# Usage:
+#   bash scripts/run_all_baselines.sh
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+cd "$PROJECT_ROOT"
+
+source .venv/bin/activate
+export PYTHONPATH="$PROJECT_ROOT/src:${PYTHONPATH:-}"
+export TOKENIZERS_PARALLELISM=false
+
+# ─── Configuration ──────────────────────────────────────────────────────────
+MAX_SAMPLES=200   # Set to null (empty) for full evaluation
+BATCH_SIZE_LOCAL=4
+BATCH_SIZE_API=1
+SAVE_EVERY=25
+
+# ─── 1. Llama-3.1-8B-Instruct (local, zero-shot) ────────────────────────────
+echo "========================================"
+echo "1. Llama-3.1-8B-Instruct zero-shot"
+echo "========================================"
+
+python scripts/baseline_eval.py \
+  --model meta-llama/Llama-3.1-8B-Instruct \
+  --output_dir outputs/baselines/llama-3.1-8b-instruct \
+  --batch_size "$BATCH_SIZE_LOCAL" \
+  --max_samples_per_split ${MAX_SAMPLES:-} \
+  --save_every "$SAVE_EVERY"
+
+python scripts/normalize_eval_metrics.py \
+  --eval_dir outputs/baselines/llama-3.1-8b-instruct
+
+# ─── 2. Qwen2.5-7B-Instruct (local, zero-shot) ──────────────────────────────
+echo "========================================"
+echo "2. Qwen2.5-7B-Instruct zero-shot"
+echo "========================================"
+
+python scripts/baseline_eval.py \
+  --model Qwen/Qwen2.5-7B-Instruct \
+  --output_dir outputs/baselines/qwen2.5-7b-instruct \
+  --batch_size "$BATCH_SIZE_LOCAL" \
+  --max_samples_per_split ${MAX_SAMPLES:-} \
+  --save_every "$SAVE_EVERY"
+
+python scripts/normalize_eval_metrics.py \
+  --eval_dir outputs/baselines/qwen2.5-7b-instruct
+
+# ─── 3. GPT-4o-mini (API, zero-shot) ────────────────────────────────────────
+if [ -n "${OPENAI_API_KEY:-}" ]; then
+  echo "========================================"
+  echo "3. GPT-4o-mini zero-shot (API)"
+  echo "========================================"
+
+  python scripts/baseline_eval.py \
+    --model gpt-4o-mini \
+    --api_provider openai \
+    --output_dir outputs/baselines/gpt-4o-mini \
+    --batch_size "$BATCH_SIZE_API" \
+    --max_samples_per_split ${MAX_SAMPLES:-} \
+    --save_every "$SAVE_EVERY"
+
+  python scripts/normalize_eval_metrics.py \
+    --eval_dir outputs/baselines/gpt-4o-mini
+else
+  echo "Skipping GPT-4o-mini (OPENAI_API_KEY not set)"
+fi
+
+# ─── 4. Package comparison results ──────────────────────────────────────────
+echo "========================================"
+echo "4. Packaging comparison results"
+echo "========================================"
+
+python - <<'PYEOF'
+import json
+from pathlib import Path
+
+results = {}
+
+for name, eval_dir in [
+    ("llama-3.1-8b-instruct", "outputs/baselines/llama-3.1-8b-instruct"),
+    ("qwen2.5-7b-instruct", "outputs/baselines/qwen2.5-7b-instruct"),
+    ("gpt-4o-mini", "outputs/baselines/gpt-4o-mini"),
+]:
+    path = Path(eval_dir) / "all_normalized_metrics.json"
+    if path.exists():
+        results[name] = json.loads(path.read_text())
+
+# Stage 1 results for comparison
+stage1 = {
+    "test_in_distribution": {"parse_json": 1.0000, "norm_field_f1": 0.7956, "norm_key_f1": 0.9811},
+    "test_template_ood": {"parse_json": 1.0000, "norm_field_f1": 0.7865, "norm_key_f1": 0.9801},
+    "test_use_case_ood": {"parse_json": 0.9998, "norm_field_f1": 0.7907, "norm_key_f1": 0.9805},
+    "test_sector_ood": {"parse_json": 1.0000, "norm_field_f1": 0.7697, "norm_key_f1": 0.9818},
+    "test_adversarial": {"parse_json": 1.0000, "norm_field_f1": 0.9697, "norm_key_f1": 1.0000},
+}
+
+# Zero-shot Qwen3-8B from journal
+qwen3_zero = {
+    "test_in_distribution": {"parse_json": 0.335, "norm_field_f1": 0.0009, "norm_key_f1": 0.0169},
+    "test_template_ood": {"parse_json": 0.340, "norm_field_f1": 0.0014, "norm_key_f1": 0.0172},
+    "test_use_case_ood": {"parse_json": 0.325, "norm_field_f1": 0.0012, "norm_key_f1": 0.0198},
+    "test_sector_ood": {"parse_json": 0.345, "norm_field_f1": 0.0008, "norm_key_f1": 0.0171},
+    "test_adversarial": {"parse_json": 0.000, "norm_field_f1": 0.0000, "norm_key_f1": 0.0000},
+}
+
+# Print comparison table
+print("\n" + "=" * 100)
+print("BASELINE COMPARISON: All Models vs Qwen3-8B QLoRA Stage 1")
+print("=" * 100)
+
+splits = ["test_in_distribution", "test_template_ood", "test_use_case_ood", "test_sector_ood", "test_adversarial"]
+
+for split in splits:
+    print(f"\n--- {split} ---")
+    print(f"{'Model':<30s} {'Parse':>8s} {'Norm Field F1':>14s} {'Norm Key F1':>12s} {'vs Stage1 Δ':>12s}")
+    print("-" * 80)
+    
+    for model_name, model_results in results.items():
+        metrics = model_results.get(split, {})
+        parse_val = metrics.get("parse_json", 0)
+        field_val = metrics.get("norm_field_f1", 0)
+        key_val = metrics.get("norm_key_f1", 0)
+        delta = field_val - stage1[split]["norm_field_f1"]
+        print(f"{model_name:<30s} {parse_val:8.4f} {field_val:14.4f} {key_val:12.4f} {delta:+12.4f}")
+    
+    # Zero-shot Qwen3-8B
+    z = qwen3_zero[split]
+    delta_z = z["norm_field_f1"] - stage1[split]["norm_field_f1"]
+    print(f"{'Qwen3-8B zero-shot':<30s} {z['parse_json']:8.4f} {z['norm_field_f1']:14.4f} {z['norm_key_f1']:12.4f} {delta_z:+12.4f}")
+    
+    # Stage 1
+    s = stage1[split]
+    print(f"{'Qwen3-8B-QLoRA (stage1)':<30s} {s['parse_json']:8.4f} {s['norm_field_f1']:14.4f} {s['norm_key_f1']:12.4f} {'(baseline)':>12s}")
+
+# Save combined results
+out_path = Path("outputs/baselines/comparison_results.json")
+out_path.parent.mkdir(parents=True, exist_ok=True)
+out_path.write_text(json.dumps({
+    "baselines": results,
+    "stage1": stage1,
+    "qwen3_zero_shot": qwen3_zero,
+}, indent=2))
+
+print(f"\nCombined results saved to {out_path}")
+PYEOF
+
+echo "========================================"
+echo "All baseline evaluations complete!"
+echo "========================================"