{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PERMANENCE — training quickstart (Colab / T4)\n",
    "\n",
    "Runs the full four-stage PERMANENCE training pipeline on a free Colab T4.\n",
    "\n",
    "1. Clone the Space\n",
    "2. Install OpenEnv + Unsloth + TRL\n",
    "3. Generate warmup traces from the live environment\n",
    "4. Run supervised warmup → format gate → GRPO → held-out evaluation\n",
    "5. Render the results plots and summary\n",
    "\n",
    "Expected runtime: ~80 minutes on a T4.\n",
    "\n",
    "**Before running:** `Runtime` → `Change runtime type` → `T4 GPU`.\n",
    "\n",
    "**Logging note:** long-running cells in this notebook stream live logs with timestamps.\n",
    "You should see continuous output while training is running (not a silent spinner).\n",
    "\n",
    "If you would rather just inspect the final evaluation artefacts without\n",
    "retraining, jump to the last section — it downloads the committed\n",
    "adapter and eval artefacts from the Hugging Face artifacts dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1) Clone the Space repository (idempotent) and lock working directory to repo root.\n",
    "from pathlib import Path\n",
    "import os, subprocess\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "\n",
    "if not REPO_DIR.exists():\n",
    "    subprocess.check_call([\n",
    "        'git', 'clone', 'https://huggingface.co/spaces/chane35/permanence', str(REPO_DIR)\n",
    "    ])\n",
    "else:\n",
    "    print(f'Repository already present at {REPO_DIR}; skipping clone.')\n",
    "\n",
    "os.chdir(REPO_DIR)\n",
    "print('Working directory:', Path.cwd())\n",
    "print('training package dir exists:', (REPO_DIR / 'training').exists())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2) Install dependencies from repo root.\n",
    "from pathlib import Path\n",
    "import os, subprocess\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "subprocess.check_call(['python', '-m', 'pip', 'install', '-q', '--upgrade', 'pip'])\n",
    "subprocess.check_call([\n",
    "    'python', '-m', 'pip', 'install', '-q',\n",
    "    'unsloth', 'trl', 'transformers', 'datasets', 'huggingface_hub', 'fastapi', 'uvicorn', 'pytest'\n",
    "])\n",
    "subprocess.check_call(['python', '-m', 'pip', 'install', '-q', '-e', '.'])\n",
    "print('Dependencies installed from', Path.cwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3) Sanity check: tests + import from the expected repo root.\n",
    "from pathlib import Path\n",
    "import os, subprocess\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "subprocess.check_call(['python', '-m', 'pytest', 'tests/', '-q'])\n",
    "subprocess.check_call([\n",
    "    'python', '-c',\n",
    "    \"from permanence.env import PermanenceEnv; env = PermanenceEnv(); obs, info = env.reset(); print('env reset ok, prompt length:', len(obs['text']))\"\n",
    "],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4) Generate the 78 env-verified warmup traces.\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import os, subprocess, importlib\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "def run_stream(cmd):\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] START: {' '.join(cmd)}\", flush=True)\n",
    "    p = subprocess.Popen(\n",
    "        cmd,\n",
    "        stdout=subprocess.PIPE,\n",
    "        stderr=subprocess.STDOUT,\n",
    "        text=True,\n",
    "        bufsize=1,\n",
    "        env={**os.environ, 'PYTHONUNBUFFERED': '1'},\n",
    "    )\n",
    "    for line in p.stdout:\n",
    "        print(line, end='')\n",
    "    rc = p.wait()\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] END (exit={rc})\", flush=True)\n",
    "    if rc != 0:\n",
    "        raise subprocess.CalledProcessError(rc, cmd)\n",
    "\n",
    "# Preflight: ensure the top-level training module is importable from current cwd.\n",
    "importlib.import_module('training.generate_warmup_traces')\n",
    "run_stream(['python', '-u', 'training/generate_warmup_traces.py'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5) Run the four-stage pipeline. This is the ~80-minute step.\n",
    "#    Tune `total_episodes` in training/config.yaml for a shorter run.\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import os, subprocess, importlib\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "def run_stream(cmd):\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] START: {' '.join(cmd)}\", flush=True)\n",
    "    p = subprocess.Popen(\n",
    "        cmd,\n",
    "        stdout=subprocess.PIPE,\n",
    "        stderr=subprocess.STDOUT,\n",
    "        text=True,\n",
    "        bufsize=1,\n",
    "        env={**os.environ, 'PYTHONUNBUFFERED': '1'},\n",
    "    )\n",
    "    for line in p.stdout:\n",
    "        print(line, end='')\n",
    "    rc = p.wait()\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] END (exit={rc})\", flush=True)\n",
    "    if rc != 0:\n",
    "        raise subprocess.CalledProcessError(rc, cmd)\n",
    "\n",
    "importlib.import_module('training.pipeline')\n",
    "run_stream(['python', '-u', '-m', 'training.pipeline', '--config', 'training/config.yaml'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6) Render the result plots and summary into results/.\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import os, subprocess\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "def run_stream(cmd):\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] START: {' '.join(cmd)}\", flush=True)\n",
    "    p = subprocess.Popen(\n",
    "        cmd,\n",
    "        stdout=subprocess.PIPE,\n",
    "        stderr=subprocess.STDOUT,\n",
    "        text=True,\n",
    "        bufsize=1,\n",
    "        env={**os.environ, 'PYTHONUNBUFFERED': '1'},\n",
    "    )\n",
    "    for line in p.stdout:\n",
    "        print(line, end='')\n",
    "    rc = p.wait()\n",
    "    print(f\"[{datetime.now().strftime('%H:%M:%S')}] END (exit={rc})\", flush=True)\n",
    "    if rc != 0:\n",
    "        raise subprocess.CalledProcessError(rc, cmd)\n",
    "\n",
    "run_stream(['python', '-u', 'tools/render_results.py'])\n",
    "\n",
    "from IPython.display import Image\n",
    "Image('results/confusion_matrix.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7) Final summary text\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "print(Path('results/summary.txt').read_text())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Just want the final numbers? Pull the committed artefacts.\n",
    "\n",
    "The `results/` folder in this repo already contains a snapshot of the\n",
    "latest evaluation artefacts — `results.json`, `comparison.csv`, and\n",
    "`training_log.json` — plus the rendered plots. You can inspect them\n",
    "directly or pull the full adapter + raw artefacts from the HF dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import os, json\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "print(json.dumps(json.load(open('results/results.json')), indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional: download the full adapter + raw training log from HF.\n",
    "from pathlib import Path\n",
    "import os\n",
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "REPO_DIR = Path('/content/permanence_repo')\n",
    "if not REPO_DIR.exists():\n",
    "    raise FileNotFoundError('Repo not found. Run Cell 2 first.')\n",
    "os.chdir(REPO_DIR)\n",
    "\n",
    "path = snapshot_download(\n",
    "    repo_id='chane35/permanence-artifacts',\n",
    "    repo_type='dataset',\n",
    "    local_dir='./hf_artifacts',\n",
    ")\n",
    "print(f'Downloaded to {path}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}