fix(colab): silence Qwen max_length FutureWarning, real 120-update training run, validated task IDs

colab/train_incident_commander.ipynb

@@ -75,9 +75,18 @@
     "import os, subprocess, sys, zipfile\n",
     "REPO_DIR = '/content/incident-commander'\n",
     "\n",
+    "# Default to the user's HF Space so the notebook works zero-config.\n",
+    "# Override either of these env vars to clone from somewhere else.\n",
+    "os.environ.setdefault('IC_HF_SPACE', 'sagnik-mukherjee/incodent-commander')\n",
+    "# Optional: GitHub mirror — also defaulted to the user's repo.\n",
+    "os.environ.setdefault('IC_REPO_URL', 'https://github.com/r1cksync/meta-rl-hack.git')\n",
+    "\n",
     "IC_REPO_URL = os.environ.get('IC_REPO_URL', '')\n",
     "if IC_REPO_URL and not os.path.isdir(REPO_DIR):\n",
-    "    subprocess.run(['git','clone','--depth','1', IC_REPO_URL, REPO_DIR], check=True)\n",
+    "    try:\n",
+    "        subprocess.run(['git','clone','--depth','1', IC_REPO_URL, REPO_DIR], check=True)\n",
+    "    except subprocess.CalledProcessError:\n",
+    "        print(f'GitHub clone failed for {IC_REPO_URL}; will try the HF Space next.')\n",
     "\n",
     "if not os.path.isdir(REPO_DIR) and os.path.exists('/content/incident-commander.zip'):\n",
     "    os.makedirs(REPO_DIR, exist_ok=True)\n",
@@ -92,7 +101,8 @@
     "assert os.path.isdir(REPO_DIR), 'No repo present — pick Option A/B/C above.'\n",
     "%cd /content/incident-commander\n",
     "sys.path.insert(0, '/content/incident-commander')\n",
-    "sys.path.insert(0, '/content/incident-commander/rl-agent')"
+    "sys.path.insert(0, '/content/incident-commander/rl-agent')\n",
+    "print('Repo ready at', REPO_DIR)"
    ]
   },
   {
@@ -102,9 +112,14 @@
    "source": [
     "## 3 · Hugging Face token\n",
     "\n",
-    "`HF_TOKEN` powers (a) actor weight downloads and (b) the Qwen2.5-72B critic over the Inference Providers router. Get one at https://huggingface.co/settings/tokens — **Read** scope is enough for training (add **Write** if you want this notebook to push your trained adapter).\n",
+    "`HF_TOKEN` powers (a) actor weight downloads and (b) the Qwen2.5-72B critic over the Inference Providers router.\n",
+    "\n",
+    "**Two ways to provide it** — the cell below tries them in order:\n",
+    "\n",
+    "1. **Colab Secret (recommended)** — open the 🔑 icon in Colab's left sidebar → **Add new secret** → name `HF_TOKEN`, paste your `hf_…` token, toggle **Notebook access** on. The cell picks it up automatically with no prompt.\n",
+    "2. **Inline `getpass`** — if no secret is set, the cell falls back to a hidden prompt where you paste the token.\n",
     "\n",
-    "> **Security:** rotate the two HF tokens that were leaked in earlier chat (`hf_RLF…`, `hf_IBf…`)."
+    "> The token you wanted to use first (`hf_IBf…Mhl`) cannot be checked into this notebook because Hugging Face's pre-receive hook blocks any file that contains a token string. Paste it via secret or `getpass` instead. **Rotate it after this run** — it has been shared in plaintext."
    ]
   },
   {
@@ -115,13 +130,27 @@
    "outputs": [],
    "source": [
     "import os, getpass\n",
+    "\n",
+    "# 1) Try Colab's built-in secrets manager (left sidebar → Secrets → add HF_TOKEN).\n",
+    "try:\n",
+    "    from google.colab import userdata           # type: ignore\n",
+    "    _t = userdata.get('HF_TOKEN')\n",
+    "    if _t:\n",
+    "        os.environ['HF_TOKEN'] = _t\n",
+    "except Exception:\n",
+    "    pass\n",
+    "\n",
+    "# 2) Otherwise prompt for it (paste your hf_… token).\n",
     "if not os.environ.get('HF_TOKEN'):\n",
     "    os.environ['HF_TOKEN'] = getpass.getpass('Paste your HF token (hf_…): ')\n",
+    "\n",
     "os.environ['HUGGING_FACE_HUB_TOKEN'] = os.environ['HF_TOKEN']\n",
     "os.environ['INCIDENT_COMMANDER_MOCK'] = 'true'\n",
-    "from huggingface_hub import login\n",
+    "\n",
+    "from huggingface_hub import login, whoami\n",
     "login(os.environ['HF_TOKEN'], add_to_git_credential=False)\n",
-    "print('HF login OK')"
+    "me = whoami(token=os.environ['HF_TOKEN'])\n",
+    "print(f\"HF login OK as: {me.get('name', me)}\")"
    ]
   },
   {
@@ -196,30 +225,64 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import warnings, logging\n",
+    "warnings.filterwarnings('ignore', category=FutureWarning, module='transformers')\n",
+    "warnings.filterwarnings('ignore', message='.*max_new_tokens.*max_length.*')\n",
+    "warnings.filterwarnings('ignore', message='.*attention mask API.*')\n",
+    "logging.getLogger('transformers').setLevel(logging.ERROR)\n",
+    "\n",
     "from colab.train_lib import CFG, train_loop\n",
+    "\n",
+    "# REAL training run — 120 PPO updates × 6 rollouts/update ≈ 11.5 k transitions\n",
+    "# (~70–90 min on a T4, ~25 min on an A100). Bump `rollouts_per_update` and\n",
+    "# `total_updates` together if you want even longer.\n",
     "CFG.update({\n",
-    "    'total_updates':       40,    # bump to 80–120 for the real run\n",
-    "    'rollouts_per_update': 4,\n",
-    "    'max_steps_per_ep':    14,\n",
-    "    'critic_provider':     'hf',\n",
-    "    'critic_model':        'Qwen/Qwen2.5-72B-Instruct',\n",
-    "    'lr':                  1e-5,\n",
-    "    'run_name':            'demo01',\n",
+    "    'total_updates':        120,\n",
+    "    'rollouts_per_update':  6,\n",
+    "    'max_steps_per_ep':     16,\n",
+    "    'critic_provider':      'hf',\n",
+    "    'critic_model':         'Qwen/Qwen2.5-72B-Instruct',\n",
+    "    'lr':                   1e-5,\n",
+    "    'kl_coef':              0.02,\n",
+    "    'clip_eps':             0.20,\n",
+    "    'gae_lambda':           0.92,\n",
+    "    'checkpoint_every':     20,\n",
+    "    'run_name':             'real01',\n",
     "    'tasks': [\n",
+    "        # Mix difficulty + saboteur/Slack templates so the policy sees the\n",
+    "        # full task distribution every update. All IDs verified against the\n",
+    "        # rl-agent/scenarios/sim/ tree.\n",
     "        'sim_easy_lambda_throttle_001',\n",
+    "        'sim_easy_lambda_throttle_010',\n",
     "        'sim_med_eb_lambda_016',\n",
+    "        'sim_med_eb_lambda_021',\n",
     "        'sim_hard_apigw_chain_001',\n",
+    "        'sim_hard_ddb_chain_021',\n",
+    "        'sim_hard_iam_chain_011',\n",
     "        'sim_advanced_cascade_users_db_001',\n",
     "        'sim_advanced_runbook_trap_postgres_001',\n",
     "        'sim_advanced_trolley_orders_db_001',\n",
     "        'sim_advanced_saboteur_duel_001',\n",
     "        'sim_advanced_slack_redherring_001',\n",
-    "        'sim_gen_app_leak_checkout_005',\n",
+    "        'sim_gen_app_leak_checkout_007',\n",
+    "        'sim_gen_app_leak_payments_019',\n",
     "        'sim_gen_db_duel_users_db_003',\n",
-    "        'sim_gen_redherring_payments_007',\n",
+    "        'sim_gen_db_duel_orders_db_015',\n",
+    "        'sim_gen_redherring_payments_013',\n",
+    "        'sim_gen_redherring_auth_001',\n",
     "        'sim_gen_cascade_payments_db_004',\n",
+    "        'sim_gen_cascade_users_db_023',\n",
+    "        'sim_gen_cache_warm_session_cache_004',\n",
+    "        'sim_gen_peak_frontend_001',\n",
+    "        'sim_gen_restore_payments_db_001',\n",
     "    ],\n",
     "})\n",
+    "\n",
+    "print(f\"Starting REAL run: {CFG['total_updates']} updates × \"\n",
+    "      f\"{CFG['rollouts_per_update']} rollouts × \"\n",
+    "      f\"{CFG['max_steps_per_ep']} steps  ≈  \"\n",
+    "      f\"{CFG['total_updates'] * CFG['rollouts_per_update'] * CFG['max_steps_per_ep']:,} transitions max\")\n",
+    "\n",
     "log_path = train_loop()\n",
     "print('Training log:', log_path)"
    ]
@@ -294,7 +357,9 @@
    "source": [
     "## 9 · Push trained adapter + logs back to Hugging Face\n",
     "\n",
-    "Set `IC_HF_USER` to your HF username before running. Creates (or re-uses) a public model repo `<user>/incident-commander-actor`."
+    "Defaults to `IC_HF_USER=sagnik-mukherjee`, so this cell will create / update the public model repo **`sagnik-mukherjee/incident-commander-actor`** with the adapter, replays, and training logs. Override `os.environ['IC_HF_USER']` if you want a different account.\n",
+    "\n",
+    "> Requires a **Write**-scope HF token. The pre-filled token (`hf_IBf…Mhl`) needs Write to push successfully — if it is Read-only, regenerate one with Write at https://huggingface.co/settings/tokens."
    ]
   },
   {
@@ -306,22 +371,25 @@
    "source": [
     "import os, glob\n",
     "from huggingface_hub import HfApi, create_repo\n",
-    "IC_HF_USER = os.environ.get('IC_HF_USER', '')\n",
-    "if not IC_HF_USER:\n",
-    "    print('Skipping push — set os.environ[\"IC_HF_USER\"] to enable.')\n",
-    "else:\n",
-    "    api  = HfApi()\n",
-    "    repo = f'{IC_HF_USER}/incident-commander-actor'\n",
-    "    create_repo(repo, exist_ok=True, repo_type='model')\n",
-    "    final = sorted(glob.glob('colab/logs/adapter_*_final'))[-1]\n",
-    "    api.upload_folder(folder_path=final, repo_id=repo, repo_type='model',\n",
+    "\n",
+    "os.environ.setdefault('IC_HF_USER', 'sagnik-mukherjee')\n",
+    "IC_HF_USER = os.environ['IC_HF_USER']\n",
+    "\n",
+    "api  = HfApi(token=os.environ['HF_TOKEN'])\n",
+    "repo = f'{IC_HF_USER}/incident-commander-actor'\n",
+    "create_repo(repo, exist_ok=True, repo_type='model', token=os.environ['HF_TOKEN'])\n",
+    "\n",
+    "finals = sorted(glob.glob('colab/logs/adapter_*_final'))\n",
+    "if finals:\n",
+    "    api.upload_folder(folder_path=finals[-1], repo_id=repo, repo_type='model',\n",
     "                      path_in_repo='adapter')\n",
-    "    api.upload_folder(folder_path='colab/logs', repo_id=repo, repo_type='model',\n",
-    "                      path_in_repo='logs', allow_patterns=['*.json'])\n",
+    "api.upload_folder(folder_path='colab/logs', repo_id=repo, repo_type='model',\n",
+    "                  path_in_repo='logs', allow_patterns=['*.json'])\n",
+    "if os.path.isdir('rl-agent/replays'):\n",
     "    api.upload_folder(folder_path='rl-agent/replays', repo_id=repo,\n",
     "                      repo_type='model', path_in_repo='replays',\n",
     "                      allow_patterns=['*.html'])\n",
-    "    print(f'Pushed → https://huggingface.co/{repo}')"
+    "print(f'Pushed → https://huggingface.co/{repo}')"
    ]
   },
   {

colab/train_lib.py

@@ -254,6 +254,20 @@ class QwenActor:
         self._train_mode = False
         self.max_seq_len = max_seq_len
 
+        # Silence the noisy `Both max_new_tokens and max_length seem to have
+        # been set` FutureWarning. Qwen ships a default max_length=32768 in
+        # its generation_config; we always pass max_new_tokens at call time,
+        # so dropping the inherited max_length is correct.
+        try:
+            self.model.generation_config.max_length = None
+        except Exception:                                       # noqa: BLE001
+            pass
+        import warnings as _w
+        _w.filterwarnings("ignore", category=FutureWarning,
+                          module="transformers")
+        _w.filterwarnings("ignore", message=".*max_new_tokens.*max_length.*")
+        _w.filterwarnings("ignore", message=".*attention mask API.*")
+
     # -----------------------------------------------------
     def _format(self, observation: str) -> str:
         msgs = [{"role": "system", "content": self.SYSTEM_PROMPT},