{ "cells": [ { "cell_type": "markdown", "id": "intro", "metadata": {}, "source": [ "# Stocker — End-to-end Colab workflow\n", "\n", "Self-contained: clones the repo, installs deps, builds the dataset, loads\n", "**`google/gemma-4-E4B-it`** in-process via `transformers` (4-bit BnB),\n", "pre-caches all 7 specialist votes, runs **GRPO** on the moderator LoRA via\n", "TRL, and saves loss / reward plots + the trained adapter.\n", "\n", "Designed for a free **Colab T4** (16 GB VRAM) — no separate vLLM server\n", "needed. If you have an L4/A100, drop `load_in_4bit=False` for full bf16.\n", "\n", "Outputs live under `training/runs//`:\n", "- `moderator-lora/` — trained PEFT adapter\n", "- `loss.png`, `reward.png` — training curves\n", "- `eval_pre.json`, `eval_post.json` — pre/post backtest reports\n", "\n", "> Tip: `Runtime → Change runtime type → T4 GPU` before running." ] }, { "cell_type": "code", "execution_count": 7, "id": "install", "metadata": {}, "outputs": [], "source": [ "# Install — keep Colab's pre-baked pandas/numpy (TF + cudf depend on them);\n", "# only upgrade what we strictly need.\n", "#\n", "# Big upgrades (transformers/trl/peft) are intentional — Colab ships old.\n", "# Everything else uses --upgrade-strategy only-if-needed so we don't\n", "# break google-colab/tensorflow/gradio/etc.\n", "!pip install -q -U 'transformers>=4.55' 'trl>=0.11' 'peft>=0.13' 'accelerate>=1.0' 'bitsandbytes>=0.43' 'datasets>=3.0'\n", "!pip install -q --upgrade-strategy=only-if-needed yfinance mplfinance pyarrow 'pydantic>=2' pydantic-settings 'openai>=1' tensorboard 'huggingface_hub>=1.10'\n", "# Indicators are computed in-repo (app/data/indicators.py) — no pandas-ta." ] }, { "cell_type": "code", "execution_count": 8, "id": "clone", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Working dir: /content/stocker\n" ] } ], "source": [ "import os, sys, pathlib\n", "\n", "# 1) Already in a stocker repo? (running from VSCode on a local clone, or\n", "# the repo was manually uploaded to Colab). Detect and skip the clone.\n", "def _is_stocker_root(p: str) -> bool:\n", " return os.path.isfile(os.path.join(p, \"app\", \"council\", \"specialists.py\"))\n", "\n", "CANDIDATES = [os.getcwd(), \"/content/stocker\", \"/workspace/stocker\"]\n", "WORKDIR = next((c for c in CANDIDATES if _is_stocker_root(c)), None)\n", "\n", "if WORKDIR is None:\n", " # 2) Not present — clone. EDIT THIS URL before running on a fresh Colab.\n", " REPO_URL = \"https://github.com//stocker.git\"\n", " WORKDIR = \"/content/stocker\"\n", " assert \"\" not in REPO_URL, (\n", " \"Edit REPO_URL in this cell to your fork before running on a fresh runtime.\"\n", " )\n", " !git clone {REPO_URL} {WORKDIR}\n", " assert _is_stocker_root(WORKDIR), f\"Clone failed — {WORKDIR}/app/council/ missing.\"\n", "\n", "os.chdir(WORKDIR)\n", "sys.path.insert(0, WORKDIR)\n", "print(\"Working dir:\", WORKDIR)" ] }, { "cell_type": "code", "execution_count": null, "id": "auth", "metadata": {}, "outputs": [], "source": [ "# Gemma 4 is Apache-2.0 (not gated) — token just lifts download rate limits.\n", "from huggingface_hub import login\n", "import getpass\n", "login(token=getpass.getpass(\"HF token (read scope is enough): \"))" ] }, { "cell_type": "code", "execution_count": null, "id": "build-data", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[1/6] Fetching OHLCV ...\n", " prices: 536 rows\n", "[2/6] Computing indicators ...\n", " indicators: 536 rows\n", "[3/6] Fetching peers + commodity ...\n", " peers: 374 rows\n", "[4/6] Loading curated news/forums/macro ...\n", " news: 25 headlines, forums: 17 posts, macro: 12 events\n", "[5/6] Rendering candlestick charts ...\n", " charts: 126 PNGs in data/charts\n", "[6/6] Done.\n", "task_easy ticker=AAPL steps= 43 chart_ok=yes\n", "task_medium ticker=INTC steps= 41 chart_ok=yes\n", "task_hard ticker=META steps= 42 chart_ok=yes\n", "\n", "All 3 tasks OK.\n" ] } ], "source": [ "# Build the bundled dataset (yfinance + indicators + chart PNGs).\n", "# Idempotent — skip if already done.\n", "!python scripts/build_dataset.py\n", "!python scripts/validate_tasks.py" ] }, { "cell_type": "code", "execution_count": null, "id": "f55a1108", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:103: UserWarning: \n", "Error while fetching `HF_TOKEN` secret value from your vault: 'Requesting secret HF_TOKEN timed out. Secrets can only be fetched when running from the Colab UI.'.\n", "You are not authenticated with the Hugging Face Hub in this notebook.\n", "If the error persists, please let us know by opening an issue on GitHub (https://github.com/huggingface/huggingface_hub/issues/new).\n", " warnings.warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "09427ffe6e7b4be094cef43d13c33fe6", "version_major": 2, "version_minor": 0 }, "text/plain": [ "processor_config.json: 0.00B [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "8874cb7e313545a2b79bad9317858169", "version_major": 2, "version_minor": 0 }, "text/plain": [ "chat_template.jinja: 0.00B [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "37bb28cb29c64f97b2699f780174d9ee", "version_major": 2, "version_minor": 0 }, "text/plain": [ "config.json: 0.00B [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "459c2365fa5e4cf1ab6897f7a5e9816f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_config.json: 0.00B [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0f6eab584b334df89d7aa973094c3e58", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer.json: 0%| | 0.00/32.2M [00:00\u001b[0;34m()\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcouncil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspecialists\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mcouncil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cached_vote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# writes .cache/council//...\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 15\u001b[0m \u001b[0mtotal\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"side\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"hold\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"quantity\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/content/stocker/app/council/runner.py\u001b[0m in \u001b[0;36m_cached_vote\u001b[0;34m(self, sp, obs)\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m \u001b[0mvote\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 76\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmkdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvote\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_dump_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/content/stocker/app/council/specialists.py\u001b[0m in \u001b[0;36mvote\u001b[0;34m(self, obs)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mvote\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mMarketObservation\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mSpecialistVote\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mmessages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_messages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomplete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_tokens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemperature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparse_json_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m return SpecialistVote(\n", "\u001b[0;32m/content/stocker/app/council/llm.py\u001b[0m in \u001b[0;36mcomplete\u001b[0;34m(self, messages, max_tokens, temperature, extra_body)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0;31m# the processor expects.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mprepped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prep_messages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m inputs = self.processor.apply_chat_template(\n\u001b[0m\u001b[1;32m 212\u001b[0m \u001b[0mprepped\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0madd_generation_prompt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/transformers/processing_utils.py\u001b[0m in \u001b[0;36mapply_chat_template\u001b[0;34m(self, conversation, chat_template, tools, documents, add_generation_prompt, continue_final_message, return_assistant_tokens_mask, tokenize, return_tensors, return_dict, load_audio_from_video, processor_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1815\u001b[0m \u001b[0mcontent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"content\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1816\u001b[0m visuals = [\n\u001b[0;32m-> 1817\u001b[0;31m \u001b[0mcontent_block\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcontent_block\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcontent\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcontent_block\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"type\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"image\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"video\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1818\u001b[0m ]\n\u001b[1;32m 1819\u001b[0m audio_fnames = [\n", "\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'" ] } ], "source": [ "# Pre-cache the 7 specialists' votes for every (task, step). Specialists\n", "# are FROZEN, so this runs once and is reused across every GRPO step.\n", "from app.council.runner import Council\n", "from app.core.environment import StockerEnv\n", "from app.core.tasks import list_task_ids\n", "\n", "council = Council(client=client, use_cache=True)\n", "total = 0\n", "for task_id in list_task_ids():\n", " env = StockerEnv(task_id=task_id)\n", " obs = env.reset().observation\n", " while True:\n", " for sp in council.specialists:\n", " council._cached_vote(sp, obs) # writes .cache/council//...\n", " total += 1\n", " result = env.step({\"side\": \"hold\", \"quantity\": 0})\n", " if result.done:\n", " break\n", " obs = result.observation\n", "print(f\"cached {total} specialist votes across {len(list_task_ids())} tasks\")" ] }, { "cell_type": "code", "execution_count": null, "id": "6af95f46", "metadata": {}, "outputs": [], "source": [ "# Pre-training baseline rollout (specialists from cache, moderator = base Gemma)\n", "!python -m training.eval_rollout --out training/runs/eval_pre\n", "!cat training/runs/eval_pre/summary.csv" ] }, { "cell_type": "code", "execution_count": null, "id": "train", "metadata": {}, "outputs": [], "source": [ "# GRPO on the moderator LoRA. Specialist votes are read from cache —\n", "# only the moderator is re-rolled per training step, so this is fast.\n", "#\n", "# Knobs: --num-generations is K candidates per prompt (GRPO group size).\n", "# On a free T4 keep batch_size=1, grad_accum=8.\n", "!python -m training.train_grpo \\\n", " --epochs 2 \\\n", " --num-generations 8 \\\n", " --batch-size 1 \\\n", " --grad-accum 8 \\\n", " --lora-rank 16 \\\n", " --lr 5e-6" ] }, { "cell_type": "code", "execution_count": null, "id": "eval", "metadata": {}, "outputs": [], "source": [ "# Post-training eval — load the trained adapter into the same client.\n", "import glob, os\n", "RUN_DIR = sorted(glob.glob(\"training/runs/grpo_*\"))[-1]\n", "LORA_DIR = os.path.join(RUN_DIR, \"moderator-lora\")\n", "print(\"Using LoRA:\", LORA_DIR)\n", "\n", "# Attach adapter and re-run eval. The adapter name \"moderator\" matches what\n", "# Moderator.decide() requests via extra_body={'lora_request': {'name': ...}}\n", "from peft import PeftModel\n", "client.model = PeftModel.from_pretrained(client.model, LORA_DIR, adapter_name=\"moderator\")\n", "client.moderator_lora = \"moderator\"\n", "\n", "!python -m training.eval_rollout --moderator-lora moderator --out training/runs/eval_post\n", "!cat training/runs/eval_post/summary.csv" ] }, { "cell_type": "code", "execution_count": null, "id": "b1d888ed", "metadata": {}, "outputs": [], "source": [ "# Show training + eval curves inline\n", "from IPython.display import Image, display\n", "import os\n", "for p in [f\"{RUN_DIR}/loss.png\", f\"{RUN_DIR}/reward.png\",\n", " \"training/runs/eval_pre/reward_curve.png\",\n", " \"training/runs/eval_post/reward_curve.png\",\n", " \"training/runs/eval_pre/portfolio_curve.png\",\n", " \"training/runs/eval_post/portfolio_curve.png\"]:\n", " if os.path.exists(p):\n", " print(p)\n", " display(Image(p))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.13" } }, "nbformat": 4, "nbformat_minor": 5 }