""" PERMANENCE — Pre-submission validation script. Run this before every git push to catch issues early. Usage (from anywhere): python tools/validate_submission.py All checks must pass before the repo is submitted. """ from __future__ import annotations import os import pathlib import sys # Always run from project root regardless of invocation cwd. _THIS = pathlib.Path(__file__).resolve() _PROJECT_ROOT = _THIS.parent.parent os.chdir(_PROJECT_ROOT) passed: list[str] = [] failed: list[str] = [] def OK(msg: str) -> None: passed.append(msg) print(f" ✓ {msg}") def FAIL(msg: str, detail: str = "") -> None: failed.append(msg) print(f" ✗ {msg}" + (f": {detail}" if detail else "")) print("=" * 65) print("PERMANENCE SUBMISSION VALIDATION") print("=" * 65) print(f"Running from: {_PROJECT_ROOT}") # ── 1. Required files exist ────────────────────────────────────── print("\n[1] Required files") required_files = [ "openenv.yaml", "pyproject.toml", "README.md", "models.py", "client.py", "server/__init__.py", "server/permanence_server.py", "server/app.py", "server/requirements.txt", # training pipeline "training/pipeline.py", "training/rewards.py", "training/stages/stage_1_sft.py", "training/stages/stage_2_gate.py", "training/stages/stage_3_grpo.py", "training/stages/stage_4_eval.py", "training/evaluate.py", "training/config.yaml", "training/config.py", "training/warmup_traces.jsonl", # Core env modules "permanence/env.py", "permanence/openenv_env.py", "permanence/reward/rubrics.py", "permanence/world/dynamics.py", "permanence/world/fs.py", "permanence/world/git.py", "permanence/world/db.py", "permanence/tasks/task_bank.py", "permanence/domains/devtools/tasks.py", "permanence/domains/devtools/actions.py", "permanence/domains/devtools/register.py", "permanence/domains/devtools/forced_variants.py", "permanence/domains/meridian/tasks.py", "permanence/domains/meridian/actions.py", "permanence/domains/meridian/register.py", "permanence/core/registry.py", "permanence/core/interfaces.py", "permanence/actions/database_actions.py", # Demos + deploy "demos/interactive_eval.py", "demos/export_ghost_demo.py", "demos/dashboard_server.py", "deploy/serving/Dockerfile", "deploy/training/Dockerfile", "deploy/training/entrypoint.sh", "tools/render_results.py", "tools/upload_all.py", ] for f in required_files: if pathlib.Path(f).exists(): OK(f) else: FAIL(f"MISSING: {f}") # ── 2. openenv.yaml fields ─────────────────────────────────────── print("\n[2] openenv.yaml") try: import yaml spec = yaml.safe_load(pathlib.Path("openenv.yaml").read_text()) OK("openenv.yaml parses") if spec else FAIL("openenv.yaml empty") OK("author: chanikya") if spec.get("author") == "chanikya" else FAIL( f"author is '{spec.get('author')}' not 'chanikya'" ) OK("spec_version present") if "spec_version" in spec else FAIL("spec_version missing") OK("entry_point present") if "entry_point" in spec else FAIL("entry_point missing") OK("app block present") if "app" in spec else FAIL("app block missing") OK(f"{len(spec.get('tasks', []))} tasks defined") if len(spec.get("tasks", [])) >= 5 else FAIL( f"Expected at least 5 tasks, got {len(spec.get('tasks', []))}" ) OK("tags include openenv") if "openenv" in spec.get("tags", []) else FAIL( "openenv tag missing" ) except Exception as e: FAIL(f"openenv.yaml error: {e}") # ── 3. pyproject.toml ──────────────────────────────────────────── print("\n[3] pyproject.toml") try: import tomllib d = tomllib.load(open("pyproject.toml", "rb")) author = d["project"]["authors"][0].get("name", "") OK("author: Chanikya") if author == "Chanikya" else FAIL( f"author is '{author}' not 'Chanikya'" ) OK("license: MIT") if d["project"]["license"]["text"] == "MIT" else FAIL("license not MIT") except Exception as e: FAIL(f"pyproject.toml error: {e}") # ── 4. README has HF frontmatter ───────────────────────────────── print("\n[4] README.md HuggingFace frontmatter") try: readme = pathlib.Path("README.md").read_text(encoding="utf-8") OK("Starts with ---") if readme.startswith("---") else FAIL( "README must start with --- (HF frontmatter)" ) OK("sdk: docker") if "sdk: docker" in readme else FAIL( "sdk: docker missing from frontmatter" ) OK("openenv tag") if "openenv" in readme[:500] else FAIL( "openenv tag missing from frontmatter" ) except Exception as e: FAIL(f"README error: {e}") # ── 5. OpenEnv compliance ──────────────────────────────────────── print("\n[5] OpenEnv compliance") try: # Ensure project root on path so we can import "models", "permanence", etc. if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) from openenv.core import Environment, Observation, Action, State from permanence.openenv_env import PermanenceOpenEnv from models import PermanenceAction, PermanenceObservation, PermanenceState OK("PermanenceOpenEnv inherits Environment") if issubclass( PermanenceOpenEnv, Environment ) else FAIL("PermanenceOpenEnv does not inherit from openenv.core.Environment") OK("PermanenceAction inherits Action") if issubclass( PermanenceAction, Action ) else FAIL("PermanenceAction does not inherit from openenv.core.Action") OK("PermanenceObservation inherits Observation") if issubclass( PermanenceObservation, Observation ) else FAIL("PermanenceObservation does not inherit from openenv.core.Observation") OK("PermanenceState inherits State") if issubclass( PermanenceState, State ) else FAIL("PermanenceState does not inherit from openenv.core.State") # Test reset/step/state env = PermanenceOpenEnv() obs = env.reset(seed=42) OK("reset() returns PermanenceObservation") if isinstance( obs, PermanenceObservation ) else FAIL(f"reset() returns {type(obs)}") action = PermanenceAction( text='' ) obs2 = env.step(action) OK("step() returns PermanenceObservation") if isinstance( obs2, PermanenceObservation ) else FAIL(f"step() returns {type(obs2)}") st = env.state OK("state property returns PermanenceState") if isinstance( st, PermanenceState ) else FAIL(f"state returns {type(st)}") meta = env.get_metadata() OK(f"get_metadata().name = {meta.name}") # Rubric tree from openenv.core.rubrics.base import Rubric OK("rubric attribute is a Rubric") if isinstance(env.rubric, Rubric) else FAIL( f"env.rubric is {type(env.rubric)}, not a Rubric" ) child_count = sum(1 for _ in env.rubric.named_children()) OK(f"Rubric has {child_count} composable children") if child_count >= 4 else FAIL( f"Rubric only has {child_count} children; expected >=4" ) env.close() OK("close() works") except Exception as e: FAIL(f"OpenEnv compliance error: {e}") # ── 6. Server app endpoints ────────────────────────────────────── print("\n[6] server/app.py endpoints") try: from fastapi.testclient import TestClient from server.app import app client = TestClient(app) r = client.get("/health") OK("/health returns 200") if r.status_code == 200 else FAIL( f"/health returns {r.status_code}" ) r = client.post("/reset", json={}) OK("/reset with empty body returns 200") if r.status_code == 200 else FAIL( f"/reset{{}} returns {r.status_code}: {r.text[:200]}" ) r = client.get("/state") OK("/state returns 200") if r.status_code == 200 else FAIL( f"/state returns {r.status_code}" ) r = client.get("/schema") OK("/schema returns 200") if r.status_code == 200 else FAIL( f"/schema returns {r.status_code}" ) r = client.get("/metadata") OK("/metadata returns 200") if r.status_code == 200 else FAIL( f"/metadata returns {r.status_code}" ) r = client.get("/api/rubric") OK("/api/rubric returns 200") if r.status_code == 200 else FAIL( f"/api/rubric returns {r.status_code}" ) r = client.get("/dashboard") OK("/dashboard returns 200") if r.status_code == 200 else FAIL( f"/dashboard returns {r.status_code}" ) except Exception as e: FAIL(f"server/app.py error: {e}") # ── 7. Dockerfile(s) ──────────────────────────────────────── print("\n[7] Dockerfiles") try: serving_df = pathlib.Path("deploy/serving/Dockerfile").read_text() OK("serving FROM python") if "FROM python" in serving_df else FAIL("serving: missing FROM python") OK("serving EXPOSE 7860") if "7860" in serving_df else FAIL("serving: missing EXPOSE 7860") OK("serving HEALTHCHECK") if "HEALTHCHECK" in serving_df else FAIL("serving: missing HEALTHCHECK") OK("serving uvicorn CMD") if "uvicorn" in serving_df and "CMD" in serving_df else FAIL("serving: missing uvicorn CMD") training_df = pathlib.Path("deploy/training/Dockerfile").read_text() OK("training FROM cuda") if "nvidia/cuda" in training_df else FAIL("training: missing cuda base image") OK("training installs unsloth") if "unsloth" in training_df else FAIL("training: no unsloth install") OK("training EXPOSE 7860") if "7860" in training_df else FAIL("training: missing EXPOSE 7860") except Exception as e: FAIL(f"Dockerfile error: {e}") # ── 8. Core env imports ────────────────────────────────────────── print("\n[8] permanence package") try: from permanence.env import PermanenceEnv env = PermanenceEnv() obs, info = env.reset() OK("PermanenceEnv.reset() works") assert "text" in obs, f"obs missing 'text': {obs}" OK("reset() returns obs with text field") _, reward, terminated, truncated, info = env.step( "" ) OK("PermanenceEnv.step() works") # New systems from permanence.reward.rubrics import build_permanence_rubric rubric = build_permanence_rubric() OK("composable rubric builds") from permanence.world.dynamics import apply_latent_dynamics OK("latent dynamics module loads") from permanence.actions.registry import ACTION_REGISTRY OK(f"action registry has {len(ACTION_REGISTRY)} actions") if len(ACTION_REGISTRY) >= 25 else FAIL( f"action registry smaller than expected: {len(ACTION_REGISTRY)}" ) except Exception as e: FAIL(f"permanence env error: {e}") # ── 9. Training modules ────────────────────────────────────────── print("\n[9] training modules") try: from training.rewards import ( reward_format, build_reward_pack, weighted_environmental_reward, ) scores = reward_format( ["x"] ) assert scores[0] >= 0.7, f"Expected >= 0.7, got {scores[0]}" OK("reward_format produces high score on perfect output") pack = build_reward_pack(total_episodes=100) assert len(pack.funcs) == 1 OK("reward pack has 1 text-only reward function (env reward added at stage 3)") assert callable(weighted_environmental_reward) OK("weighted_environmental_reward exported for stage 3 wiring") except Exception as e: FAIL(f"rewards module error: {e}") try: from training import pipeline assert pipeline.STAGES == ["sft", "gate", "grpo", "eval"] OK(f"pipeline module exposes 4 stages: {pipeline.STAGES}") except ImportError as e: if "unsloth" in str(e).lower() or "torch" in str(e).lower() or "trl" in str(e).lower(): OK(f"pipeline.py skipped (GPU dependency: {e})") else: FAIL(f"pipeline.py import error: {e}") except Exception as e: FAIL(f"pipeline.py error: {e}") try: for stage_mod in [ "training.stages.stage_1_sft", "training.stages.stage_2_gate", "training.stages.stage_3_grpo", "training.stages.stage_4_eval", ]: __import__(stage_mod) OK("all 4 pipeline stages importable") except ImportError as e: if "unsloth" in str(e).lower() or "torch" in str(e).lower() or "trl" in str(e).lower(): OK(f"pipeline stages skipped (GPU dependency: {e})") else: FAIL(f"stage import error: {e}") except Exception as e: FAIL(f"stage error: {e}") # ── FINAL RESULT ───────────────────────────────────────────────── print() print("=" * 65) n_ok, n_fail = len(passed), len(failed) print(f"RESULTS: {n_ok} PASSED | {n_fail} FAILED") print("=" * 65) if n_fail > 0: print("\nFAILED CHECKS:") for f in failed: print(f" ✗ {f}") print("\nFix all failures before pushing.") sys.exit(1) else: print("\n✓ ALL CHECKS PASSED — REPO IS SUBMISSION-READY") sys.exit(0)