"""
PERMANENCE — Pre-submission validation script.
Run this before every git push to catch issues early.
Usage (from anywhere):
python tools/validate_submission.py
All checks must pass before the repo is submitted.
"""
from __future__ import annotations
import os
import pathlib
import sys
# Always run from project root regardless of invocation cwd.
_THIS = pathlib.Path(__file__).resolve()
_PROJECT_ROOT = _THIS.parent.parent
os.chdir(_PROJECT_ROOT)
passed: list[str] = []
failed: list[str] = []
def OK(msg: str) -> None:
passed.append(msg)
print(f" ✓ {msg}")
def FAIL(msg: str, detail: str = "") -> None:
failed.append(msg)
print(f" ✗ {msg}" + (f": {detail}" if detail else ""))
print("=" * 65)
print("PERMANENCE SUBMISSION VALIDATION")
print("=" * 65)
print(f"Running from: {_PROJECT_ROOT}")
# ── 1. Required files exist ──────────────────────────────────────
print("\n[1] Required files")
required_files = [
"openenv.yaml",
"pyproject.toml",
"README.md",
"models.py",
"client.py",
"server/__init__.py",
"server/permanence_server.py",
"server/app.py",
"server/requirements.txt",
# training pipeline
"training/pipeline.py",
"training/rewards.py",
"training/stages/stage_1_sft.py",
"training/stages/stage_2_gate.py",
"training/stages/stage_3_grpo.py",
"training/stages/stage_4_eval.py",
"training/evaluate.py",
"training/config.yaml",
"training/config.py",
"training/warmup_traces.jsonl",
# Core env modules
"permanence/env.py",
"permanence/openenv_env.py",
"permanence/reward/rubrics.py",
"permanence/world/dynamics.py",
"permanence/world/fs.py",
"permanence/world/git.py",
"permanence/world/db.py",
"permanence/tasks/task_bank.py",
"permanence/domains/devtools/tasks.py",
"permanence/domains/devtools/actions.py",
"permanence/domains/devtools/register.py",
"permanence/domains/devtools/forced_variants.py",
"permanence/domains/meridian/tasks.py",
"permanence/domains/meridian/actions.py",
"permanence/domains/meridian/register.py",
"permanence/core/registry.py",
"permanence/core/interfaces.py",
"permanence/actions/database_actions.py",
# Demos + deploy
"demos/interactive_eval.py",
"demos/export_ghost_demo.py",
"demos/dashboard_server.py",
"deploy/serving/Dockerfile",
"deploy/training/Dockerfile",
"deploy/training/entrypoint.sh",
"tools/render_results.py",
"tools/upload_all.py",
]
for f in required_files:
if pathlib.Path(f).exists():
OK(f)
else:
FAIL(f"MISSING: {f}")
# ── 2. openenv.yaml fields ───────────────────────────────────────
print("\n[2] openenv.yaml")
try:
import yaml
spec = yaml.safe_load(pathlib.Path("openenv.yaml").read_text())
OK("openenv.yaml parses") if spec else FAIL("openenv.yaml empty")
OK("author: chanikya") if spec.get("author") == "chanikya" else FAIL(
f"author is '{spec.get('author')}' not 'chanikya'"
)
OK("spec_version present") if "spec_version" in spec else FAIL("spec_version missing")
OK("entry_point present") if "entry_point" in spec else FAIL("entry_point missing")
OK("app block present") if "app" in spec else FAIL("app block missing")
OK(f"{len(spec.get('tasks', []))} tasks defined") if len(spec.get("tasks", [])) >= 5 else FAIL(
f"Expected at least 5 tasks, got {len(spec.get('tasks', []))}"
)
OK("tags include openenv") if "openenv" in spec.get("tags", []) else FAIL(
"openenv tag missing"
)
except Exception as e:
FAIL(f"openenv.yaml error: {e}")
# ── 3. pyproject.toml ────────────────────────────────────────────
print("\n[3] pyproject.toml")
try:
import tomllib
d = tomllib.load(open("pyproject.toml", "rb"))
author = d["project"]["authors"][0].get("name", "")
OK("author: Chanikya") if author == "Chanikya" else FAIL(
f"author is '{author}' not 'Chanikya'"
)
OK("license: MIT") if d["project"]["license"]["text"] == "MIT" else FAIL("license not MIT")
except Exception as e:
FAIL(f"pyproject.toml error: {e}")
# ── 4. README has HF frontmatter ─────────────────────────────────
print("\n[4] README.md HuggingFace frontmatter")
try:
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
OK("Starts with ---") if readme.startswith("---") else FAIL(
"README must start with --- (HF frontmatter)"
)
OK("sdk: docker") if "sdk: docker" in readme else FAIL(
"sdk: docker missing from frontmatter"
)
OK("openenv tag") if "openenv" in readme[:500] else FAIL(
"openenv tag missing from frontmatter"
)
except Exception as e:
FAIL(f"README error: {e}")
# ── 5. OpenEnv compliance ────────────────────────────────────────
print("\n[5] OpenEnv compliance")
try:
# Ensure project root on path so we can import "models", "permanence", etc.
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
from openenv.core import Environment, Observation, Action, State
from permanence.openenv_env import PermanenceOpenEnv
from models import PermanenceAction, PermanenceObservation, PermanenceState
OK("PermanenceOpenEnv inherits Environment") if issubclass(
PermanenceOpenEnv, Environment
) else FAIL("PermanenceOpenEnv does not inherit from openenv.core.Environment")
OK("PermanenceAction inherits Action") if issubclass(
PermanenceAction, Action
) else FAIL("PermanenceAction does not inherit from openenv.core.Action")
OK("PermanenceObservation inherits Observation") if issubclass(
PermanenceObservation, Observation
) else FAIL("PermanenceObservation does not inherit from openenv.core.Observation")
OK("PermanenceState inherits State") if issubclass(
PermanenceState, State
) else FAIL("PermanenceState does not inherit from openenv.core.State")
# Test reset/step/state
env = PermanenceOpenEnv()
obs = env.reset(seed=42)
OK("reset() returns PermanenceObservation") if isinstance(
obs, PermanenceObservation
) else FAIL(f"reset() returns {type(obs)}")
action = PermanenceAction(
text=''
)
obs2 = env.step(action)
OK("step() returns PermanenceObservation") if isinstance(
obs2, PermanenceObservation
) else FAIL(f"step() returns {type(obs2)}")
st = env.state
OK("state property returns PermanenceState") if isinstance(
st, PermanenceState
) else FAIL(f"state returns {type(st)}")
meta = env.get_metadata()
OK(f"get_metadata().name = {meta.name}")
# Rubric tree
from openenv.core.rubrics.base import Rubric
OK("rubric attribute is a Rubric") if isinstance(env.rubric, Rubric) else FAIL(
f"env.rubric is {type(env.rubric)}, not a Rubric"
)
child_count = sum(1 for _ in env.rubric.named_children())
OK(f"Rubric has {child_count} composable children") if child_count >= 4 else FAIL(
f"Rubric only has {child_count} children; expected >=4"
)
env.close()
OK("close() works")
except Exception as e:
FAIL(f"OpenEnv compliance error: {e}")
# ── 6. Server app endpoints ──────────────────────────────────────
print("\n[6] server/app.py endpoints")
try:
from fastapi.testclient import TestClient
from server.app import app
client = TestClient(app)
r = client.get("/health")
OK("/health returns 200") if r.status_code == 200 else FAIL(
f"/health returns {r.status_code}"
)
r = client.post("/reset", json={})
OK("/reset with empty body returns 200") if r.status_code == 200 else FAIL(
f"/reset{{}} returns {r.status_code}: {r.text[:200]}"
)
r = client.get("/state")
OK("/state returns 200") if r.status_code == 200 else FAIL(
f"/state returns {r.status_code}"
)
r = client.get("/schema")
OK("/schema returns 200") if r.status_code == 200 else FAIL(
f"/schema returns {r.status_code}"
)
r = client.get("/metadata")
OK("/metadata returns 200") if r.status_code == 200 else FAIL(
f"/metadata returns {r.status_code}"
)
r = client.get("/api/rubric")
OK("/api/rubric returns 200") if r.status_code == 200 else FAIL(
f"/api/rubric returns {r.status_code}"
)
r = client.get("/dashboard")
OK("/dashboard returns 200") if r.status_code == 200 else FAIL(
f"/dashboard returns {r.status_code}"
)
except Exception as e:
FAIL(f"server/app.py error: {e}")
# ── 7. Dockerfile(s) ────────────────────────────────────────
print("\n[7] Dockerfiles")
try:
serving_df = pathlib.Path("deploy/serving/Dockerfile").read_text()
OK("serving FROM python") if "FROM python" in serving_df else FAIL("serving: missing FROM python")
OK("serving EXPOSE 7860") if "7860" in serving_df else FAIL("serving: missing EXPOSE 7860")
OK("serving HEALTHCHECK") if "HEALTHCHECK" in serving_df else FAIL("serving: missing HEALTHCHECK")
OK("serving uvicorn CMD") if "uvicorn" in serving_df and "CMD" in serving_df else FAIL("serving: missing uvicorn CMD")
training_df = pathlib.Path("deploy/training/Dockerfile").read_text()
OK("training FROM cuda") if "nvidia/cuda" in training_df else FAIL("training: missing cuda base image")
OK("training installs unsloth") if "unsloth" in training_df else FAIL("training: no unsloth install")
OK("training EXPOSE 7860") if "7860" in training_df else FAIL("training: missing EXPOSE 7860")
except Exception as e:
FAIL(f"Dockerfile error: {e}")
# ── 8. Core env imports ──────────────────────────────────────────
print("\n[8] permanence package")
try:
from permanence.env import PermanenceEnv
env = PermanenceEnv()
obs, info = env.reset()
OK("PermanenceEnv.reset() works")
assert "text" in obs, f"obs missing 'text': {obs}"
OK("reset() returns obs with text field")
_, reward, terminated, truncated, info = env.step(
""
)
OK("PermanenceEnv.step() works")
# New systems
from permanence.reward.rubrics import build_permanence_rubric
rubric = build_permanence_rubric()
OK("composable rubric builds")
from permanence.world.dynamics import apply_latent_dynamics
OK("latent dynamics module loads")
from permanence.actions.registry import ACTION_REGISTRY
OK(f"action registry has {len(ACTION_REGISTRY)} actions") if len(ACTION_REGISTRY) >= 25 else FAIL(
f"action registry smaller than expected: {len(ACTION_REGISTRY)}"
)
except Exception as e:
FAIL(f"permanence env error: {e}")
# ── 9. Training modules ──────────────────────────────────────────
print("\n[9] training modules")
try:
from training.rewards import (
reward_format,
build_reward_pack,
weighted_environmental_reward,
)
scores = reward_format(
["x"]
)
assert scores[0] >= 0.7, f"Expected >= 0.7, got {scores[0]}"
OK("reward_format produces high score on perfect output")
pack = build_reward_pack(total_episodes=100)
assert len(pack.funcs) == 1
OK("reward pack has 1 text-only reward function (env reward added at stage 3)")
assert callable(weighted_environmental_reward)
OK("weighted_environmental_reward exported for stage 3 wiring")
except Exception as e:
FAIL(f"rewards module error: {e}")
try:
from training import pipeline
assert pipeline.STAGES == ["sft", "gate", "grpo", "eval"]
OK(f"pipeline module exposes 4 stages: {pipeline.STAGES}")
except ImportError as e:
if "unsloth" in str(e).lower() or "torch" in str(e).lower() or "trl" in str(e).lower():
OK(f"pipeline.py skipped (GPU dependency: {e})")
else:
FAIL(f"pipeline.py import error: {e}")
except Exception as e:
FAIL(f"pipeline.py error: {e}")
try:
for stage_mod in [
"training.stages.stage_1_sft",
"training.stages.stage_2_gate",
"training.stages.stage_3_grpo",
"training.stages.stage_4_eval",
]:
__import__(stage_mod)
OK("all 4 pipeline stages importable")
except ImportError as e:
if "unsloth" in str(e).lower() or "torch" in str(e).lower() or "trl" in str(e).lower():
OK(f"pipeline stages skipped (GPU dependency: {e})")
else:
FAIL(f"stage import error: {e}")
except Exception as e:
FAIL(f"stage error: {e}")
# ── FINAL RESULT ─────────────────────────────────────────────────
print()
print("=" * 65)
n_ok, n_fail = len(passed), len(failed)
print(f"RESULTS: {n_ok} PASSED | {n_fail} FAILED")
print("=" * 65)
if n_fail > 0:
print("\nFAILED CHECKS:")
for f in failed:
print(f" ✗ {f}")
print("\nFix all failures before pushing.")
sys.exit(1)
else:
print("\n✓ ALL CHECKS PASSED — REPO IS SUBMISSION-READY")
sys.exit(0)