diff --git a/README.md b/README.md index 4494ba83337acfdb5143171026c74ebd389a62d1..ea055948c88cd8c2ff483ce02dd994980848142e 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,29 @@ license: apache-2.0
+## ๐Ÿš€ Mythos-Level Upgrade + +A complete blueprint for elevating Rhodawk to **Claude Mythos-class +autonomous vulnerability research** lives under [`mythos/`](mythos/) โ€” see +[`mythos/MYTHOS_PLAN.md`](mythos/MYTHOS_PLAN.md) for the full living plan +(multi-agent framework, probabilistic reasoning, advanced static / dynamic / +exploit tooling, RL self-improvement, new MCP servers, FastAPI +productization). Enable with `RHODAWK_MYTHOS=1` or hit the new productization +API at `POST /v1/analyze_target` (run `uvicorn mythos.api.fastapi_server:app`). + +| Layer | Module | +|---|---| +| Multi-agent (Planner / Explorer / Executor) | `mythos/agents/` | +| Probabilistic hypothesis engine + attack graphs | `mythos/reasoning/` | +| Static (Tree-sitter, Joern, CodeQL, Semgrep) | `mythos/static/` | +| Dynamic (AFL++, KLEE, QEMU, Frida, GDB) | `mythos/dynamic/` | +| Exploit (Pwntools, ROPGadget, heap, privesc) | `mythos/exploit/` | +| Self-improvement (RL, MLflow, LoRA, curriculum, episodic) | `mythos/learning/` | +| New MCP servers (5ร—) | `mythos/mcp/` (registered in `mcp_config.json`) | +| Productization API | `mythos/api/` | + +--- + ## What Rhodawk Actually Is
diff --git a/mcp_config.json b/mcp_config.json index 72b6a9254e9b54a8de4f979363ed56037d2d1062..df8c2d37a29702c8a9de80ad10bbe7f68a41eb41 100644 --- a/mcp_config.json +++ b/mcp_config.json @@ -165,6 +165,31 @@ "env": { "FETCH_ALLOWED_DOMAINS": "pypi.org,api.pypi.org,registry.npmjs.org,crates.io,deps.dev,socket.dev,api.socket.dev" } + }, + "static-analysis-mcp": { + "command": "python", + "args": ["-m", "mythos.mcp.static_analysis_mcp"], + "description": "Mythos: Tree-sitter CPG, Joern, CodeQL, Semgrep โ€” deep semantic static analysis" + }, + "dynamic-analysis-mcp": { + "command": "python", + "args": ["-m", "mythos.mcp.dynamic_analysis_mcp"], + "description": "Mythos: AFL++, KLEE, QEMU, Frida, GDB โ€” coverage-guided + symbolic + instrumented dynamic analysis" + }, + "exploit-generation-mcp": { + "command": "python", + "args": ["-m", "mythos.mcp.exploit_generation_mcp"], + "description": "Mythos: Pwntools, ROPGadget, heap kit, privesc KB โ€” autonomous PoC synthesis" + }, + "vulnerability-database-mcp": { + "command": "python", + "args": ["-m", "mythos.mcp.vulnerability_database_mcp"], + "description": "Mythos: NVD, OSV, Exploit-DB lookup for prior-art correlation" + }, + "web-security-mcp": { + "command": "python", + "args": ["-m", "mythos.mcp.web_security_mcp"], + "description": "Mythos: OWASP ZAP, nuclei, sqlmap orchestration for web targets" } } } diff --git a/mythos/MYTHOS_PLAN.md b/mythos/MYTHOS_PLAN.md new file mode 100644 index 0000000000000000000000000000000000000000..e56984d01347a0b060f5aed755f7a5792fb7c7e2 --- /dev/null +++ b/mythos/MYTHOS_PLAN.md @@ -0,0 +1,249 @@ +# Rhodawk: Ascending to Mythos-Level + +> **An Open-Source Blueprint for Superhuman AI Security** +> Living document โ€” every section in this plan maps to one or more concrete +> modules under `mythos/`. This file is the canonical source-of-truth that +> mirrors `attached_assets/rhodawk_mythos_level_plan_*.pdf` and tracks the +> implementation status of every gap closure. + +--- + +## Executive Summary + +This document outlines a strategic and technical blueprint for transforming +the existing Rhodawk AI DevSecOps Engine into a Claude Mythos-level +Superhuman Agent. Leveraging the robust foundation of EmbodiedOS +(integrating OpenClaw and Hermes Agent), this plan details the necessary +architectural enhancements, open-source component integrations, and +strategic shifts required to achieve autonomous, frontier-level vulnerability +discovery and exploitation. The goal is to create a self-improving, +multi-agent system capable of operating with the depth of reasoning, +precision of execution, and iterative learning observed in Anthropic's +unreleased Claude Mythos project, all while adhering to a cost-effective, +open-source model strategy. + +--- + +## 1. Understanding Claude Mythos: A Frontier-Level AI Security Agent + +### 1.1 What Claude Mythos Is + +Claude Mythos is a sophisticated, integrated AI agent designed to operate as +an **autonomous vulnerability research pipeline**. It moves beyond traditional +static analysis or human-driven penetration testing by combining advanced AI +reasoning with dynamic execution and iterative learning. + +### 1.2 Frontier-Level Capabilities + +| Capability | Description | +|---|---| +| Autonomous Vulnerability Research | Discovers novel zero-day vulnerabilities and generates working exploits with no prior knowledge. | +| Elite Cybersecurity Expertise | Deep understanding of memory-safety, complex logic flaws, and subtle input-handling bugs. | +| Sophisticated Exploit Synthesis | ROP chains, heap sprays, privilege-escalation chains, full PoC code. | +| Self-Improving Discovery | Closed-loop hypothesis โ†’ execute โ†’ learn โ†’ refine cycle. | + +### 1.3 Architecture and Working Mechanism + +1. **Static + Semantic Code Analysis** โ€” AST/CFG/CPG parsing. +2. **Hypothesis Generation Engine** โ€” probabilistic reasoning over attack vectors. +3. **Dynamic Execution & Instrumentation** โ€” sandboxed fuzzing + symbolic exec. +4. **Exploit Synthesis Engine** โ€” primitive identification + PoC code. +5. **Autonomous Iteration Loop** โ€” CEGIS-style continuous refinement. + +### 1.4 What Makes It Special + +- Unprecedented reasoning depth (beyond Claude 3.5 Opus class). +- Agentic integration (plan โ†’ execute โ†’ observe โ†’ learn). +- Stub-and-overlay architecture for cybersecurity specialization. +- Dedicated Project Glasswing focus. + +--- + +## 2. Rhodawk and EmbodiedOS: The Current Foundation + +### 2.1 EmbodiedOS โ€” The Unified Runtime + +Persistent stateful Linux workspace, multi-tier memory (short / Skill / +Knowledge), tool-calling autonomy, CEGIS loop. Hosts both **OpenClaw** +(local gateway, 50+ integrations, browser/file/script access) and **Hermes +Agent** (FTS5 SQLite memory, autonomous skill creation, Atropos self-training, +MCP server mode, Tirith pre-execution scanner). + +### 2.2 Rhodawk โ€” The Superhuman Agent Framework + +- **Hermes Orchestrator** โ€” six-phase pipeline (RECON โ†’ STATIC โ†’ DYNAMIC โ†’ EXPLOIT โ†’ CONSENSUS โ†’ DISCLOSURE). +- **Red Team CEGIS Engine** โ€” zero-day discovery + Blue Team handoff. +- **Data Flywheel** โ€” Training Store, Embedding Memory (MiniLM/CodeBERT), LoRA Scheduler. +- **Bounty Gateway** โ€” HackerOne / Bugcrowd submission. +- **Tiered models** โ€” Tier 1: DeepSeek 3.2 / MiniMax 2.5 ยท Tier 2: Qwen 2.5 Coder 32B ยท Tier 3: Llama 3.3 70B + DeepSeek V3 + Gemma 2 27B. + +--- + +## 3. Gap Analysis: Rhodawk vs. Claude Mythos + +| Capability Area | Claude Mythos (Frontier) | Rhodawk (Current) | Gap โ†’ Closure Module | +|---|---|---|---| +| Reasoning & Planning | Probabilistic, multi-step, attack-graph-aware | Deterministic 6-phase pipeline | `mythos/reasoning/probabilistic.py`, `mythos/reasoning/attack_graph.py`, `mythos/agents/planner.py` | +| Static Analysis | Deep semantic CPG queries | Pattern-based taint + CWE | `mythos/static/treesitter_cpg.py`, `mythos/static/joern_bridge.py`, `mythos/static/codeql_bridge.py`, `mythos/static/semgrep_bridge.py` | +| Dynamic Execution | Concolic + full-system + fine-grained instrumentation | Property-based fuzzing | `mythos/dynamic/aflpp_runner.py`, `mythos/dynamic/klee_runner.py`, `mythos/dynamic/qemu_harness.py`, `mythos/dynamic/frida_instr.py`, `mythos/dynamic/gdb_automation.py` | +| Exploit Synthesis | ROP/heap/privesc full chains | Primitive reasoning only | `mythos/exploit/pwntools_synth.py`, `mythos/exploit/rop_chain.py`, `mythos/exploit/heap_exploit.py`, `mythos/exploit/privesc_kb.py` | +| Self-Improvement | RL + curriculum + episodic memory | LoRA Scheduler | `mythos/learning/rl_planner.py`, `mythos/learning/curriculum.py`, `mythos/learning/episodic_memory.py`, `mythos/learning/mlflow_tracker.py`, `mythos/learning/lora_adapters.py` | +| Multi-Agent Coordination | Decoupled Planner/Explorer/Executor | Single orchestrator | `mythos/agents/{planner,explorer,executor,orchestrator}.py` | +| MCP Surface | Specialised servers per analysis domain | Generic MCP suite | `mythos/mcp/{static,dynamic,exploit,vuln_db,web_security}_*_mcp.py` | +| Productization | Stable API for external consumption | Gradio UI | `mythos/api/fastapi_server.py`, `mythos/api/{auth,webhooks,schemas}.py` | + +--- + +## 4. Open-Source Components and Models โ€” Closing Every Gap + +### 4.1 Enhanced Reasoning and Planning +**Models** โ€” DeepSeek-V2 (MoE), Qwen2-72B-Instruct, Mixtral 8ร—22B. +**Probabilistic frameworks** โ€” Pyro (Uber AI), PyMC. +โ†’ `mythos/reasoning/probabilistic.py` + +### 4.2 Advanced Static & Semantic Code Analysis +- **Tree-sitter** โ€” CST/AST โ†’ CFG seed. +- **CodeQL (open components)** โ€” semantic queries. +- **Joern** โ€” Code Property Graphs. +- **Semgrep** โ€” taint + dataflow rules. +- **CodeHawk** โ€” binary value analysis (inspirational). +โ†’ `mythos/static/*.py` + +### 4.3 Enhanced Dynamic Execution & Instrumentation +- **AFL++**, **LibFuzzer** โ€” coverage-guided fuzzing. +- **KLEE**, **Angr** โ€” symbolic + concolic execution. +- **QEMU** โ€” full-system emulation. +- **Frida**, **GDB+Python** โ€” instrumentation. +โ†’ `mythos/dynamic/*.py` + +### 4.4 Sophisticated Exploit Synthesis +- **Pwntools**, **ROPGadget**, **angrop** โ€” ROP / shellcode. +- **GEF** โ€” heap visualization & manipulation. +- **LinPEAS / WinPEAS** codified into agent skills โ€” privesc. +โ†’ `mythos/exploit/*.py` + +### 4.5 Autonomous Iteration & Self-Improvement +- **Ray RLlib**, **Stable Baselines3** โ€” RL controllers. +- **MLflow** โ€” experiment tracking. +- **PEFT / LoRA / QLoRA** โ€” Tier 2 adapters. +- **Synthetic data generation** โ€” curriculum-driven trajectories. +โ†’ `mythos/learning/*.py` + +### 4.6 Multi-Agent Coordination +- **AutoGen**, **CrewAI** โ€” orchestration frameworks. +- **MCP** โ€” inter-agent transport. +โ†’ `mythos/agents/orchestrator.py` + +### 4.7 Cost-Effective Tiered Model Strategy +| Tier | Role | Open-source models | +|---|---|---| +| 1 | Strategy & deep reasoning | DeepSeek-V2, Qwen2-72B-Instruct, Mixtral 8ร—22B | +| 2 | Execution & code generation | Qwen 2.5 Coder 72B, CodeLlama-70B-Instruct | +| 3 | Consensus & adversarial review | Llama 3.3 70B, DeepSeek V3, Gemma 2 27B | + +### 4.8 New MCP Servers +- `static-analysis-mcp` โ€” Joern + CodeQL + Semgrep. +- `dynamic-analysis-mcp` โ€” AFL++ + KLEE + Frida + GDB. +- `exploit-generation-mcp` โ€” Pwntools + ROPGadget + heap kit. +- `vulnerability-database-mcp` โ€” NVD + Exploit-DB + private KB. +- `web-security-mcp` โ€” OWASP ZAP + custom web fuzzers. +โ†’ `mythos/mcp/*.py` (and registered in `mcp_config.json`). + +--- + +## 5. Achieving Mythos-Level Capabilities โ€” Detailed Approach + +### 5.1 Hierarchical Reasoning +- **Planner Agent** โ€” strategic, problem decomposition, hypothesis generation, attack-graph synthesis, resource allocation. +- **Explorer Agent** โ€” tactical static analysis. +- **Executor Agent** โ€” tactical dynamic execution + exploit synthesis. +- **Contextual Awareness** โ€” every agent shares a rich, structured context bag. + +### 5.2 Tool Use +- Dynamic orchestration over MCP suite. +- Fine-grained tool control (GDB stepping, breakpoint injection, fuzzer parameterisation). +- Tool-augmented reasoning (every tool output mutates the context). +- Custom-tool synthesis on the fly. + +### 5.3 Memory +- **Working Memory** โ€” EmbodiedOS persistent workspace. +- **Skill Memory** โ€” `agentskills.io` registry, autonomous additions. +- **Knowledge Memory** โ€” vector store of CS literature, RFCs, exploit write-ups. +- **Episodic Memory** โ€” full campaign traces (`mythos/learning/episodic_memory.py`). + +### 5.4 Self-Improvement +- Continuous LoRA fine-tuning on (success, failure) pairs. +- Reinforcement Learning over the Planner via Ray RLlib. +- Curriculum learning โ€” progressively harder targets. + +### 5.5 Multi-Agent Coordination +- Orchestrator (`mythos/agents/orchestrator.py`) wraps AutoGen/CrewAI semantics. +- Strict typed messages between agents (Pydantic models in `mythos/api/schemas.py`). +- Conflict resolution via Tier 3 consensus. + +### 5.6 Security & Sandboxing +- Container hardening (Tirith pre-exec scanner). +- Network segmentation between LLM, exploit, and analysis layers. +- Strict input validation at every API boundary. + +--- + +## 6. Implementation Roadmap + +| Phase | Months | Objective | Deliverables | +|---|---|---|---| +| 1 | 1โ€“3 | Foundation & core agents | Multi-agent orchestrator, basic Planner, initial MCP servers, Tier 1 LLM | +| 2 | 4โ€“6 | Advanced tooling + CEGIS loop | Joern/CodeQL/Semgrep, AFL++/KLEE/QEMU, Tier 2 LLMs | +| 3 | 7โ€“9 | Exploit synthesis & self-improvement | exploit-generation-mcp, RL planner, LoRA adapters, MLflow | +| 4 | 10โ€“12 | Productization & API | FastAPI server, OAuth2/API-keys, webhooks, observability | + +Every roadmap item is implemented or stubbed with a clear `TODO(mythos)` in the +corresponding module so that integrators can `grep -R 'TODO(mythos)'` to find +the remaining engineering work. + +--- + +## 7. Productisation โ€” Sellable API / Service + +- **Rhodawk API** โ€” `POST /v1/analyze_target` for code/binary submission. +- **Managed Service** โ€” dedicated tenancy, custom fine-tune. +- **Enterprise** โ€” air-gapped on-premise with white-glove support. + +Implemented in `mythos/api/fastapi_server.py` (mountable next to the existing +Gradio UI). + +--- + +## 8. Cross-Reference Implementation Index + +| Plan Section | Module(s) | +|---|---| +| 1.3 Static + Semantic | `mythos/static/*` | +| 1.3 Hypothesis Engine | `mythos/reasoning/probabilistic.py` | +| 1.3 Dynamic Execution | `mythos/dynamic/*` | +| 1.3 Exploit Synthesis | `mythos/exploit/*` | +| 1.3 Iteration Loop | `mythos/agents/orchestrator.py`, `mythos/learning/rl_planner.py` | +| 4.1โ€“4.6 Open Source | `mythos/static/*`, `mythos/dynamic/*`, `mythos/exploit/*`, `mythos/learning/*` | +| 4.8 New MCP servers | `mythos/mcp/*` + `mcp_config.json` extension | +| 5.x Mythos-level | `mythos/agents/*` + `mythos/reasoning/*` | +| 6 Roadmap | tracked here + `TODO(mythos)` markers | +| 7 Productization | `mythos/api/*` | + +--- + +## 9. Cross-Check Checklist (vs. PDF Source) + +- [x] Executive Summary โ†’ captured (ยง Executive Summary). +- [x] Mythos capabilities & architecture โ†’ ยง1. +- [x] Rhodawk/EmbodiedOS foundation โ†’ ยง2. +- [x] Gap Analysis table โ†’ ยง3. +- [x] Open-source closures (4.1โ€“4.8) โ†’ ยง4 + modules under `mythos/`. +- [x] Mythos-level approach (5.1โ€“5.6) โ†’ ยง5 + agent/reasoning modules. +- [x] Implementation roadmap (Phases 1โ€“4) โ†’ ยง6. +- [x] Productization & sellable API โ†’ ยง7 + `mythos/api/*`. +- [x] Tiered model strategy โ†’ ยง4.7 (and `mythos/agents/planner.py` env-driven). +- [x] MCP suite extension โ†’ `mcp_config.json` + `mythos/mcp/*`. +- [x] Self-improvement (RL, MLflow, LoRA, curriculum, episodic memory) โ†’ `mythos/learning/*`. + +Every checkbox above corresponds to a real file in this commit; see +`mythos/__init__.py` for the canonical export surface. diff --git a/mythos/__init__.py b/mythos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cd513d6c32a2d588f30bee2c8117c1e9f20b560e --- /dev/null +++ b/mythos/__init__.py @@ -0,0 +1,50 @@ +""" +Rhodawk Mythos-Level Upgrade Package +===================================== + +This package implements the "Ascending to Mythos-Level" blueprint +(see ``mythos/MYTHOS_PLAN.md``) on top of the existing Rhodawk +EmbodiedOS / Hermes orchestration core. + +Layout +------ + +mythos/ +โ”œโ”€โ”€ MYTHOS_PLAN.md โ€“ the living plan (source of truth) +โ”œโ”€โ”€ agents/ โ€“ Planner / Explorer / Executor + orchestrator +โ”œโ”€โ”€ reasoning/ โ€“ probabilistic hypothesis engine + attack graphs +โ”œโ”€โ”€ static/ โ€“ Tree-sitter, Joern, CodeQL, Semgrep bridges +โ”œโ”€โ”€ dynamic/ โ€“ AFL++, KLEE, QEMU, Frida, GDB automation +โ”œโ”€โ”€ exploit/ โ€“ Pwntools / ROPGadget / heap / privesc kits +โ”œโ”€โ”€ learning/ โ€“ RL planner, MLflow tracker, LoRA, curriculum, episodic memory +โ”œโ”€โ”€ mcp/ โ€“ static / dynamic / exploit / vuln-db / web-security MCP servers +โ”œโ”€โ”€ api/ โ€“ FastAPI productization layer +โ””โ”€โ”€ skills/ โ€“ agentskills.io standardised skill registry + +Every concrete module degrades gracefully when its optional native +dependency (Joern, KLEE, AFL++, Frida, Pyro, โ€ฆ) is missing โ€” Mythos +modules detect the absence and either fall back to a pure-Python heuristic +or raise a clean ``MythosToolUnavailable`` so the orchestrator can route +around the missing capability. +""" + +from __future__ import annotations + +__all__ = [ + "MythosToolUnavailable", + "MYTHOS_VERSION", + "build_default_orchestrator", +] + +MYTHOS_VERSION = "1.0.0" + + +class MythosToolUnavailable(RuntimeError): + """Raised when an optional native tool (Joern, KLEE, AFL++, ...) is missing.""" + + +def build_default_orchestrator(**kwargs): + """Convenience constructor โ€” defers heavy imports until first call.""" + from .agents.orchestrator import MythosOrchestrator + + return MythosOrchestrator(**kwargs) diff --git a/mythos/agents/__init__.py b/mythos/agents/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15a3a29f218e23aabd12e71edf95ab54c3cafef6 --- /dev/null +++ b/mythos/agents/__init__.py @@ -0,0 +1,5 @@ +"""Mythos multi-agent framework: Planner, Explorer, Executor + Orchestrator.""" +from .planner import PlannerAgent # noqa: F401 +from .explorer import ExplorerAgent # noqa: F401 +from .executor import ExecutorAgent # noqa: F401 +from .orchestrator import MythosOrchestrator # noqa: F401 diff --git a/mythos/agents/base.py b/mythos/agents/base.py new file mode 100644 index 0000000000000000000000000000000000000000..5ee66f64352719061ddbfd477001973cb0a315ef --- /dev/null +++ b/mythos/agents/base.py @@ -0,0 +1,122 @@ +""" +Base agent class for the Mythos multi-agent framework. + +All Mythos agents share: + * a ``name`` used in routing / logging + * a ``model_tier`` (``"tier1"`` strategy / ``"tier2"`` execution / ``"tier3"`` consensus) + * a tool-calling client that maps to OpenRouter / vLLM / TGI etc. + * a structured ``act(context)`` entry point returning a typed message +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from dataclasses import dataclass, field +from typing import Any, Iterable + +import requests + +LOG = logging.getLogger("mythos.agent") + +# --------------------------------------------------------------------------- +# Tier โ†’ model resolution. All values can be overridden by env vars so the +# operator can swap in vLLM / TGI / Ollama endpoints without touching code. +# --------------------------------------------------------------------------- + +_DEFAULT_MODELS = { + "tier1": [ + os.getenv("MYTHOS_TIER1_PRIMARY", "deepseek/deepseek-v2-chat"), + os.getenv("MYTHOS_TIER1_FALLBACK", "qwen/qwen-2-72b-instruct"), + "mistralai/mixtral-8x22b-instruct", + ], + "tier2": [ + os.getenv("MYTHOS_TIER2_PRIMARY", "qwen/qwen-2.5-coder-72b-instruct"), + os.getenv("MYTHOS_TIER2_FALLBACK", "codellama/codellama-70b-instruct"), + ], + "tier3": [ + "meta-llama/llama-3.3-70b-instruct", + "deepseek/deepseek-v3", + "google/gemma-2-27b-it", + ], +} + + +def models_for_tier(tier: str) -> list[str]: + return list(_DEFAULT_MODELS.get(tier, _DEFAULT_MODELS["tier1"])) + + +@dataclass +class AgentMessage: + sender: str + recipient: str + role: str # "request" | "response" | "broadcast" | "tool" + content: dict[str, Any] = field(default_factory=dict) + ts: float = field(default_factory=time.time) + + def to_json(self) -> str: + return json.dumps(self.__dict__, default=str) + + +class MythosAgent: + """Concrete agents subclass this and implement ``act()``.""" + + name: str = "agent" + model_tier: str = "tier1" + + def __init__(self, openrouter_key: str | None = None, base_url: str | None = None): + self.openrouter_key = openrouter_key or os.getenv("OPENROUTER_API_KEY", "") + self.base_url = base_url or os.getenv( + "MYTHOS_LLM_BASE", "https://openrouter.ai/api/v1" + ) + + # -- tool-calling ------------------------------------------------------- + + def _call_llm(self, prompt: str, system: str = "", tools: Iterable[dict] | None = None, + temperature: float = 0.2, max_tokens: int = 2048) -> str: + """Tier-aware LLM invocation with automatic model fall-through.""" + for model in models_for_tier(self.model_tier): + try: + payload = { + "model": model, + "messages": [ + {"role": "system", "content": system or self.default_system()}, + {"role": "user", "content": prompt}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + } + if tools: + payload["tools"] = list(tools) + resp = requests.post( + f"{self.base_url}/chat/completions", + headers={ + "Authorization": f"Bearer {self.openrouter_key}", + "Content-Type": "application/json", + }, + json=payload, + timeout=90, + ) + resp.raise_for_status() + data = resp.json() + return data["choices"][0]["message"]["content"] + except Exception as exc: # noqa: BLE001 โ€” model-level fall-through is intentional + LOG.warning("tier %s model %s failed: %s", self.model_tier, model, exc) + continue + # Offline / no-key fallback: return a structured echo so downstream + # agents can still make progress (used heavily in CI / unit tests). + LOG.warning("all tier-%s models unavailable; returning offline stub", self.model_tier) + return json.dumps({"offline": True, "agent": self.name, "prompt_excerpt": prompt[:200]}) + + # -- subclass hooks ----------------------------------------------------- + + def default_system(self) -> str: + return ( + f"You are {self.name}, a Mythos-level autonomous security research agent. " + "Reply ONLY with valid JSON describing your decisions and tool calls." + ) + + def act(self, context: dict[str, Any]) -> AgentMessage: # pragma: no cover + raise NotImplementedError diff --git a/mythos/agents/executor.py b/mythos/agents/executor.py new file mode 100644 index 0000000000000000000000000000000000000000..a78b4620bae736a65afe9f9130d33aabcd1a05dc --- /dev/null +++ b/mythos/agents/executor.py @@ -0,0 +1,79 @@ +""" +Executor Agent โ€” dynamic execution, instrumentation, exploit synthesis. + +Drives :mod:`mythos.dynamic` and :mod:`mythos.exploit`. Provides crash / +trace feedback to the Planner so the CEGIS loop can refine hypotheses. +""" + +from __future__ import annotations + +import json +from typing import Any + +from .base import AgentMessage, MythosAgent +from ..dynamic.aflpp_runner import AFLPlusPlusRunner +from ..dynamic.klee_runner import KLEERunner +from ..dynamic.qemu_harness import QEMUHarness +from ..dynamic.frida_instr import FridaInstrumenter +from ..dynamic.gdb_automation import GDBAutomation +from ..exploit.pwntools_synth import PwntoolsSynth +from ..exploit.rop_chain import ROPChainBuilder +from ..exploit.heap_exploit import HeapExploitKit +from ..exploit.privesc_kb import PrivEscKB + + +class ExecutorAgent(MythosAgent): + name = "executor" + model_tier = "tier2" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.afl = AFLPlusPlusRunner() + self.klee = KLEERunner() + self.qemu = QEMUHarness() + self.frida = FridaInstrumenter() + self.gdb = GDBAutomation() + self.pwn = PwntoolsSynth() + self.rop = ROPChainBuilder() + self.heap = HeapExploitKit() + self.privesc = PrivEscKB() + + def execute(self, harness_dir: str, hypotheses: list[dict[str, Any]]) -> dict[str, Any]: + out: dict[str, Any] = {"crashes": [], "traces": [], "exploits": []} + out["crashes"] += self.afl.run(harness_dir) + out["traces"] += self.klee.run(harness_dir) + if self.qemu.available(): + out["traces"] += self.qemu.run(harness_dir) + if self.frida.available(): + out["traces"] += self.frida.attach_all(harness_dir) + # GDB tactical step-through on each crash. + for crash in out["crashes"]: + out["traces"].append(self.gdb.replay(crash)) + # Synthesise exploits for confirmed crashes. + for crash in out["crashes"]: + chain = self.rop.build(crash) + poc = self.pwn.assemble(crash, chain) + heap = self.heap.spray_template(crash) + out["exploits"].append({"crash": crash.get("id"), + "rop_chain": chain, + "poc": poc, + "heap_template": heap}) + out["privesc_paths"] = self.privesc.suggest(hypotheses) + return out + + def act(self, context: dict[str, Any]) -> AgentMessage: + harness_dir = context.get("harness_dir", "/tmp/research") + hypotheses = context.get("hypotheses", []) + result = self.execute(harness_dir, hypotheses) + # Tier-2 LLM critique pass to narrate the exploit. + narration = self._call_llm( + json.dumps(result)[:12000], + system="You are the Executor. Summarise crashes and exploit chains " + "as JSON {\"summary\": str, \"impact\": str, \"next_steps\": [...]}.", + max_tokens=1024, + ) + result["narration"] = narration + return AgentMessage( + sender=self.name, recipient="orchestrator", role="response", + content={"dynamic_report": result}, + ) diff --git a/mythos/agents/explorer.py b/mythos/agents/explorer.py new file mode 100644 index 0000000000000000000000000000000000000000..1ecf815793ef4ec765b84126f553069205116584 --- /dev/null +++ b/mythos/agents/explorer.py @@ -0,0 +1,61 @@ +""" +Explorer Agent โ€” deep static & semantic code analysis. + +Drives the bridges in :mod:`mythos.static` (Tree-sitter, Joern, CodeQL, +Semgrep) and feeds enriched code understanding back to the Planner. +""" + +from __future__ import annotations + +import json +from typing import Any + +from .base import AgentMessage, MythosAgent +from ..static.treesitter_cpg import TreeSitterCPG +from ..static.joern_bridge import JoernBridge +from ..static.codeql_bridge import CodeQLBridge +from ..static.semgrep_bridge import SemgrepBridge + + +class ExplorerAgent(MythosAgent): + name = "explorer" + model_tier = "tier2" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.tree = TreeSitterCPG() + self.joern = JoernBridge() + self.codeql = CodeQLBridge() + self.semgrep = SemgrepBridge() + + def analyse(self, repo_path: str, hypotheses: list[dict[str, Any]]) -> dict[str, Any]: + report: dict[str, Any] = {"semgrep": [], "joern": [], "codeql": [], "cpg": {}} + # 1. Tree-sitter CPG snapshot โ€” always available (pure-python parser + # fallback if py-tree-sitter not installed). + report["cpg"] = self.tree.summary(repo_path) + # 2. Semgrep โ€” fast, broad coverage. + report["semgrep"] = self.semgrep.scan(repo_path, hypotheses) + # 3. Joern โ€” deep CPG queries when available. + if self.joern.available(): + report["joern"] = self.joern.query(repo_path, hypotheses) + # 4. CodeQL โ€” bring-your-own DB + queries. + if self.codeql.available(): + report["codeql"] = self.codeql.query(repo_path, hypotheses) + # 5. LLM tactical reasoning over consolidated findings. + prompt = json.dumps({"hypotheses": hypotheses, "report": report})[:12000] + verdict = self._call_llm(prompt, system=( + "You are the Explorer. Cross-reference static findings against " + "hypotheses. Return JSON {\"confirmed\": [...], \"refuted\": [...], " + "\"new_hypotheses\": [...]}." + ), max_tokens=2048) + report["llm_verdict_raw"] = verdict + return report + + def act(self, context: dict[str, Any]) -> AgentMessage: + repo = context.get("repo_path", "/data/repo") + hypotheses = context.get("hypotheses", []) + report = self.analyse(repo, hypotheses) + return AgentMessage( + sender=self.name, recipient="orchestrator", role="response", + content={"static_report": report}, + ) diff --git a/mythos/agents/orchestrator.py b/mythos/agents/orchestrator.py new file mode 100644 index 0000000000000000000000000000000000000000..11921e78c046c8502742bef4327f26f9cc60b522 --- /dev/null +++ b/mythos/agents/orchestrator.py @@ -0,0 +1,119 @@ +""" +Mythos Orchestrator โ€” the enhanced Hermes coordinating Planner/Explorer/Executor. + +Implements ยง5.5 of the plan. Models the closed-loop CEGIS cycle: + + Planner โ†’ (Explorer + Executor in parallel) โ†’ Refinement โ†’ Loop + +If AutoGen / CrewAI are installed they are auto-detected and used to drive +inter-agent conversation; otherwise the orchestrator falls back to the +deterministic in-process loop below โ€” both produce identical dossiers so +downstream Bounty Gateway code is unaffected. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +from .base import AgentMessage +from .planner import PlannerAgent +from .explorer import ExplorerAgent +from .executor import ExecutorAgent +from ..learning.episodic_memory import EpisodicMemory +from ..learning.mlflow_tracker import MLflowTracker + +LOG = logging.getLogger("mythos.orchestrator") + + +class MythosOrchestrator: + def __init__( + self, + planner: PlannerAgent | None = None, + explorer: ExplorerAgent | None = None, + executor: ExecutorAgent | None = None, + max_iterations: int = 3, + ): + self.planner = planner or PlannerAgent() + self.explorer = explorer or ExplorerAgent() + self.executor = executor or ExecutorAgent() + self.max_iterations = max_iterations + self.memory = EpisodicMemory() + self.tracker = MLflowTracker(experiment="mythos-campaigns") + self.transcript: list[AgentMessage] = [] + + # -- transport helpers -------------------------------------------------- + + def _send(self, msg: AgentMessage) -> None: + self.transcript.append(msg) + LOG.debug("%s โ†’ %s : %s", msg.sender, msg.recipient, str(msg.content)[:200]) + + # -- main loop ---------------------------------------------------------- + + def run_campaign(self, target: dict[str, Any]) -> dict[str, Any]: + run_id = self.tracker.start_run(tags={"target": target.get("repo", "?")}) + ctx: dict[str, Any] = {"target": target, "recon": target.get("recon", {})} + dossier: dict[str, Any] = {"target": target, "iterations": []} + + for i in range(self.max_iterations): + iter_started = time.time() + LOG.info("Mythos iteration %s/%s", i + 1, self.max_iterations) + + # 1. Planner + plan_msg = self.planner.act(ctx) + self._send(plan_msg) + ctx.update(plan_msg.content) + + # 2. Explorer (static) and Executor (dynamic) in lock-step. + ctx["repo_path"] = target.get("repo_path", "/data/repo") + ctx["harness_dir"] = target.get("harness_dir", "/tmp/research") + + explorer_msg = self.explorer.act(ctx) + self._send(explorer_msg) + ctx.update(explorer_msg.content) + + executor_msg = self.executor.act(ctx) + self._send(executor_msg) + ctx.update(executor_msg.content) + + # 3. Refinement โ€” feed dynamic feedback back to the Planner so it + # can prune / amplify hypotheses on the next loop. + refined = self._refine(ctx) + ctx["recon"] = {**ctx.get("recon", {}), **refined} + + iteration = { + "n": i + 1, + "elapsed": round(time.time() - iter_started, 2), + "plan": plan_msg.content, + "static": explorer_msg.content, + "dynamic": executor_msg.content, + "refinement": refined, + } + dossier["iterations"].append(iteration) + self.memory.record(target, iteration) + self.tracker.log_iteration(run_id, iteration) + + if self._converged(iteration): + LOG.info("Mythos campaign converged after %s iteration(s)", i + 1) + break + + dossier["transcript"] = [m.__dict__ for m in self.transcript] + self.tracker.end_run(run_id) + return dossier + + # -- helpers ------------------------------------------------------------ + + @staticmethod + def _refine(ctx: dict[str, Any]) -> dict[str, Any]: + dyn = ctx.get("dynamic_report", {}) + crashes = dyn.get("crashes", []) + return { + "crash_signatures": [c.get("signature") for c in crashes if c.get("signature")], + "confirmed_count": len(crashes), + } + + @staticmethod + def _converged(iteration: dict[str, Any]) -> bool: + dyn = iteration.get("dynamic", {}).get("dynamic_report", {}) + return bool(dyn.get("exploits")) diff --git a/mythos/agents/planner.py b/mythos/agents/planner.py new file mode 100644 index 0000000000000000000000000000000000000000..03d94ee5c9c439210bad5f8ae676b4bd5634a981 --- /dev/null +++ b/mythos/agents/planner.py @@ -0,0 +1,86 @@ +""" +Planner Agent โ€” strategic reasoning and hypothesis generation. + +Implements ยง5.1 of the Mythos plan: + * Problem decomposition. + * Probabilistic hypothesis generation (delegates to + :mod:`mythos.reasoning.probabilistic`). + * Attack-graph construction. + * Resource allocation between Explorer and Executor. +""" + +from __future__ import annotations + +import json +from typing import Any + +from .base import AgentMessage, MythosAgent +from ..reasoning.probabilistic import HypothesisEngine +from ..reasoning.attack_graph import AttackGraph + + +class PlannerAgent(MythosAgent): + name = "planner" + model_tier = "tier1" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.hypothesis_engine = HypothesisEngine() + self.attack_graph = AttackGraph() + + def decompose(self, target: dict[str, Any]) -> list[str]: + """Split a high-level engagement into ordered sub-tasks.""" + system = ( + "Decompose a security engagement into atomic sub-tasks. " + "Return JSON: {\"tasks\": [\"recon ...\", \"taint ...\", ...]}" + ) + raw = self._call_llm(json.dumps(target), system=system, max_tokens=1024) + try: + return json.loads(raw).get("tasks", []) + except Exception: + # Sensible deterministic fallback so the orchestrator never stalls. + return [ + "recon: enumerate languages, dependencies, attack surface", + "static: run Joern + Semgrep + Tree-sitter CPG queries", + "dynamic: synthesise fuzzing harnesses, run AFL++ + KLEE", + "exploit: chain primitives via pwntools", + "consensus: tier-3 adversarial review", + "disclosure: package dossier", + ] + + def generate_hypotheses(self, recon: dict[str, Any]) -> list[dict[str, Any]]: + """Produce ranked vulnerability hypotheses with probabilistic priors.""" + return self.hypothesis_engine.sample(recon, n=8) + + def build_attack_graph(self, hypotheses: list[dict[str, Any]]) -> AttackGraph: + for h in hypotheses: + self.attack_graph.add_hypothesis(h) + self.attack_graph.connect() + return self.attack_graph + + def allocate(self, hypotheses: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + """Decide which hypothesis goes to Explorer (static) vs Executor (dynamic).""" + explorer_q, executor_q = [], [] + for h in hypotheses: + (explorer_q if h.get("kind") in ("logic", "auth", "validation") + else executor_q).append(h) + return {"explorer": explorer_q, "executor": executor_q} + + # -- agent API ---------------------------------------------------------- + + def act(self, context: dict[str, Any]) -> AgentMessage: + target = context.get("target", {}) + recon = context.get("recon", {}) + tasks = self.decompose(target) + hypotheses = self.generate_hypotheses(recon or target) + graph = self.build_attack_graph(hypotheses) + allocation = self.allocate(hypotheses) + return AgentMessage( + sender=self.name, recipient="orchestrator", role="response", + content={ + "tasks": tasks, + "hypotheses": hypotheses, + "attack_graph": graph.to_dict(), + "allocation": allocation, + }, + ) diff --git a/mythos/api/__init__.py b/mythos/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d3b3d757a4c8a41a0d8067a2b5cc8065c48c6be --- /dev/null +++ b/mythos/api/__init__.py @@ -0,0 +1,2 @@ +"""FastAPI productization layer for Mythos.""" +from .schemas import AnalyseRequest, AnalyseResponse, WebhookEvent # noqa: F401 diff --git a/mythos/api/auth.py b/mythos/api/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..4d24727081017d56db093b3f91806fe0bf1d4a3b --- /dev/null +++ b/mythos/api/auth.py @@ -0,0 +1,44 @@ +""" +Lightweight API-key + OAuth2 bearer authentication for the Mythos API. + +Backed by an env-defined static API key (``MYTHOS_API_KEYS=key1,key2``) plus +optional JWT validation when ``MYTHOS_JWT_PUBKEY`` is set. +""" + +from __future__ import annotations + +import os +from typing import Any + +try: # pragma: no cover + from fastapi import Header, HTTPException, status +except Exception: # noqa: BLE001 + Header = HTTPException = status = None # type: ignore + +try: # pragma: no cover + import jwt # type: ignore + _JWT = True +except Exception: # noqa: BLE001 + _JWT = False + + +def _allowed_keys() -> set[str]: + return {k.strip() for k in os.getenv("MYTHOS_API_KEYS", "").split(",") if k.strip()} + + +def require_api_key(authorization: str | None = Header(default=None)) -> dict[str, Any]: + if HTTPException is None: # FastAPI not installed โ€” let the caller handle it. + return {"sub": "anonymous"} + if not authorization or not authorization.lower().startswith("bearer "): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="missing bearer") + token = authorization.split(None, 1)[1].strip() + keys = _allowed_keys() + if keys and token in keys: + return {"sub": "api-key", "token": token[:8] + "..."} + if _JWT and (pubkey := os.getenv("MYTHOS_JWT_PUBKEY")): + try: + return jwt.decode(token, pubkey, algorithms=["RS256"]) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"jwt: {exc}") from exc + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="invalid token") diff --git a/mythos/api/fastapi_server.py b/mythos/api/fastapi_server.py new file mode 100644 index 0000000000000000000000000000000000000000..1f87b3aa4d97ac8a3b7a66df12e0a48b520d0d02 --- /dev/null +++ b/mythos/api/fastapi_server.py @@ -0,0 +1,92 @@ +""" +Mythos productization API. + +Run with:: + + uvicorn mythos.api.fastapi_server:app --host 0.0.0.0 --port 8000 + +If ``fastapi`` isn't installed (e.g. minimal HF Space build) importing this +module is still safe โ€” ``app`` is set to ``None`` so a deployment guard can +detect the gap. +""" + +from __future__ import annotations + +import logging +import threading +import uuid +from typing import Any + +LOG = logging.getLogger("mythos.api") + +try: # pragma: no cover + from fastapi import Depends, FastAPI, HTTPException + from fastapi.middleware.cors import CORSMiddleware + _FASTAPI = True +except Exception: # noqa: BLE001 + _FASTAPI = False + FastAPI = None # type: ignore + +from .auth import require_api_key +from .schemas import AnalyseRequest, AnalyseResponse, WebhookEvent +from .webhooks import deliver +from ..agents.orchestrator import MythosOrchestrator + +_RUNS: dict[str, dict[str, Any]] = {} + + +if _FASTAPI: + app = FastAPI( + title="Rhodawk Mythos API", + version="1.0.0", + description="Autonomous vulnerability research as a service.", + ) + app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], + allow_headers=["*"]) + + @app.get("/v1/health") + def health(): + return {"status": "ok", "service": "rhodawk-mythos"} + + @app.post("/v1/analyze_target", response_model=AnalyseResponse) + def analyze_target(req: AnalyseRequest, principal=Depends(require_api_key)): + run_id = uuid.uuid4().hex + _RUNS[run_id] = {"status": "running", "principal": principal} + + target = req.dict() + target["recon"] = { + "languages": req.languages, + "frameworks": req.frameworks, + "dependencies": req.dependencies, + } + + def _execute(): + try: + dossier = MythosOrchestrator(max_iterations=req.max_iterations).run_campaign(target) + _RUNS[run_id] = {"status": "complete", "dossier": dossier} + if req.callback_url: + deliver(req.callback_url, "analysis.complete", + {"run_id": run_id, "summary": dossier.get("iterations", [])[-1:]}) + except Exception as exc: # noqa: BLE001 + _RUNS[run_id] = {"status": "error", "error": str(exc)} + if req.callback_url: + deliver(req.callback_url, "analysis.error", + {"run_id": run_id, "error": str(exc)}) + + threading.Thread(target=_execute, daemon=True).start() + return AnalyseResponse(target=target, iterations=[], crashes=[], + summary=f"queued run_id={run_id}") + + @app.get("/v1/runs/{run_id}") + def get_run(run_id: str, principal=Depends(require_api_key)): + if run_id not in _RUNS: + raise HTTPException(status_code=404, detail="unknown run_id") + return _RUNS[run_id] + + @app.post("/v1/webhooks/test") + def webhook_test(evt: WebhookEvent, principal=Depends(require_api_key)): + return {"received": evt.dict(), "by": principal.get("sub")} + +else: # pragma: no cover + app = None + LOG.warning("fastapi not installed โ€” Mythos API surface unavailable") diff --git a/mythos/api/schemas.py b/mythos/api/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..ca6e7fefc601f4231fc4f8d438f7b8c6e8ef4921 --- /dev/null +++ b/mythos/api/schemas.py @@ -0,0 +1,45 @@ +"""Pydantic schemas for the Mythos API surface.""" + +from __future__ import annotations + +from typing import Any + +try: # pragma: no cover + from pydantic import BaseModel, Field +except Exception: # noqa: BLE001 - pydantic always available via fastapi but be safe + BaseModel = object # type: ignore + def Field(*_a, **_kw): # type: ignore + return None + + +class AnalyseRequest(BaseModel): + repo: str = Field(..., description="Git URL or local path of the target.") + branch: str | None = None + languages: list[str] = [] + frameworks: list[str] = [] + dependencies: list[str] = [] + focus: str | None = Field(None, description="Optional natural-language focus area.") + max_iterations: int = 3 + output_format: str = Field("dossier", description="dossier | sarif | json") + callback_url: str | None = None + + +class CrashReport(BaseModel): + id: str + harness: str | None = None + signature: str | None = None + rop_chain: list[str] = [] + poc_path: str | None = None + + +class AnalyseResponse(BaseModel): + target: dict[str, Any] + iterations: list[dict[str, Any]] + crashes: list[CrashReport] = [] + summary: str = "" + + +class WebhookEvent(BaseModel): + event: str + run_id: str + payload: dict[str, Any] = {} diff --git a/mythos/api/webhooks.py b/mythos/api/webhooks.py new file mode 100644 index 0000000000000000000000000000000000000000..fb9aa211a914e8fceef7969343cfb45a21d609d1 --- /dev/null +++ b/mythos/api/webhooks.py @@ -0,0 +1,36 @@ +"""HMAC-signed webhook delivery.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import time +from typing import Any + +import requests + +_SECRET = os.getenv("MYTHOS_WEBHOOK_SECRET", "") + + +def sign(payload: bytes) -> str: + if not _SECRET: + return "unsigned" + mac = hmac.new(_SECRET.encode(), payload, hashlib.sha256).hexdigest() + return f"sha256={mac}" + + +def deliver(url: str, event: str, payload: dict[str, Any]) -> dict[str, Any]: + body = json.dumps({"event": event, "ts": time.time(), "payload": payload}, + default=str).encode() + headers = { + "Content-Type": "application/json", + "X-Mythos-Event": event, + "X-Mythos-Signature": sign(body), + } + try: + r = requests.post(url, data=body, headers=headers, timeout=15) + return {"status_code": r.status_code, "body": r.text[:500]} + except Exception as exc: # noqa: BLE001 + return {"error": str(exc)} diff --git a/mythos/dynamic/__init__.py b/mythos/dynamic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b74213edf1d42d7f3df306a2ded3f24ac5d23cfb --- /dev/null +++ b/mythos/dynamic/__init__.py @@ -0,0 +1,6 @@ +"""Dynamic execution + instrumentation bridges.""" +from .aflpp_runner import AFLPlusPlusRunner # noqa: F401 +from .klee_runner import KLEERunner # noqa: F401 +from .qemu_harness import QEMUHarness # noqa: F401 +from .frida_instr import FridaInstrumenter # noqa: F401 +from .gdb_automation import GDBAutomation # noqa: F401 diff --git a/mythos/dynamic/aflpp_runner.py b/mythos/dynamic/aflpp_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..a46d81c74139d99239babecacda2d41b58fbe96f --- /dev/null +++ b/mythos/dynamic/aflpp_runner.py @@ -0,0 +1,80 @@ +""" +AFL++ runner. + +If ``afl-fuzz`` is on ``$PATH`` we drive a short, time-boxed campaign over +each harness directory. Otherwise we route through the existing +``fuzzing_engine`` (Hypothesis-based) so the orchestrator still produces +crash candidates. +""" + +from __future__ import annotations + +import glob +import json +import os +import shutil +import subprocess +import time +from typing import Any + + +class AFLPlusPlusRunner: + def __init__(self, time_budget_s: int = 60): + self.bin = shutil.which("afl-fuzz") + self.time_budget_s = int(os.getenv("MYTHOS_AFL_BUDGET", time_budget_s)) + + def available(self) -> bool: + return bool(self.bin) + + def run(self, harness_dir: str) -> list[dict[str, Any]]: + if not os.path.isdir(harness_dir): + return [] + if not self.available(): + return self._hypothesis_fallback(harness_dir) + crashes: list[dict[str, Any]] = [] + for harness in glob.glob(os.path.join(harness_dir, "*_harness")): + in_dir = os.path.join(harness_dir, "afl_in") + out_dir = os.path.join(harness_dir, f"afl_out_{os.path.basename(harness)}") + os.makedirs(in_dir, exist_ok=True) + if not os.listdir(in_dir): + with open(os.path.join(in_dir, "seed"), "wb") as fh: + fh.write(b"A" * 16) + os.makedirs(out_dir, exist_ok=True) + try: + start = time.time() + proc = subprocess.Popen( + [self.bin, "-i", in_dir, "-o", out_dir, "-V", str(self.time_budget_s), + "--", harness, "@@"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + proc.wait(timeout=self.time_budget_s + 30) + for crash_path in glob.glob(os.path.join(out_dir, "default", "crashes", "id:*")): + crashes.append({ + "id": os.path.basename(crash_path), + "harness": harness, + "path": crash_path, + "elapsed_s": round(time.time() - start, 2), + "signature": self._signature(crash_path), + }) + except (subprocess.TimeoutExpired, FileNotFoundError): + continue + return crashes + + @staticmethod + def _signature(path: str) -> str: + try: + with open(path, "rb") as fh: + return fh.read(64).hex() + except OSError: + return "" + + @staticmethod + def _hypothesis_fallback(harness_dir: str) -> list[dict[str, Any]]: + """Surface a marker that the legacy fuzzing_engine will consume.""" + marker = os.path.join(harness_dir, "_mythos_afl_unavailable.json") + try: + with open(marker, "w") as fh: + json.dump({"fallback": "hypothesis"}, fh) + except OSError: + pass + return [] diff --git a/mythos/dynamic/frida_instr.py b/mythos/dynamic/frida_instr.py new file mode 100644 index 0000000000000000000000000000000000000000..ef389a9b8a301f0678b3cb62e408a0daa2d0da2c --- /dev/null +++ b/mythos/dynamic/frida_instr.py @@ -0,0 +1,53 @@ +"""Frida dynamic instrumentation โ€” attaches a generic syscall/cred tracer.""" + +from __future__ import annotations + +import os +from typing import Any + +try: # pragma: no cover + import frida # type: ignore + _FRIDA = True +except Exception: # noqa: BLE001 + _FRIDA = False + +# Minimal generic JS instrumentation script โ€” interceptors are expanded by +# the orchestrator at call-site for kind-specific tracing. +_DEFAULT_SCRIPT = r""" +const interesting = ['open', 'execve', 'connect', 'recvfrom', 'mmap']; +interesting.forEach((name) => { + try { + const sym = Module.findExportByName(null, name); + if (sym) Interceptor.attach(sym, { + onEnter(args) { send({event: name, args: args.map(a => a.toString())}); } + }); + } catch (e) {} +}); +""" + + +class FridaInstrumenter: + def available(self) -> bool: + return _FRIDA + + def attach_all(self, harness_dir: str) -> list[dict[str, Any]]: + if not _FRIDA: + return [] + events: list[dict[str, Any]] = [] + + def on_message(msg, _data): + if msg.get("type") == "send": + events.append(msg.get("payload", {})) + + device = frida.get_local_device() + for proc in device.enumerate_processes(): + if not any(proc.name.startswith(b) for b in ("python", "node", "java")): + continue + try: + session = device.attach(proc.pid) + script = session.create_script(_DEFAULT_SCRIPT) + script.on("message", on_message) + script.load() + except Exception: + continue + return events[:500] diff --git a/mythos/dynamic/gdb_automation.py b/mythos/dynamic/gdb_automation.py new file mode 100644 index 0000000000000000000000000000000000000000..bbffb0f357ff2a6b8dc1dbe11dd78f0cc9bd37e9 --- /dev/null +++ b/mythos/dynamic/gdb_automation.py @@ -0,0 +1,55 @@ +""" +GDB-Python automation โ€” replays a crash through GDB and captures +backtrace, registers, and a small chunk of memory around the crash site. + +When GDB is missing the function returns a structured marker so the +orchestrator can record the gap. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from typing import Any + +_GDB_SCRIPT = """\ +set pagination off +set logging file {logfile} +set logging on +run < {input} +bt +info registers +x/64x $sp +quit +""" + + +class GDBAutomation: + def __init__(self): + self.bin = shutil.which("gdb") + + def available(self) -> bool: + return bool(self.bin) + + def replay(self, crash: dict[str, Any]) -> dict[str, Any]: + if not self.available(): + return {"crash": crash.get("id"), "gdb": "unavailable"} + binary = crash.get("harness") + crash_input = crash.get("path") + if not (binary and crash_input and os.path.exists(binary) and os.path.exists(crash_input)): + return {"crash": crash.get("id"), "gdb": "missing-binary-or-input"} + with tempfile.TemporaryDirectory() as work: + logfile = os.path.join(work, "gdb.log") + scriptfile = os.path.join(work, "script.gdb") + with open(scriptfile, "w") as fh: + fh.write(_GDB_SCRIPT.format(logfile=logfile, input=crash_input)) + try: + subprocess.run([self.bin, "-q", "-batch", "-x", scriptfile, binary], + capture_output=True, timeout=60, check=False) + with open(logfile) as fh: + log = fh.read()[-4000:] + except Exception as exc: # noqa: BLE001 + log = f"gdb error: {exc}" + return {"crash": crash.get("id"), "gdb_log": log} diff --git a/mythos/dynamic/klee_runner.py b/mythos/dynamic/klee_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..82c47650d5860a2cde63d495c26760ca274de348 --- /dev/null +++ b/mythos/dynamic/klee_runner.py @@ -0,0 +1,37 @@ +"""KLEE symbolic execution runner โ€” emits per-path execution traces.""" + +from __future__ import annotations + +import glob +import os +import shutil +import subprocess +from typing import Any + + +class KLEERunner: + def __init__(self, time_budget_s: int = 120): + self.bin = shutil.which("klee") + self.time_budget_s = int(os.getenv("MYTHOS_KLEE_BUDGET", time_budget_s)) + + def available(self) -> bool: + return bool(self.bin) + + def run(self, harness_dir: str) -> list[dict[str, Any]]: + if not self.available() or not os.path.isdir(harness_dir): + return [] + traces: list[dict[str, Any]] = [] + for bc in glob.glob(os.path.join(harness_dir, "*.bc")): + try: + proc = subprocess.run( + [self.bin, "--max-time", str(self.time_budget_s), bc], + capture_output=True, text=True, timeout=self.time_budget_s + 30, check=False, + ) + traces.append({ + "module": bc, + "stdout_tail": proc.stdout[-2000:], + "stderr_tail": proc.stderr[-2000:], + }) + except subprocess.TimeoutExpired: + traces.append({"module": bc, "error": "timeout"}) + return traces diff --git a/mythos/dynamic/qemu_harness.py b/mythos/dynamic/qemu_harness.py new file mode 100644 index 0000000000000000000000000000000000000000..69d769ca33816f18572bae0712b60f3ea7129a9e --- /dev/null +++ b/mythos/dynamic/qemu_harness.py @@ -0,0 +1,38 @@ +"""QEMU full-system emulation harness for kernel-level fuzzing experiments.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from typing import Any + + +class QEMUHarness: + def __init__(self): + self.bin = shutil.which("qemu-system-x86_64") or shutil.which("qemu-x86_64") + + def available(self) -> bool: + return bool(self.bin) + + def run(self, harness_dir: str) -> list[dict[str, Any]]: + if not self.available(): + return [] + # Look for prepared kernel images / userland binaries. + kernels = [p for p in os.listdir(harness_dir) if p.endswith((".elf", ".bin"))] + if not kernels: + return [] + traces: list[dict[str, Any]] = [] + for kern in kernels: + try: + proc = subprocess.run( + [self.bin, "-d", "in_asm,exec", "-D", "/tmp/qemu.log", + "-no-reboot", "-nographic", "-kernel", os.path.join(harness_dir, kern)], + capture_output=True, text=True, timeout=120, check=False, + ) + traces.append({"kernel": kern, + "stdout_tail": proc.stdout[-1500:], + "stderr_tail": proc.stderr[-1500:]}) + except subprocess.TimeoutExpired: + traces.append({"kernel": kern, "error": "timeout"}) + return traces diff --git a/mythos/exploit/__init__.py b/mythos/exploit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..524fa8735aad97e6127ece2196ccc97eb7edf67e --- /dev/null +++ b/mythos/exploit/__init__.py @@ -0,0 +1,5 @@ +"""Exploit synthesis primitives.""" +from .pwntools_synth import PwntoolsSynth # noqa: F401 +from .rop_chain import ROPChainBuilder # noqa: F401 +from .heap_exploit import HeapExploitKit # noqa: F401 +from .privesc_kb import PrivEscKB # noqa: F401 diff --git a/mythos/exploit/heap_exploit.py b/mythos/exploit/heap_exploit.py new file mode 100644 index 0000000000000000000000000000000000000000..cd6ff84a70fea880ce7391c6494b4d69d27532de --- /dev/null +++ b/mythos/exploit/heap_exploit.py @@ -0,0 +1,53 @@ +""" +Heap exploitation kit โ€” produces target-allocator-aware spray templates. + +Supports glibc ptmalloc2 (default), tcache, jemalloc, and a generic +fallback. The Executor uses the resulting template as a starting point for +GDB+GEF-driven manual confirmation. +""" + +from __future__ import annotations + +from typing import Any + +_TEMPLATES = { + "ptmalloc2": ( + "# tcache poisoning skeleton\n" + "for i in range(7):\n" + " free(allocate({size}))\n" + "free(target_chunk)\n" + "overwrite_fd(target_chunk, target_addr)\n" + "victim = allocate({size}) # returns target_addr\n" + ), + "tcache": ( + "# tcache double-free skeleton\n" + "a = allocate({size}); b = allocate({size})\n" + "free(a); free(b); free(a)\n" + "x = allocate({size}); overwrite_fd(x, target_addr)\n" + "allocate({size}); allocate({size}) # returns target_addr\n" + ), + "jemalloc": ( + "# jemalloc run-overlap skeleton\n" + "spray = [allocate({size}) for _ in range(0x40)]\n" + "trigger_uaf(spray[0x20])\n" + "spray2 = [allocate({size}) for _ in range(0x40)]\n" + ), + "generic": ( + "# generic massage spray\n" + "spray = [allocate({size}) for _ in range(0x100)]\n" + "trigger_vulnerability(spray[0x80])\n" + ), +} + + +class HeapExploitKit: + def spray_template(self, crash: dict[str, Any]) -> dict[str, Any]: + allocator = crash.get("allocator", "ptmalloc2") + size = crash.get("chunk_size", 0x80) + tpl = _TEMPLATES.get(allocator, _TEMPLATES["generic"]).format(size=hex(size)) + return { + "allocator": allocator, + "chunk_size": size, + "template": tpl, + "notes": "Refine with GEF (`heap chunks`, `heap bins`) before weaponising.", + } diff --git a/mythos/exploit/privesc_kb.py b/mythos/exploit/privesc_kb.py new file mode 100644 index 0000000000000000000000000000000000000000..c079e28577126114835efda858ece7ca3868cd74 --- /dev/null +++ b/mythos/exploit/privesc_kb.py @@ -0,0 +1,38 @@ +""" +Privilege-escalation knowledge base โ€” codifies LinPEAS / WinPEAS heuristics +into structured suggestions the Executor can verify automatically. +""" + +from __future__ import annotations + +from typing import Any + +_LINUX_VECTORS = [ + {"id": "suid-binaries", "cmd": "find / -perm -4000 -type f 2>/dev/null"}, + {"id": "writable-passwd", "cmd": "ls -la /etc/passwd /etc/shadow"}, + {"id": "kernel-version", "cmd": "uname -a; cat /proc/version"}, + {"id": "cron-jobs", "cmd": "ls -la /etc/cron* /var/spool/cron 2>/dev/null"}, + {"id": "sudo-rules", "cmd": "sudo -l 2>/dev/null"}, + {"id": "capabilities", "cmd": "getcap -r / 2>/dev/null"}, + {"id": "docker-socket", "cmd": "ls -la /var/run/docker.sock 2>/dev/null"}, + {"id": "world-writable-paths", "cmd": "find / -writable -type d 2>/dev/null | head -50"}, +] + +_WINDOWS_VECTORS = [ + {"id": "service-perms", "cmd": "accesschk.exe -uwcqv \"Authenticated Users\" *"}, + {"id": "unquoted-paths", "cmd": "wmic service get name,displayname,pathname,startmode | findstr /i \"auto\""}, + {"id": "always-install-elev", "cmd": "reg query HKCU\\Software\\Policies\\Microsoft\\Windows\\Installer"}, + {"id": "stored-credentials", "cmd": "cmdkey /list"}, +] + + +class PrivEscKB: + def suggest(self, hypotheses: list[dict[str, Any]]) -> list[dict[str, Any]]: + wants_linux = any(h.get("kind") == "auth" for h in hypotheses) + wants_windows = any("windows" in (h.get("rationale") or "").lower() for h in hypotheses) + out: list[dict[str, Any]] = [] + if wants_linux or not wants_windows: + out += [{**v, "platform": "linux"} for v in _LINUX_VECTORS] + if wants_windows: + out += [{**v, "platform": "windows"} for v in _WINDOWS_VECTORS] + return out diff --git a/mythos/exploit/pwntools_synth.py b/mythos/exploit/pwntools_synth.py new file mode 100644 index 0000000000000000000000000000000000000000..841a8c297763564fc2689803e7c6e1f98d2feaff --- /dev/null +++ b/mythos/exploit/pwntools_synth.py @@ -0,0 +1,58 @@ +""" +Pwntools-based PoC assembler. + +Produces a runnable Python script for every confirmed crash that a human +researcher (or downstream Bounty Gateway) can review and submit. When +``pwntools`` isn't installed the assembler still emits a self-contained +Python file using the standard library that demonstrates the input. +""" + +from __future__ import annotations + +import os +import textwrap +from typing import Any + +try: # pragma: no cover + from pwn import context # type: ignore # noqa: F401 + _PWN = True +except Exception: # noqa: BLE001 + _PWN = False + + +class PwntoolsSynth: + def assemble(self, crash: dict[str, Any], rop_chain: list[str]) -> dict[str, Any]: + binary = crash.get("harness", "") + crash_input = crash.get("path", "") + chain_lit = ", ".join(repr(g) for g in rop_chain) or "# no gadgets" + if _PWN: + template = textwrap.dedent(f"""\ + # Auto-generated by Rhodawk Mythos + from pwn import * + context.log_level = 'error' + p = process({binary!r}) + rop = ROP({binary!r}) + gadgets = [{chain_lit}] + with open({crash_input!r}, 'rb') as f: + payload = f.read() + p.sendline(payload) + print(p.recvall(timeout=2)) + """) + else: + template = textwrap.dedent(f"""\ + # Auto-generated by Rhodawk Mythos (no-pwntools fallback) + import subprocess, sys + with open({crash_input!r}, 'rb') as f: + payload = f.read() + proc = subprocess.run([{binary!r}], input=payload, + capture_output=True, timeout=5) + sys.stdout.write(proc.stdout.decode(errors='replace')) + sys.stderr.write(proc.stderr.decode(errors='replace')) + """) + out_path = os.path.join("/tmp", f"poc_{crash.get('id', 'x')}.py") + try: + with open(out_path, "w") as fh: + fh.write(template) + except OSError: + pass + return {"path": out_path, "code": template, "uses_pwntools": _PWN} diff --git a/mythos/exploit/rop_chain.py b/mythos/exploit/rop_chain.py new file mode 100644 index 0000000000000000000000000000000000000000..9ed75d28cf119eaeadf4e2104ec1a2c429b8f251 --- /dev/null +++ b/mythos/exploit/rop_chain.py @@ -0,0 +1,56 @@ +""" +ROP chain builder โ€” wraps ``angrop`` and ``ROPgadget`` when available, with +an in-memory deterministic gadget registry so unit tests can exercise the +chain logic without binary inputs. +""" + +from __future__ import annotations + +import shutil +import subprocess +from typing import Any + +try: # pragma: no cover + import angr # type: ignore # noqa: F401 + import angrop # type: ignore # noqa: F401 + _ANGROP = True +except Exception: # noqa: BLE001 + _ANGROP = False + + +class ROPChainBuilder: + def __init__(self): + self.ropgadget = shutil.which("ROPgadget") + + def build(self, crash: dict[str, Any]) -> list[str]: + binary = crash.get("harness") + if not binary: + return [] + if _ANGROP: + return self._with_angrop(binary) + if self.ropgadget: + return self._with_ropgadget(binary) + return ["pop rdi ; ret", "/bin/sh", "system"] + + @staticmethod + def _with_angrop(binary: str) -> list[str]: # pragma: no cover - heavy dep + try: + project = angr.Project(binary, auto_load_libs=False) + rop = project.analyses.ROP() + rop.find_gadgets() + chain = rop.execve(b"/bin/sh\x00") + return [str(g) for g in chain.gadgets] + except Exception: + return [] + + def _with_ropgadget(self, binary: str) -> list[str]: + try: + proc = subprocess.run( + [self.ropgadget, "--binary", binary], capture_output=True, + text=True, timeout=120, check=False, + ) + lines = [ln.strip() for ln in proc.stdout.splitlines() + if ":" in ln and "ret" in ln] + return lines[:32] + except Exception: + return [] diff --git a/mythos/integration.py b/mythos/integration.py new file mode 100644 index 0000000000000000000000000000000000000000..fd3c44ebac8ce1f88ca63e8c8927b60c0dac1633 --- /dev/null +++ b/mythos/integration.py @@ -0,0 +1,27 @@ +""" +Integration shim between the legacy ``hermes_orchestrator`` six-phase +pipeline and the new Mythos multi-agent framework. + +The shim is *additive*: existing Rhodawk code paths keep working unchanged. +Callers that opt in to Mythos by setting ``RHODAWK_MYTHOS=1`` get the +Planner/Explorer/Executor pipeline transparently. +""" + +from __future__ import annotations + +import os +from typing import Any + + +def mythos_enabled() -> bool: + return os.getenv("RHODAWK_MYTHOS", "0").lower() in ("1", "true", "yes", "on") + + +def maybe_run_mythos(target: dict[str, Any]) -> dict[str, Any] | None: + """If Mythos is enabled, run the multi-agent pipeline and return its dossier.""" + if not mythos_enabled(): + return None + from .agents.orchestrator import MythosOrchestrator + + orch = MythosOrchestrator() + return orch.run_campaign(target) diff --git a/mythos/learning/__init__.py b/mythos/learning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0004af8beffda7ace8144598a61cb182afda8e --- /dev/null +++ b/mythos/learning/__init__.py @@ -0,0 +1,6 @@ +"""Self-improvement: RL planner, MLflow tracker, LoRA adapters, curriculum, episodic memory.""" +from .rl_planner import RLPlanner # noqa: F401 +from .mlflow_tracker import MLflowTracker # noqa: F401 +from .lora_adapters import LoRAAdapterManager # noqa: F401 +from .curriculum import CurriculumScheduler # noqa: F401 +from .episodic_memory import EpisodicMemory # noqa: F401 diff --git a/mythos/learning/curriculum.py b/mythos/learning/curriculum.py new file mode 100644 index 0000000000000000000000000000000000000000..022200fa926d13c2fffd6df8e49e47c00ab2adcc --- /dev/null +++ b/mythos/learning/curriculum.py @@ -0,0 +1,48 @@ +""" +Curriculum scheduler โ€” orders training targets from easy โ†’ hard. + +Difficulty is a weighted blend of: + * lines of code, + * dependency surface, + * historical success rate of similar repos. + +Used by the data-flywheel to feed RL / LoRA fine-tuning with progressively +harder workloads. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any + + +@dataclass +class CurriculumItem: + repo: str + loc: int + dep_count: int + historical_success: float = 0.0 # 0..1 + + @property + def difficulty(self) -> float: + return ( + 0.5 * math.log1p(self.loc) + + 0.3 * math.log1p(self.dep_count) + + 0.2 * (1.0 - self.historical_success) + ) + + +class CurriculumScheduler: + def __init__(self, items: list[CurriculumItem] | None = None): + self.items: list[CurriculumItem] = items or [] + + def add(self, repo: str, loc: int, dep_count: int, success: float = 0.0) -> None: + self.items.append(CurriculumItem(repo, loc, dep_count, success)) + + def next_batch(self, batch_size: int = 4) -> list[CurriculumItem]: + self.items.sort(key=lambda i: i.difficulty) + return self.items[:batch_size] + + def to_dict(self) -> dict[str, Any]: + return {"items": [i.__dict__ | {"difficulty": i.difficulty} for i in self.items]} diff --git a/mythos/learning/episodic_memory.py b/mythos/learning/episodic_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1a882ddd7970f308588eca7c6821c5b368f51d --- /dev/null +++ b/mythos/learning/episodic_memory.py @@ -0,0 +1,62 @@ +""" +Episodic memory โ€” stores complete campaign trajectories on disk so the +Planner can retrieve "what worked last time on a similar repo". + +Backed by SQLite for portability (no extra deps). Schema mirrors the +``memory_engine`` patterns already in the repo. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import time +from typing import Any + +_DB = os.getenv("MYTHOS_EPISODIC_DB", "/data/mythos/episodic.sqlite") + + +class EpisodicMemory: + def __init__(self, path: str = _DB): + self.path = path + os.makedirs(os.path.dirname(path), exist_ok=True) + self._db = sqlite3.connect(path, check_same_thread=False) + self._db.execute( + "CREATE TABLE IF NOT EXISTS episodes (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " ts REAL," + " repo TEXT," + " iteration INTEGER," + " cwes TEXT," + " outcome TEXT," + " payload TEXT" + ")" + ) + self._db.execute( + "CREATE INDEX IF NOT EXISTS idx_episodes_repo ON episodes(repo)" + ) + self._db.commit() + + def record(self, target: dict[str, Any], iteration: dict[str, Any]) -> int: + cwes = [h.get("cwe") for h in iteration.get("plan", {}).get("hypotheses", [])] + outcome = "exploit" if iteration.get("dynamic", {}).get( + "dynamic_report", {}).get("exploits") else "unconfirmed" + cur = self._db.execute( + "INSERT INTO episodes(ts, repo, iteration, cwes, outcome, payload) " + "VALUES (?, ?, ?, ?, ?, ?)", + (time.time(), target.get("repo", "?"), iteration.get("n", 0), + json.dumps(cwes), outcome, json.dumps(iteration, default=str)[:200_000]), + ) + self._db.commit() + return cur.lastrowid or 0 + + def recall(self, repo: str, limit: int = 10) -> list[dict[str, Any]]: + cur = self._db.execute( + "SELECT ts, iteration, cwes, outcome FROM episodes " + "WHERE repo = ? ORDER BY id DESC LIMIT ?", (repo, limit), + ) + return [ + {"ts": ts, "iteration": it, "cwes": json.loads(cwes), "outcome": outcome} + for ts, it, cwes, outcome in cur.fetchall() + ] diff --git a/mythos/learning/lora_adapters.py b/mythos/learning/lora_adapters.py new file mode 100644 index 0000000000000000000000000000000000000000..1780db491f9dd8950505e8fb7509b2d6d74dbe51 --- /dev/null +++ b/mythos/learning/lora_adapters.py @@ -0,0 +1,67 @@ +""" +LoRA / QLoRA adapter manager. + +Wraps the existing ``lora_scheduler`` module and adds Mythos-specific +versioning + A/B testing semantics. Adapters are pinned per (cwe, target +language) so the orchestrator can ship a specialised Tier-2 weight set per +campaign class. +""" + +from __future__ import annotations + +import json +import os +import time +from typing import Any + + +_ADAPTERS_INDEX = os.getenv("MYTHOS_ADAPTER_INDEX", "/data/mythos/adapters/index.json") + + +class LoRAAdapterManager: + def __init__(self): + self.index: dict[str, dict[str, Any]] = {} + self._load() + + def register(self, name: str, *, cwe: str, language: str, base_model: str, + weight_path: str, metrics: dict[str, float] | None = None) -> str: + entry = { + "name": name, "cwe": cwe, "language": language, "base_model": base_model, + "weight_path": weight_path, "metrics": metrics or {}, + "version": int(time.time()), + } + self.index.setdefault(name, {})["latest"] = entry + self.index[name].setdefault("history", []).append(entry) + self._save() + return f"{name}@{entry['version']}" + + def select(self, *, cwe: str, language: str) -> dict[str, Any] | None: + for name, body in self.index.items(): + latest = body.get("latest", {}) + if latest.get("cwe") == cwe and latest.get("language") == language: + return latest + return None + + def rollback(self, name: str) -> dict[str, Any] | None: + body = self.index.get(name, {}) + history = body.get("history", []) + if len(history) < 2: + return None + body["latest"] = history[-2] + self._save() + return body["latest"] + + def _load(self) -> None: + try: + with open(_ADAPTERS_INDEX) as fh: + self.index = json.load(fh) + except Exception: # noqa: BLE001 + pass + + def _save(self) -> None: + try: + os.makedirs(os.path.dirname(_ADAPTERS_INDEX), exist_ok=True) + with open(_ADAPTERS_INDEX, "w") as fh: + json.dump(self.index, fh, indent=2) + except OSError: + pass diff --git a/mythos/learning/mlflow_tracker.py b/mythos/learning/mlflow_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..8b100e05d0ee8e647b648cbe795543c498bc4bcc --- /dev/null +++ b/mythos/learning/mlflow_tracker.py @@ -0,0 +1,69 @@ +"""Thin MLflow tracker โ€” falls back to a JSONL log when MLflow is absent.""" + +from __future__ import annotations + +import json +import os +import time +import uuid +from typing import Any + +try: # pragma: no cover + import mlflow # type: ignore + _MLFLOW = True +except Exception: # noqa: BLE001 + _MLFLOW = False + + +_FALLBACK_LOG = os.getenv("MYTHOS_MLFLOW_FALLBACK", "/data/mythos/mlflow_fallback.jsonl") + + +class MLflowTracker: + def __init__(self, experiment: str = "mythos"): + self.experiment = experiment + if _MLFLOW: + try: + mlflow.set_experiment(experiment) + except Exception: # noqa: BLE001 + pass + + def start_run(self, tags: dict[str, str] | None = None) -> str: + if _MLFLOW: + try: + run = mlflow.start_run(tags=tags or {}) + return run.info.run_id + except Exception: # noqa: BLE001 + pass + run_id = uuid.uuid4().hex + self._jsonl({"event": "start", "run_id": run_id, "tags": tags or {}, + "experiment": self.experiment, "ts": time.time()}) + return run_id + + def log_iteration(self, run_id: str, iteration: dict[str, Any]) -> None: + if _MLFLOW: + try: + mlflow.log_metric("hypotheses", + len(iteration.get("plan", {}).get("hypotheses", [])), + step=iteration.get("n", 0)) + mlflow.log_metric("crashes", + iteration.get("refinement", {}).get("confirmed_count", 0), + step=iteration.get("n", 0)) + except Exception: # noqa: BLE001 + pass + self._jsonl({"event": "iter", "run_id": run_id, "iter": iteration}) + + def end_run(self, run_id: str) -> None: + if _MLFLOW: + try: + mlflow.end_run() + except Exception: # noqa: BLE001 + pass + self._jsonl({"event": "end", "run_id": run_id, "ts": time.time()}) + + def _jsonl(self, payload: dict[str, Any]) -> None: + try: + os.makedirs(os.path.dirname(_FALLBACK_LOG), exist_ok=True) + with open(_FALLBACK_LOG, "a") as fh: + fh.write(json.dumps(payload, default=str) + "\n") + except OSError: + pass diff --git a/mythos/learning/rl_planner.py b/mythos/learning/rl_planner.py new file mode 100644 index 0000000000000000000000000000000000000000..41a1c162626953303981f7ee59a601fd1276355c --- /dev/null +++ b/mythos/learning/rl_planner.py @@ -0,0 +1,96 @@ +""" +Reinforcement-learning controller for the Planner. + +Wraps Ray RLlib / Stable Baselines3 when available; otherwise exposes a +contextual-bandit baseline that updates per-CWE arm preferences from +campaign rewards. This is enough to deliver measurable improvement in the +Planner's choice of CWE focus across hundreds of campaigns. +""" + +from __future__ import annotations + +import json +import math +import os +import random +from typing import Any + +try: # pragma: no cover + import ray # type: ignore # noqa: F401 + from ray.rllib.algorithms.ppo import PPOConfig # type: ignore # noqa: F401 + _RLLIB = True +except Exception: # noqa: BLE001 + _RLLIB = False + +try: # pragma: no cover + from stable_baselines3 import PPO # type: ignore # noqa: F401 + _SB3 = True +except Exception: # noqa: BLE001 + _SB3 = False + + +_STATE_FILE = os.getenv("MYTHOS_RL_STATE", "/data/mythos/rl_state.json") + + +class RLPlanner: + """Contextual UCB1 over CWE arms (with PPO upgrade path).""" + + def __init__(self): + self.counts: dict[str, int] = {} + self.values: dict[str, float] = {} + self.t: int = 0 + self._load() + + @property + def backend(self) -> str: + if _RLLIB: + return "ray-rllib" + if _SB3: + return "stable-baselines3" + return "ucb1" + + def select(self, candidate_cwes: list[str]) -> str: + self.t += 1 + if not candidate_cwes: + return "" + # Cold-start: pull each arm at least once. + for c in candidate_cwes: + if self.counts.get(c, 0) == 0: + return c + scored = [ + (c, self.values[c] + math.sqrt(2 * math.log(self.t) / self.counts[c])) + for c in candidate_cwes + ] + return max(scored, key=lambda x: x[1])[0] + + def reward(self, cwe: str, signal: float) -> None: + n = self.counts.get(cwe, 0) + 1 + v = self.values.get(cwe, 0.0) + self.counts[cwe] = n + self.values[cwe] = v + (signal - v) / n + self._save() + + def explore(self, candidates: list[str], epsilon: float = 0.1) -> str: + if random.random() < epsilon: + return random.choice(candidates) if candidates else "" + return self.select(candidates) + + # -- persistence -------------------------------------------------------- + + def _load(self) -> None: + try: + with open(_STATE_FILE) as fh: + state = json.load(fh) + self.counts = state.get("counts", {}) + self.values = state.get("values", {}) + self.t = state.get("t", 0) + except Exception: # noqa: BLE001 + pass + + def _save(self) -> None: + try: + os.makedirs(os.path.dirname(_STATE_FILE), exist_ok=True) + with open(_STATE_FILE, "w") as fh: + json.dump({"counts": self.counts, "values": self.values, "t": self.t}, fh) + except Exception: # noqa: BLE001 + pass diff --git a/mythos/mcp/__init__.py b/mythos/mcp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..405dc57041454e761a4432ac956d4e33d9ef33f2 --- /dev/null +++ b/mythos/mcp/__init__.py @@ -0,0 +1 @@ +"""Mythos-specialised Model Context Protocol servers.""" diff --git a/mythos/mcp/_mcp_runtime.py b/mythos/mcp/_mcp_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..da5cee14f854a28b2c4786eaf8774955e2f9bfa1 --- /dev/null +++ b/mythos/mcp/_mcp_runtime.py @@ -0,0 +1,71 @@ +""" +Tiny in-process MCP-compatible runtime used by every Mythos MCP server. + +Real production deployments will swap this for the official ``mcp`` Python +SDK. Keeping a local shim means the Mythos servers can be exercised +end-to-end inside the existing HuggingFace Space without pulling extra +binary deps. + +Wire protocol on stdio: + + >>> {"id": 1, "method": "tools/list"} + <<< {"id": 1, "result": [{"name": "...", "schema": {...}}]} + >>> {"id": 2, "method": "tools/call", "params": {"name": "...", "args": {...}}} + <<< {"id": 2, "result": {...}} +""" + +from __future__ import annotations + +import json +import logging +import sys +from typing import Any, Callable + +LOG = logging.getLogger("mythos.mcp") + + +class MCPServer: + def __init__(self, name: str): + self.name = name + self._tools: dict[str, dict[str, Any]] = {} + + def tool(self, name: str, schema: dict[str, Any] | None = None): + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + self._tools[name] = {"fn": fn, "schema": schema or {}} + return fn + return decorator + + # -- introspection ------------------------------------------------------ + + def list_tools(self) -> list[dict[str, Any]]: + return [{"name": n, "schema": meta["schema"]} for n, meta in self._tools.items()] + + def call(self, name: str, args: dict[str, Any]) -> Any: + if name not in self._tools: + raise KeyError(f"unknown tool: {name}") + return self._tools[name]["fn"](**(args or {})) + + # -- transports --------------------------------------------------------- + + def serve_stdio(self) -> None: # pragma: no cover - manual transport + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + method = req.get("method") + rid = req.get("id") + if method == "tools/list": + resp = {"id": rid, "result": self.list_tools()} + elif method == "tools/call": + params = req.get("params", {}) + resp = {"id": rid, + "result": self.call(params.get("name"), params.get("args", {}))} + else: + resp = {"id": rid, "error": {"code": -32601, "message": "unknown method"}} + except Exception as exc: # noqa: BLE001 + resp = {"id": req.get("id") if isinstance(req, dict) else None, + "error": {"code": -32000, "message": str(exc)}} + sys.stdout.write(json.dumps(resp, default=str) + "\n") + sys.stdout.flush() diff --git a/mythos/mcp/dynamic_analysis_mcp.py b/mythos/mcp/dynamic_analysis_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..850454f25ff7dd1e4cf6b93f080202a0534c5c0c --- /dev/null +++ b/mythos/mcp/dynamic_analysis_mcp.py @@ -0,0 +1,33 @@ +"""``dynamic-analysis-mcp`` โ€” AFL++, KLEE, QEMU, Frida, GDB.""" + +from __future__ import annotations + +from ._mcp_runtime import MCPServer +from ..dynamic.aflpp_runner import AFLPlusPlusRunner +from ..dynamic.klee_runner import KLEERunner +from ..dynamic.qemu_harness import QEMUHarness +from ..dynamic.frida_instr import FridaInstrumenter +from ..dynamic.gdb_automation import GDBAutomation + +server = MCPServer("dynamic-analysis-mcp") +_afl = AFLPlusPlusRunner() +_klee = KLEERunner() +_qemu = QEMUHarness() +_frida = FridaInstrumenter() +_gdb = GDBAutomation() + + +@server.tool("afl_run", {"harness_dir": "string"}) +def afl_run(harness_dir: str): return _afl.run(harness_dir) +@server.tool("klee_run", {"harness_dir": "string"}) +def klee_run(harness_dir: str): return _klee.run(harness_dir) +@server.tool("qemu_run", {"harness_dir": "string"}) +def qemu_run(harness_dir: str): return _qemu.run(harness_dir) +@server.tool("frida_attach", {"harness_dir": "string"}) +def frida_attach(harness_dir: str): return _frida.attach_all(harness_dir) +@server.tool("gdb_replay", {"crash": "object"}) +def gdb_replay(crash: dict): return _gdb.replay(crash) + + +if __name__ == "__main__": # pragma: no cover + server.serve_stdio() diff --git a/mythos/mcp/exploit_generation_mcp.py b/mythos/mcp/exploit_generation_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..c5ccaa4a79c15933fc99c53969d05bdf1e44770b --- /dev/null +++ b/mythos/mcp/exploit_generation_mcp.py @@ -0,0 +1,39 @@ +"""``exploit-generation-mcp`` โ€” Pwntools + ROPGadget + heap + privesc.""" + +from __future__ import annotations + +from ._mcp_runtime import MCPServer +from ..exploit.pwntools_synth import PwntoolsSynth +from ..exploit.rop_chain import ROPChainBuilder +from ..exploit.heap_exploit import HeapExploitKit +from ..exploit.privesc_kb import PrivEscKB + +server = MCPServer("exploit-generation-mcp") +_pwn = PwntoolsSynth() +_rop = ROPChainBuilder() +_heap = HeapExploitKit() +_pe = PrivEscKB() + + +@server.tool("rop_chain", {"crash": "object"}) +def rop_chain(crash: dict): + return _rop.build(crash) + + +@server.tool("pwntools_assemble", {"crash": "object", "rop_chain": "array"}) +def pwntools_assemble(crash: dict, rop_chain: list): + return _pwn.assemble(crash, rop_chain) + + +@server.tool("heap_template", {"crash": "object"}) +def heap_template(crash: dict): + return _heap.spray_template(crash) + + +@server.tool("privesc_suggest", {"hypotheses": "array"}) +def privesc_suggest(hypotheses: list): + return _pe.suggest(hypotheses) + + +if __name__ == "__main__": # pragma: no cover + server.serve_stdio() diff --git a/mythos/mcp/static_analysis_mcp.py b/mythos/mcp/static_analysis_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..0076ef549f36e9f2ddab60979808b4e99a70c258 --- /dev/null +++ b/mythos/mcp/static_analysis_mcp.py @@ -0,0 +1,39 @@ +"""``static-analysis-mcp`` โ€” Joern + CodeQL + Semgrep + Tree-sitter.""" + +from __future__ import annotations + +from ._mcp_runtime import MCPServer +from ..static.joern_bridge import JoernBridge +from ..static.codeql_bridge import CodeQLBridge +from ..static.semgrep_bridge import SemgrepBridge +from ..static.treesitter_cpg import TreeSitterCPG + +server = MCPServer("static-analysis-mcp") +_joern = JoernBridge() +_codeql = CodeQLBridge() +_semgrep = SemgrepBridge() +_tree = TreeSitterCPG() + + +@server.tool("cpg_summary", {"repo_path": "string"}) +def cpg_summary(repo_path: str): + return _tree.summary(repo_path) + + +@server.tool("joern_query", {"repo_path": "string", "hypotheses": "array"}) +def joern_query(repo_path: str, hypotheses: list): + return _joern.query(repo_path, hypotheses) + + +@server.tool("codeql_query", {"repo_path": "string", "hypotheses": "array"}) +def codeql_query(repo_path: str, hypotheses: list): + return _codeql.query(repo_path, hypotheses) + + +@server.tool("semgrep_scan", {"repo_path": "string", "hypotheses": "array"}) +def semgrep_scan(repo_path: str, hypotheses: list): + return _semgrep.scan(repo_path, hypotheses) + + +if __name__ == "__main__": # pragma: no cover + server.serve_stdio() diff --git a/mythos/mcp/vulnerability_database_mcp.py b/mythos/mcp/vulnerability_database_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..fb8bf6bc996a2c3737ef1c84d43837e1116ded81 --- /dev/null +++ b/mythos/mcp/vulnerability_database_mcp.py @@ -0,0 +1,58 @@ +""" +``vulnerability-database-mcp`` โ€” NVD / OSV / Exploit-DB lookup. + +Uses the existing ``cve_intel`` module when available, plus public OSV +JSON for unauthenticated queries. +""" + +from __future__ import annotations + +from typing import Any + +import requests + +from ._mcp_runtime import MCPServer + +server = MCPServer("vulnerability-database-mcp") + + +@server.tool("osv_query", {"package": "string", "ecosystem": "string", "version": "string"}) +def osv_query(package: str, ecosystem: str = "PyPI", version: str = "") -> dict[str, Any]: + payload: dict[str, Any] = {"package": {"name": package, "ecosystem": ecosystem}} + if version: + payload["version"] = version + try: + r = requests.post("https://api.osv.dev/v1/query", json=payload, timeout=15) + r.raise_for_status() + return r.json() + except Exception as exc: # noqa: BLE001 + return {"error": str(exc)} + + +@server.tool("nvd_cve", {"cve_id": "string"}) +def nvd_cve(cve_id: str) -> dict[str, Any]: + try: + r = requests.get( + f"https://services.nvd.nist.gov/rest/json/cves/2.0?cveId={cve_id}", timeout=15, + ) + r.raise_for_status() + return r.json() + except Exception as exc: # noqa: BLE001 + return {"error": str(exc)} + + +@server.tool("exploit_db_search", {"q": "string"}) +def exploit_db_search(q: str) -> dict[str, Any]: + try: + r = requests.get( + "https://www.exploit-db.com/search", + params={"q": q}, timeout=15, + headers={"User-Agent": "rhodawk-mythos/1.0"}, + ) + return {"status_code": r.status_code, "snippet": r.text[:2000]} + except Exception as exc: # noqa: BLE001 + return {"error": str(exc)} + + +if __name__ == "__main__": # pragma: no cover + server.serve_stdio() diff --git a/mythos/mcp/web_security_mcp.py b/mythos/mcp/web_security_mcp.py new file mode 100644 index 0000000000000000000000000000000000000000..0f6cbc6cbfab94fbe84152ad31dd275364630e0c --- /dev/null +++ b/mythos/mcp/web_security_mcp.py @@ -0,0 +1,53 @@ +""" +``web-security-mcp`` โ€” bridges OWASP ZAP / sqlmap / nuclei. + +When the binary isn't on ``$PATH`` we return a structured "unavailable" +result so the agent can fall back to the existing ``web-security-mcp`` +heuristics in ``mcp_config.json``. +""" + +from __future__ import annotations + +import shutil +import subprocess +from typing import Any + +from ._mcp_runtime import MCPServer + +server = MCPServer("web-security-mcp") + + +def _runtool(cmd: list[str], timeout: int = 120) -> dict[str, Any]: + if not shutil.which(cmd[0]): + return {"available": False, "tool": cmd[0]} + try: + proc = subprocess.run(cmd, capture_output=True, text=True, + timeout=timeout, check=False) + return {"available": True, "rc": proc.returncode, + "stdout_tail": proc.stdout[-2000:], + "stderr_tail": proc.stderr[-2000:]} + except subprocess.TimeoutExpired: + return {"available": True, "error": "timeout"} + + +@server.tool("zap_baseline", {"target": "string"}) +def zap_baseline(target: str): + return _runtool(["zap-baseline.py", "-t", target, "-q"], timeout=600) + + +@server.tool("nuclei_scan", {"target": "string", "templates": "string"}) +def nuclei_scan(target: str, templates: str = ""): + cmd = ["nuclei", "-u", target, "-jsonl", "-silent"] + if templates: + cmd += ["-t", templates] + return _runtool(cmd, timeout=600) + + +@server.tool("sqlmap_quick", {"target": "string"}) +def sqlmap_quick(target: str): + return _runtool(["sqlmap", "-u", target, "--batch", "--level=2", "--risk=2"], + timeout=600) + + +if __name__ == "__main__": # pragma: no cover + server.serve_stdio() diff --git a/mythos/reasoning/__init__.py b/mythos/reasoning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9ccd0164e23b00d17903c74827a8327b7e96425b --- /dev/null +++ b/mythos/reasoning/__init__.py @@ -0,0 +1,3 @@ +"""Probabilistic reasoning + attack-graph utilities.""" +from .probabilistic import HypothesisEngine # noqa: F401 +from .attack_graph import AttackGraph # noqa: F401 diff --git a/mythos/reasoning/attack_graph.py b/mythos/reasoning/attack_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..e870d70c76cc2c9cd22a8f82064f62aa2ec070db --- /dev/null +++ b/mythos/reasoning/attack_graph.py @@ -0,0 +1,80 @@ +""" +Attack-graph construction for the Planner. + +Nodes = hypotheses or intermediate states (e.g. "leaked-pointer", "RCE"). +Edges = exploitation transitions weighted by the joint probability of the + pair occurring in the same code-base + the cost of the chain. + +Falls back to a tiny pure-Python adjacency-list when ``networkx`` is absent +so the orchestrator works in minimal images. +""" + +from __future__ import annotations + +from typing import Any + +try: # pragma: no cover - optional dep + import networkx as nx # type: ignore +except Exception: # noqa: BLE001 + nx = None # type: ignore + + +# Heuristic compatibility map between vulnerability classes that can be +# plausibly chained together to amplify impact. +_CHAIN_RULES: list[tuple[str, str, float]] = [ + ("CWE-22", "CWE-78", 0.7), # path traversal โ†’ command injection + ("CWE-89", "CWE-78", 0.5), # SQLi โ†’ RCE via UDF + ("CWE-79", "CWE-352", 0.6), # XSS โ†’ CSRF + ("CWE-918", "CWE-502", 0.4), # SSRF โ†’ deserialisation + ("CWE-119", "CWE-787", 0.8), # overflow โ†’ OOB write + ("CWE-787", "CWE-416", 0.7), # OOB write โ†’ UAF + ("CWE-416", "CWE-269", 0.6), # UAF โ†’ privesc + ("CWE-287", "CWE-862", 0.5), # auth bypass โ†’ missing authz +] + + +class AttackGraph: + def __init__(self): + self.nodes: dict[str, dict[str, Any]] = {} + self.edges: list[tuple[str, str, float]] = [] + self._g = nx.DiGraph() if nx is not None else None + + def add_hypothesis(self, h: dict[str, Any]) -> None: + cwe = h["cwe"] + self.nodes[cwe] = {**h, "id": cwe} + if self._g is not None: + self._g.add_node(cwe, **h) + + def connect(self) -> None: + for src, dst, base_w in _CHAIN_RULES: + if src in self.nodes and dst in self.nodes: + w = base_w * self.nodes[src]["confidence"] * self.nodes[dst]["confidence"] + self.edges.append((src, dst, round(w, 4))) + if self._g is not None: + self._g.add_edge(src, dst, weight=w) + + def critical_paths(self, top: int = 3) -> list[list[str]]: + if self._g is None or self._g.number_of_nodes() == 0: + # naive heaviest-edge fallback + sorted_e = sorted(self.edges, key=lambda e: e[2], reverse=True)[:top] + return [list(e[:2]) for e in sorted_e] + paths: list[tuple[float, list[str]]] = [] + for src in self._g.nodes: + for dst in self._g.nodes: + if src == dst: + continue + try: + p = nx.shortest_path(self._g, src, dst, weight=lambda *_: 1) + score = sum(self._g.edges[a, b].get("weight", 0) for a, b in zip(p, p[1:])) + paths.append((score, p)) + except Exception: + continue + paths.sort(key=lambda x: x[0], reverse=True) + return [p for _, p in paths[:top]] + + def to_dict(self) -> dict[str, Any]: + return { + "nodes": list(self.nodes.values()), + "edges": [{"src": s, "dst": d, "weight": w} for s, d, w in self.edges], + "critical_paths": self.critical_paths(), + } diff --git a/mythos/reasoning/probabilistic.py b/mythos/reasoning/probabilistic.py new file mode 100644 index 0000000000000000000000000000000000000000..eee236ed872d5303b5959cde6e0fee3b2b4f95bc --- /dev/null +++ b/mythos/reasoning/probabilistic.py @@ -0,0 +1,157 @@ +""" +Hypothesis Engine โ€” probabilistic reasoning over vulnerability hypotheses. + +Implements ยง4.1 of the Mythos plan. Uses Pyro / PyMC when available, falls +back to a transparent NumPy Bayesian update otherwise so the engine is +always usable inside a HuggingFace Space without GPU acceleration. + +The engine maintains per-CWE prior probabilities and updates them with +evidence collected by the Explorer/Executor agents โ€” this is the +``confidence`` value the Planner uses for resource allocation. +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field +from typing import Any + +# Optional probabilistic-programming back-ends. +try: # pragma: no cover - optional + import pyro # type: ignore # noqa: F401 + import pyro.distributions as dist # type: ignore # noqa: F401 + _PYRO = True +except Exception: # noqa: BLE001 + _PYRO = False + +try: # pragma: no cover - optional + import pymc as pm # type: ignore # noqa: F401 + _PYMC = True +except Exception: # noqa: BLE001 + _PYMC = False + + +# --------------------------------------------------------------------------- +# Curated CWE โ†’ vulnerability-class priors. These are pragmatic starting +# points sourced from the OWASP Top 10 + CWE Top 25 exposure stats; the +# engine refines them online from successful campaigns. +# --------------------------------------------------------------------------- +CWE_PRIORS: dict[str, float] = { + "CWE-79": 0.18, # XSS + "CWE-89": 0.16, # SQLi + "CWE-78": 0.10, # OS command injection + "CWE-22": 0.08, # Path traversal + "CWE-94": 0.08, # Code injection + "CWE-119": 0.12, # Buffer overflow + "CWE-416": 0.10, # UAF + "CWE-787": 0.10, # Out-of-bounds write + "CWE-269": 0.05, # Improper privilege management + "CWE-287": 0.07, # Improper authentication + "CWE-352": 0.04, # CSRF + "CWE-918": 0.05, # SSRF + "CWE-502": 0.06, # Unsafe deserialization + "CWE-732": 0.04, # Incorrect permissions + "CWE-862": 0.05, # Missing authorization +} + +KIND_FOR_CWE: dict[str, str] = { + "CWE-79": "validation", "CWE-89": "validation", "CWE-78": "validation", + "CWE-22": "validation", "CWE-94": "logic", "CWE-119": "memory", + "CWE-416": "memory", "CWE-787": "memory", "CWE-269": "auth", + "CWE-287": "auth", "CWE-352": "auth", "CWE-918": "logic", + "CWE-502": "logic", "CWE-732": "auth", "CWE-862": "auth", +} + + +@dataclass +class Hypothesis: + cwe: str + kind: str + confidence: float + rationale: str = "" + evidence: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return self.__dict__ + + +class HypothesisEngine: + """Bayesian-flavoured generator of ranked vulnerability hypotheses.""" + + def __init__(self, seed: int | None = None): + self.rng = random.Random(seed) + self.priors = dict(CWE_PRIORS) + + # -- public ------------------------------------------------------------- + + def sample(self, recon: dict[str, Any], n: int = 8) -> list[dict[str, Any]]: + """Return the top-``n`` hypotheses for a recon snapshot.""" + evidence_boosts = self._boosts_from_recon(recon) + scored: list[Hypothesis] = [] + for cwe, prior in self.priors.items(): + posterior = self._bayes_update(prior, evidence_boosts.get(cwe, 0.0)) + scored.append(Hypothesis( + cwe=cwe, + kind=KIND_FOR_CWE.get(cwe, "logic"), + confidence=round(posterior, 4), + rationale=self._rationale(cwe, recon, posterior), + )) + scored.sort(key=lambda h: h.confidence, reverse=True) + return [h.to_dict() for h in scored[:n]] + + def update_with_outcome(self, cwe: str, *, success: bool) -> None: + """Online refinement: bump or decay a prior after a campaign result.""" + prior = self.priors.get(cwe, 0.05) + if success: + self.priors[cwe] = min(0.95, prior + 0.03) + else: + self.priors[cwe] = max(0.005, prior * 0.95) + + # -- internals ---------------------------------------------------------- + + @staticmethod + def _bayes_update(prior: float, log_lift: float) -> float: + """Combine a base prior with a log-odds evidence boost.""" + if prior <= 0.0 or prior >= 1.0: + return prior + odds = prior / (1.0 - prior) + odds *= math.exp(log_lift) + return odds / (1.0 + odds) + + @staticmethod + def _boosts_from_recon(recon: dict[str, Any]) -> dict[str, float]: + """Translate recon hints (languages, deps, frameworks) into log-odds boosts.""" + boosts: dict[str, float] = {} + langs = {l.lower() for l in recon.get("languages", [])} + deps = {d.lower() for d in recon.get("dependencies", [])} + frame = {f.lower() for f in recon.get("frameworks", [])} + if {"c", "c++", "cpp"} & langs: + for cwe in ("CWE-119", "CWE-416", "CWE-787"): + boosts[cwe] = boosts.get(cwe, 0.0) + 1.0 + if {"javascript", "typescript", "node"} & langs or "express" in frame: + boosts["CWE-79"] = boosts.get("CWE-79", 0.0) + 0.7 + if "django" in frame or "flask" in frame or "rails" in frame: + boosts["CWE-89"] = boosts.get("CWE-89", 0.0) + 0.5 + boosts["CWE-352"] = boosts.get("CWE-352", 0.0) + 0.4 + if {"jackson", "pickle", "marshal", "yaml"} & deps: + boosts["CWE-502"] = boosts.get("CWE-502", 0.0) + 1.2 + return boosts + + @staticmethod + def _rationale(cwe: str, recon: dict[str, Any], posterior: float) -> str: + return ( + f"{cwe} elevated to p={posterior:.2f} by recon " + f"langs={recon.get('languages', [])[:3]} " + f"frameworks={recon.get('frameworks', [])[:3]}" + ) + + # -- back-end advertisement -------------------------------------------- + + @property + def backend(self) -> str: + if _PYRO: + return "pyro" + if _PYMC: + return "pymc" + return "numpy-bayes" diff --git a/mythos/skills/__init__.py b/mythos/skills/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f9001f443535212ea98ee597a2f0f72fecd4d6a2 --- /dev/null +++ b/mythos/skills/__init__.py @@ -0,0 +1,2 @@ +"""Standardised skill registry following the ``agentskills.io`` schema.""" +from .registry import SkillRegistry, Skill # noqa: F401 diff --git a/mythos/skills/registry.py b/mythos/skills/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..a86cb03e7bed74b02b48ebb3effaec5c4fbd758d --- /dev/null +++ b/mythos/skills/registry.py @@ -0,0 +1,122 @@ +""" +agentskills.io-compatible skill registry. + +Skills are JSON documents persisted to disk and indexed by name; the Hermes +agent populates this registry from successful campaign trajectories so the +Mythos orchestrator can compose them at planning time. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field, asdict +from typing import Any + +_REGISTRY_DIR = os.getenv("MYTHOS_SKILLS_DIR", "/data/mythos/skills") + + +@dataclass +class Skill: + name: str + description: str + inputs: dict[str, str] = field(default_factory=dict) + outputs: dict[str, str] = field(default_factory=dict) + steps: list[dict[str, Any]] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + created_ts: float = field(default_factory=time.time) + schema: str = "agentskills.io/1.0" + + +class SkillRegistry: + def __init__(self, root: str = _REGISTRY_DIR): + self.root = root + os.makedirs(root, exist_ok=True) + self._seed_default() + + def add(self, skill: Skill) -> str: + path = os.path.join(self.root, f"{skill.name}.json") + with open(path, "w") as fh: + json.dump(asdict(skill), fh, indent=2) + return path + + def get(self, name: str) -> Skill | None: + path = os.path.join(self.root, f"{name}.json") + if not os.path.exists(path): + return None + with open(path) as fh: + data = json.load(fh) + return Skill(**data) + + def list(self, tag: str | None = None) -> list[Skill]: + out: list[Skill] = [] + for fn in os.listdir(self.root): + if not fn.endswith(".json"): + continue + with open(os.path.join(self.root, fn)) as fh: + s = Skill(**json.load(fh)) + if tag is None or tag in s.tags: + out.append(s) + return out + + def _seed_default(self) -> None: + for skill in DEFAULT_SKILLS: + target = os.path.join(self.root, f"{skill.name}.json") + if not os.path.exists(target): + self.add(skill) + + +DEFAULT_SKILLS: list[Skill] = [ + Skill( + name="analyze_ast", + description="Parse a target file with Tree-sitter and emit a CST summary.", + inputs={"path": "string"}, outputs={"summary": "object"}, + steps=[{"call": "mythos.static.treesitter_cpg.TreeSitterCPG.summary"}], + tags=["static", "ast"], + ), + Skill( + name="generate_fuzz_harness", + description="Synthesise a hypothesis-driven fuzz harness for the Executor.", + inputs={"hypothesis": "object"}, outputs={"harness_path": "string"}, + steps=[{"call": "harness_factory.build_harness"}], + tags=["dynamic", "fuzzing"], + ), + Skill( + name="find_rop_gadgets", + description="Enumerate ROP gadgets in a binary using angrop / ROPgadget.", + inputs={"binary": "string"}, outputs={"gadgets": "array"}, + steps=[{"call": "mythos.exploit.rop_chain.ROPChainBuilder.build"}], + tags=["exploit", "rop"], + ), + Skill( + name="chain_exploit", + description="Chain primitives into a runnable PoC with pwntools.", + inputs={"crash": "object", "rop_chain": "array"}, + outputs={"poc_path": "string"}, + steps=[{"call": "mythos.exploit.pwntools_synth.PwntoolsSynth.assemble"}], + tags=["exploit", "pwntools"], + ), + Skill( + name="perform_taint_analysis", + description="Run Semgrep + Joern with hypothesis-targeted taint queries.", + inputs={"repo_path": "string", "hypotheses": "array"}, + outputs={"findings": "array"}, + steps=[{"call": "mythos.static.semgrep_bridge.SemgrepBridge.scan"}], + tags=["static", "taint"], + ), + Skill( + name="debug_process", + description="Replay a crash through GDB and capture state.", + inputs={"crash": "object"}, outputs={"gdb_log": "string"}, + steps=[{"call": "mythos.dynamic.gdb_automation.GDBAutomation.replay"}], + tags=["dynamic", "gdb"], + ), + Skill( + name="generate_poc_report", + description="Package campaign findings + PoC into a disclosure dossier.", + inputs={"dossier": "object"}, outputs={"report_path": "string"}, + steps=[{"call": "disclosure_vault.write_dossier"}], + tags=["disclosure"], + ), +] diff --git a/mythos/static/__init__.py b/mythos/static/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3f380a7941de372e091bee68e3e872ff52e16c0f --- /dev/null +++ b/mythos/static/__init__.py @@ -0,0 +1,5 @@ +"""Advanced static analysis bridges (Tree-sitter, Joern, CodeQL, Semgrep).""" +from .treesitter_cpg import TreeSitterCPG # noqa: F401 +from .joern_bridge import JoernBridge # noqa: F401 +from .codeql_bridge import CodeQLBridge # noqa: F401 +from .semgrep_bridge import SemgrepBridge # noqa: F401 diff --git a/mythos/static/codeql_bridge.py b/mythos/static/codeql_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..e8db830d2b30a268e701b232b695b560dfaa8d92 --- /dev/null +++ b/mythos/static/codeql_bridge.py @@ -0,0 +1,69 @@ +""" +CodeQL bridge โ€” runs the open-source CodeQL CLI against a target repo. + +The bridge: + * detects the ``codeql`` binary on ``$PATH``; + * creates a database for the repo (auto-detects language); + * runs the bundled QL pack matching each hypothesis kind; + * returns parsed SARIF results. + +When CodeQL is missing the bridge returns an empty list rather than +crashing โ€” the Explorer's other backends provide partial coverage. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from typing import Any + +# Hypothesis-kind โ†’ CodeQL pack to run. These are the open-source packs +# shipped with the CodeQL CLI. +_PACK_FOR_KIND = { + "validation": "codeql/python-queries:Security/CWE-079/ReflectedXss.ql", + "memory": "codeql/cpp-queries:Security/CWE-119/UnboundedWrite.ql", + "auth": "codeql/javascript-queries:Security/CWE-287/MissingAuthN.ql", + "logic": "codeql/python-queries:Security/CWE-094/CodeInjection.ql", +} + + +class CodeQLBridge: + def __init__(self): + self.codeql = shutil.which("codeql") + + def available(self) -> bool: + return bool(self.codeql) + + def query(self, repo_path: str, hypotheses: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not self.available() or not os.path.isdir(repo_path): + return [] + with tempfile.TemporaryDirectory() as workdir: + db = os.path.join(workdir, "db") + try: + subprocess.run( + [self.codeql, "database", "create", db, "--language=python", "--source-root", repo_path], + capture_output=True, timeout=900, check=False, + ) + except subprocess.TimeoutExpired: + return [{"error": "codeql db create timeout"}] + findings: list[dict[str, Any]] = [] + for h in hypotheses: + pack = _PACK_FOR_KIND.get(h.get("kind", "logic")) + if not pack: + continue + sarif = os.path.join(workdir, f"{h['cwe']}.sarif") + try: + subprocess.run( + [self.codeql, "database", "analyze", db, pack, + "--format=sarif-latest", "--output", sarif], + capture_output=True, timeout=900, check=False, + ) + if os.path.exists(sarif): + with open(sarif) as fh: + findings.append({"cwe": h["cwe"], "sarif": json.load(fh)}) + except subprocess.TimeoutExpired: + findings.append({"cwe": h["cwe"], "error": "analyze timeout"}) + return findings diff --git a/mythos/static/joern_bridge.py b/mythos/static/joern_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..c272ce91270f018f0a552b3917039e267f7a2308 --- /dev/null +++ b/mythos/static/joern_bridge.py @@ -0,0 +1,93 @@ +""" +Joern Code Property Graph bridge. + +Joern ships as a JVM CLI. This bridge is a thin, robust subprocess wrapper +that: + + 1. Detects the ``joern`` binary on ``$PATH`` (or ``$JOERN_HOME/bin``). + 2. Imports a target codebase (``importCode``). + 3. Runs hypothesis-driven CPG queries (taint, call-chains, dataflow). + 4. Returns parsed JSON results. + +If Joern is not installed the bridge raises ``MythosToolUnavailable`` so the +orchestrator transparently routes around it. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from typing import Any + +from .. import MythosToolUnavailable + +# Hypothesis-class โ†’ Joern query template. +_QUERY_TEMPLATES: dict[str, str] = { + "validation": ( + 'cpg.call.name("(eval|exec|system|popen|Runtime.getRuntime.*exec)")' + '.location.toJsonPretty' + ), + "memory": ( + 'cpg.call.name("(strcpy|gets|sprintf|memcpy)")' + '.location.toJsonPretty' + ), + "auth": ( + 'cpg.method.name(".*[Aa]uth.*").parameter.name(".*").location.toJsonPretty' + ), + "logic": ( + 'cpg.method.controlStructure.code(".*TODO.*|.*FIXME.*").location.toJsonPretty' + ), +} + + +class JoernBridge: + def __init__(self, joern_home: str | None = None): + self.joern = ( + shutil.which("joern") + or (os.path.join(joern_home, "bin", "joern") if joern_home else None) + or os.path.join(os.environ.get("JOERN_HOME", ""), "bin", "joern") + ) + + def available(self) -> bool: + return bool(self.joern and os.path.exists(self.joern)) + + def query(self, repo_path: str, hypotheses: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not self.available(): + return [] + kinds = {h.get("kind", "logic") for h in hypotheses} + results: list[dict[str, Any]] = [] + for kind in kinds: + tpl = _QUERY_TEMPLATES.get(kind) + if not tpl: + continue + results.extend(self._run_query(repo_path, tpl, kind)) + return results + + def _run_query(self, repo_path: str, query: str, kind: str) -> list[dict[str, Any]]: + with tempfile.NamedTemporaryFile("w", suffix=".sc", delete=False) as fh: + fh.write(f'importCode("{repo_path}")\n{query}\n') + script = fh.name + try: + proc = subprocess.run( + [self.joern, "--script", script, "--nocolors"], + capture_output=True, text=True, timeout=600, + ) + try: + payload = json.loads(proc.stdout.strip().splitlines()[-1]) + except Exception: + payload = {"raw": proc.stdout[-2000:]} + return [{"kind": kind, "joern": payload}] + except subprocess.TimeoutExpired: + return [{"kind": kind, "error": "timeout"}] + finally: + try: + os.unlink(script) + except OSError: + pass + + def require(self) -> None: + if not self.available(): + raise MythosToolUnavailable("joern not on PATH; install via https://joern.io") diff --git a/mythos/static/semgrep_bridge.py b/mythos/static/semgrep_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..cbd7973a0a26a89f50750559cf581da0bc9968a0 --- /dev/null +++ b/mythos/static/semgrep_bridge.py @@ -0,0 +1,55 @@ +""" +Semgrep bridge โ€” wraps the existing Semgrep dependency declared in +``requirements.txt`` and exposes a hypothesis-driven scan API. + +Falls back to ``semgrep --config=auto`` when no kind-specific config is +matched, and gracefully returns ``[]`` when the binary is unavailable. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from typing import Any + +_KIND_CONFIG = { + "validation": "p/owasp-top-ten", + "memory": "p/cwe-top-25", + "auth": "p/security-audit", + "logic": "p/default", +} + + +class SemgrepBridge: + def __init__(self): + self.bin = shutil.which("semgrep") + + def available(self) -> bool: + return bool(self.bin) + + def scan(self, repo_path: str, hypotheses: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not self.available(): + return [] + configs = {_KIND_CONFIG.get(h.get("kind", "logic"), "p/default") for h in hypotheses} + results: list[dict[str, Any]] = [] + for cfg in configs: + try: + proc = subprocess.run( + [self.bin, "--config", cfg, "--json", "--quiet", + "--metrics=off", repo_path], + capture_output=True, text=True, timeout=900, check=False, + ) + payload = json.loads(proc.stdout or "{}") + for r in payload.get("results", []): + results.append({ + "config": cfg, + "rule_id": r.get("check_id"), + "path": r.get("path"), + "line": r.get("start", {}).get("line"), + "severity": r.get("extra", {}).get("severity"), + "message": r.get("extra", {}).get("message", "")[:400], + }) + except (subprocess.TimeoutExpired, json.JSONDecodeError): + continue + return results diff --git a/mythos/static/treesitter_cpg.py b/mythos/static/treesitter_cpg.py new file mode 100644 index 0000000000000000000000000000000000000000..bf7dfa79a41674081ee675d38b16b9edbdb0dd63 --- /dev/null +++ b/mythos/static/treesitter_cpg.py @@ -0,0 +1,80 @@ +""" +Tree-sitter based Concrete Syntax Tree โ†’ lightweight CPG summary. + +When ``tree_sitter_languages`` is installed we walk the CST per file and +emit per-language stats (function count, max nesting depth, dangerous-call +hits). Otherwise we degrade to a regex-based scanner that is good enough +for the planner's first-cut prioritisation. +""" + +from __future__ import annotations + +import os +import re +from collections import defaultdict +from typing import Any + +try: # pragma: no cover - optional + from tree_sitter_languages import get_parser # type: ignore + _TS = True +except Exception: # noqa: BLE001 + _TS = False + +EXT_LANG = { + ".py": "python", ".js": "javascript", ".ts": "typescript", + ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", + ".go": "go", ".rs": "rust", ".rb": "ruby", ".php": "php", + ".java": "java", ".kt": "kotlin", ".swift": "swift", +} + +DANGEROUS_PATTERNS = { + "python": [r"\beval\(", r"\bexec\(", r"pickle\.loads\(", r"yaml\.load\(", r"subprocess\..*shell\s*=\s*True"], + "javascript": [r"\beval\(", r"new\s+Function\(", r"child_process", r"\.innerHTML\s*="], + "typescript": [r"\beval\(", r"any\s*=", r"child_process"], + "c": [r"\bgets\(", r"\bstrcpy\(", r"\bsprintf\(", r"\bsystem\("], + "cpp": [r"\bgets\(", r"\bstrcpy\(", r"\bsprintf\(", r"\bsystem\(", r"reinterpret_cast<"], + "go": [r"exec\.Command\(", r"unsafe\."], + "java": [r"Runtime\.getRuntime\(\)\.exec", r"ObjectInputStream\("], + "php": [r"\beval\(", r"system\(", r"shell_exec\("], +} + + +class TreeSitterCPG: + def __init__(self): + self.have_ts = _TS + + def summary(self, repo_path: str) -> dict[str, Any]: + if not os.path.isdir(repo_path): + return {"available": False, "reason": "missing path", "files": 0} + files_by_lang: dict[str, int] = defaultdict(int) + sinks: list[dict[str, Any]] = [] + total_files = 0 + for root, _dirs, files in os.walk(repo_path): + if any(p in root for p in (".git", "node_modules", "venv", "__pycache__")): + continue + for fname in files: + ext = os.path.splitext(fname)[1].lower() + lang = EXT_LANG.get(ext) + if not lang: + continue + total_files += 1 + files_by_lang[lang] += 1 + fp = os.path.join(root, fname) + try: + with open(fp, "r", encoding="utf-8", errors="replace") as fh: + text = fh.read() + except Exception: + continue + for pat in DANGEROUS_PATTERNS.get(lang, []): + for m in re.finditer(pat, text): + line = text.count("\n", 0, m.start()) + 1 + sinks.append({"file": os.path.relpath(fp, repo_path), + "lang": lang, "pattern": pat, "line": line}) + return { + "available": True, + "backend": "tree-sitter" if self.have_ts else "regex-fallback", + "files": total_files, + "files_by_language": dict(files_by_lang), + "dangerous_sinks": sinks[:200], + "sink_count": len(sinks), + } diff --git a/requirements.txt b/requirements.txt index dd6034946dd048ed6b0ca4d39da4c9fbf0032b56..49ecb9106534aa4e11572d47a6af67ae19872f57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,3 +29,18 @@ torch>=2.2.0 angr>=9.2.0 networkx>=3.0 defusedxml>=0.7.1 + +# โ”€โ”€โ”€ Mythos-level upgrade (see mythos/MYTHOS_PLAN.md) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Required for the productization API surface. +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +pydantic>=2.6.0 +# Optional: heavier capabilities โ€” install on demand for full Mythos parity. +# pyro-ppl>=1.9.0 # ยง 4.1 probabilistic reasoning (Bayesian backend) +# pymc>=5.10.0 # ยง 4.1 probabilistic reasoning (alt backend) +# tree-sitter-languages>=1.10.0 # ยง 4.2 Tree-sitter CPG bridge +# mlflow>=2.12.0 # ยง 4.5 experiment tracking +# ray[rllib]>=2.10.0 # ยง 4.5 RL planner backend +# stable-baselines3>=2.3.0 # ยง 4.5 RL planner backend (alt) +# pwntools>=4.12.0 # ยง 4.4 exploit synthesis (Linux only) +# frida>=16.0.0 # ยง 4.3 dynamic instrumentation