#!/usr/bin/env python3 """Run the pinned conversion, included verification, and optional repeat build.""" from __future__ import annotations import argparse import hashlib import json import os import platform import sys import tempfile from pathlib import Path import numpy as np from convert import convert from recipe import ( ARCHITECTURE, DEFAULT_RECIPE, RECIPES, REPOSITORY_ROOT, RECIPE_VERSION, SOURCE_REPOSITORY, SOURCE_REVISION, ArtifactRecipe, resolve_recipe, ) from verify import verify, write_report def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.partial-{os.getpid()}") with temporary.open("w", encoding="utf-8", newline="\n") as handle: json.dump(value, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) def _update_sha256sums(path: Path, digest: str, filename: str) -> None: entries: dict[str, str] = {} if path.exists(): with path.open(encoding="ascii") as handle: for line_number, line in enumerate(handle, 1): stripped = line.rstrip("\n") if not stripped: continue existing_digest, separator, existing_filename = stripped.partition(" ") if ( separator != " " or len(existing_digest) != 64 or any( character not in "0123456789abcdef" for character in existing_digest ) or not existing_filename ): raise ValueError( f"{path}:{line_number}: malformed SHA256SUMS entry" ) entries[existing_filename] = existing_digest entries[filename] = digest temporary = path.with_name(f".{path.name}.partial-{os.getpid()}") with temporary.open("w", encoding="ascii", newline="\n") as handle: for entry_filename in sorted(entries): handle.write(f"{entries[entry_filename]} {entry_filename}\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) def _build_input_hashes() -> dict[str, str]: relative_paths = [ Path("requirements-linux-aarch64-py314.lock"), Path("manifest/source.json"), *( Path("scripts") / name for name in ( "convert.py", "download_sources.py", "formats.py", "recipe.py", "reproduce.py", "verify.py", ) ), ] hashes: dict[str, str] = {} for relative_path in relative_paths: path = REPOSITORY_ROOT / relative_path digest = hashlib.sha256() with path.open("rb") as handle: while chunk := handle.read(1024 * 1024): digest.update(chunk) hashes[relative_path.as_posix()] = digest.hexdigest() return hashes def reproduce( source_dir: Path, output: Path, manifest_dir: Path, *, force: bool, repeat_check: bool, recipe: ArtifactRecipe | str | None = None, ) -> None: artifact_recipe = resolve_recipe(recipe) output = output.resolve() if output.exists() and not force: raise FileExistsError( f"output already exists: {output}; pass --force to replace it" ) output.parent.mkdir(parents=True, exist_ok=True) build_input_hashes = _build_input_hashes() repeat_digest: str | None = None with tempfile.TemporaryDirectory( prefix=f".{output.name}.candidate-", dir=output.parent ) as candidate_directory: candidate = Path(candidate_directory) / output.name size, digest = convert(source_dir, candidate, recipe=artifact_recipe) report = verify(source_dir, candidate, recipe=artifact_recipe) artifact = report["artifact"] if artifact["size"] != size or artifact["sha256"] != digest: raise ValueError("converter and verifier disagree on artifact identity") if repeat_check: with tempfile.TemporaryDirectory( prefix=f".{output.name}.repeat-", dir=output.parent ) as repeat_directory: repeat_output = Path(repeat_directory) / output.name repeat_size, repeat_digest = convert( source_dir, repeat_output, recipe=artifact_recipe ) if repeat_size != size or repeat_digest != digest: raise ValueError( "repeat build is not byte-identical: " f"first={size}/{digest}, second={repeat_size}/{repeat_digest}" ) os.replace(candidate, output) directory_fd = os.open(output.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory_fd) finally: os.close(directory_fd) manifest_dir = manifest_dir.resolve() write_report(manifest_dir / artifact_recipe.manifest_filename("validation"), report) build = { "artifact": artifact, "build_inputs": build_input_hashes, "environment": { "machine": platform.machine(), "numpy": np.__version__, "platform": platform.platform(), "python": platform.python_version(), }, "format": { "architecture": ARCHITECTURE, "gguf_version": 3, "recipe": artifact_recipe.name, "recipe_version": RECIPE_VERSION, }, "repeat_build": { "performed": repeat_check, "sha256": repeat_digest, "status": "byte-identical" if repeat_check else "not-run", }, "source": { "repository": SOURCE_REPOSITORY, "revision": SOURCE_REVISION, }, } _write_json(manifest_dir / artifact_recipe.manifest_filename("build"), build) _update_sha256sums(output.parent / "SHA256SUMS", digest, output.name) print(f"[reproduce] PASS: {output.name} {size:,} bytes {digest}", flush=True) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--sources", type=Path, required=True) parser.add_argument( "--recipe", choices=sorted(RECIPES), default=DEFAULT_RECIPE.name, help=f"Artifact recipe (default: {DEFAULT_RECIPE.name})", ) parser.add_argument( "--output", type=Path, help="Output path (default: the selected recipe's canonical filename)", ) parser.add_argument("--manifest-dir", type=Path, default=Path("manifest")) parser.add_argument("--force", action="store_true") parser.add_argument( "--repeat-check", action="store_true", help="Build a second clean GGUF and require a byte-identical SHA-256", ) return parser.parse_args() def main() -> None: args = parse_args() recipe = resolve_recipe(args.recipe) output = args.output if args.output is not None else Path(recipe.output_filename) try: reproduce( args.sources, output, args.manifest_dir, force=args.force, repeat_check=args.repeat_check, recipe=recipe, ) except Exception as error: print(f"[reproduce] ERROR: {error}", file=sys.stderr) raise SystemExit(1) from error if __name__ == "__main__": main()