Publish frozen reproduction executor image source
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +1 -0
- Dockerfile +28 -0
- LICENSE +161 -0
- README.md +108 -3
- SCIENCE-SPEC.yaml +1653 -0
- assets/fig2.png +3 -0
- configs/config.yaml +52 -0
- configs/experiment/maze_mpnn.yaml +40 -0
- configs/experiment/maze_sheaf.yaml +51 -0
- configs/experiment/mnist_mpnn.yaml +39 -0
- configs/experiment/mnist_sheaf.yaml +52 -0
- configs/experiment/sudoku_mpnn.yaml +36 -0
- configs/experiment/sudoku_sheaf.yaml +44 -0
- configs/experiment/sudoku_sheaf_lora.yaml +46 -0
- pyproject.toml +71 -0
- scripts/aggregate_attempts.py +42 -0
- scripts/build_preupload_privacy.py +87 -0
- scripts/build_registered_data.py +57 -0
- scripts/evaluate_repro.py +103 -0
- scripts/freeze_science_spec.py +122 -0
- scripts/import_data.py +275 -0
- scripts/launch_job.py +107 -0
- scripts/reduce_results.py +56 -0
- scripts/replace_scaffold_cells.py +37 -0
- scripts/scan_privacy.py +31 -0
- scripts/smoke_repro.py +128 -0
- scripts/train.py +261 -0
- scripts/train_repro.py +200 -0
- scripts/visualize.py +153 -0
- src/repro_control/__init__.py +5 -0
- src/repro_control/aggregation.py +69 -0
- src/repro_control/archives.py +75 -0
- src/repro_control/artifacts.py +85 -0
- src/repro_control/c5_runtime.py +52 -0
- src/repro_control/checkpoints.py +93 -0
- src/repro_control/cnn.py +109 -0
- src/repro_control/cnn_training.py +120 -0
- src/repro_control/configs.py +194 -0
- src/repro_control/constants.py +12 -0
- src/repro_control/data.py +271 -0
- src/repro_control/evaluation.py +398 -0
- src/repro_control/hashing.py +68 -0
- src/repro_control/heartbeat.py +30 -0
- src/repro_control/hf_provider.py +179 -0
- src/repro_control/interventions.py +93 -0
- src/repro_control/launcher.py +432 -0
- src/repro_control/manifests.py +152 -0
- src/repro_control/privacy.py +95 -0
- src/repro_control/reducers.py +179 -0
- src/repro_control/runtime.py +93 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
assets/fig2.png filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# syntax=docker/dockerfile:1.7
|
| 2 |
+
# Linux/amd64 descriptor resolved from the official Python registry on 2026-07-28.
|
| 3 |
+
ARG BASE_IMAGE=python:3.12-slim@sha256:cab2dbf575e971934a81e4622f5aba17aa7929719bd7e31033a3a83b97fd0464
|
| 4 |
+
FROM ${BASE_IMAGE}
|
| 5 |
+
|
| 6 |
+
ARG SCIENCE_SPEC_SHA256=c0ded12138cb7c15a15966294ed9bdb87d3a66d2373a7d90050578dbb973c6fd
|
| 7 |
+
LABEL org.opencontainers.image.title="neutral-sheaf-admm-reproduction"
|
| 8 |
+
LABEL org.opencontainers.image.licenses="Apache-2.0"
|
| 9 |
+
LABEL org.opencontainers.image.revision="1e2b5d648361802234348b0b1a7fb3a222128e7d"
|
| 10 |
+
LABEL reproduction.science-spec-sha256="${SCIENCE_SPEC_SHA256}"
|
| 11 |
+
|
| 12 |
+
ENV PYTHONDONTWRITEBYTECODE=1 \
|
| 13 |
+
PYTHONUNBUFFERED=1 \
|
| 14 |
+
JAX_DEFAULT_MATMUL_PRECISION=highest \
|
| 15 |
+
TRACE_MODE=none \
|
| 16 |
+
TRACKIO_LOGBOOK_AUTONOTE=0
|
| 17 |
+
|
| 18 |
+
WORKDIR /repro
|
| 19 |
+
COPY pyproject.toml uv.lock LICENSE SCIENCE-SPEC.yaml ./
|
| 20 |
+
RUN test "$(sha256sum SCIENCE-SPEC.yaml | cut -d' ' -f1)" = "${SCIENCE_SPEC_SHA256}" \
|
| 21 |
+
&& python -m pip install --no-cache-dir uv==0.11.9 \
|
| 22 |
+
&& uv sync --frozen --no-dev
|
| 23 |
+
COPY configs ./configs
|
| 24 |
+
COPY src ./src
|
| 25 |
+
COPY scripts ./scripts
|
| 26 |
+
ENV PATH="/repro/.venv/bin:${PATH}" PYTHONPATH="/repro/src"
|
| 27 |
+
EXPOSE 7860
|
| 28 |
+
CMD ["python", "-m", "http.server", "7860"]
|
LICENSE
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction, and
|
| 10 |
+
distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by the
|
| 13 |
+
copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all other
|
| 16 |
+
entities that control, are controlled by, or are under common control with
|
| 17 |
+
that entity. For the purposes of this definition, "control" means (i) the
|
| 18 |
+
power, direct or indirect, to cause the direction or management of such
|
| 19 |
+
entity, whether by contract or otherwise, or (ii) ownership of fifty percent
|
| 20 |
+
(50%) or more of the outstanding shares, or (iii) beneficial ownership of
|
| 21 |
+
such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity exercising
|
| 24 |
+
permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation source, and
|
| 28 |
+
configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical transformation or
|
| 31 |
+
translation of a Source form, including but not limited to compiled object
|
| 32 |
+
code, generated documentation, and conversions to other media types.
|
| 33 |
+
|
| 34 |
+
"Work" shall mean the work of authorship, whether in Source or Object form,
|
| 35 |
+
made available under the License, as indicated by a copyright notice that is
|
| 36 |
+
included in or attached to the work.
|
| 37 |
+
|
| 38 |
+
"Derivative Works" shall mean any work, whether in Source or Object form,
|
| 39 |
+
that is based on (or derived from) the Work and for which the editorial
|
| 40 |
+
revisions, annotations, elaborations, or other modifications represent, as a
|
| 41 |
+
whole, an original work of authorship. For the purposes of this License,
|
| 42 |
+
Derivative Works shall not include works that remain separable from, or merely
|
| 43 |
+
link (or bind by name) to the interfaces of, the Work and Derivative Works
|
| 44 |
+
thereof.
|
| 45 |
+
|
| 46 |
+
"Contribution" shall mean any work of authorship, including the original
|
| 47 |
+
version of the Work and any modifications or additions to that Work or
|
| 48 |
+
Derivative Works thereof, that is intentionally submitted to Licensor for
|
| 49 |
+
inclusion in the Work by the copyright owner or by an individual or Legal
|
| 50 |
+
Entity authorized to submit on behalf of the copyright owner. For the purposes
|
| 51 |
+
of this definition, "submitted" means any form of electronic, verbal, or
|
| 52 |
+
written communication sent to the Licensor or its representatives, including
|
| 53 |
+
but not limited to communication on electronic mailing lists, source code
|
| 54 |
+
control systems, and issue tracking systems that are managed by, or on behalf
|
| 55 |
+
of, the Licensor for the purpose of discussing and improving the Work, but
|
| 56 |
+
excluding communication that is conspicuously marked or otherwise designated
|
| 57 |
+
in writing by the copyright owner as "Not a Contribution."
|
| 58 |
+
|
| 59 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
|
| 60 |
+
of whom a Contribution has been received by Licensor and subsequently
|
| 61 |
+
incorporated within the Work.
|
| 62 |
+
|
| 63 |
+
2. Grant of Copyright License. Subject to the terms and conditions of this
|
| 64 |
+
License, each Contributor hereby grants to You a perpetual, worldwide,
|
| 65 |
+
non-exclusive, no-charge, royalty-free, irrevocable copyright license to
|
| 66 |
+
reproduce, prepare Derivative Works of, publicly display, publicly perform,
|
| 67 |
+
sublicense, and distribute the Work and such Derivative Works in Source or
|
| 68 |
+
Object form.
|
| 69 |
+
|
| 70 |
+
3. Grant of Patent License. Subject to the terms and conditions of this
|
| 71 |
+
License, each Contributor hereby grants to You a perpetual, worldwide,
|
| 72 |
+
non-exclusive, no-charge, royalty-free, irrevocable patent license to make,
|
| 73 |
+
have made, use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 74 |
+
where such license applies only to those patent claims licensable by such
|
| 75 |
+
Contributor that are necessarily infringed by their Contribution(s) alone or
|
| 76 |
+
by combination of their Contribution(s) with the Work to which such
|
| 77 |
+
Contribution(s) was submitted. If You institute patent litigation against any
|
| 78 |
+
entity (including a cross-claim or counterclaim in a lawsuit) alleging that the
|
| 79 |
+
Work or a Contribution incorporated within the Work constitutes direct or
|
| 80 |
+
contributory patent infringement, then any patent licenses granted to You
|
| 81 |
+
under this License for that Work shall terminate as of the date such
|
| 82 |
+
litigation is filed.
|
| 83 |
+
|
| 84 |
+
4. Redistribution. You may reproduce and distribute copies of the Work or
|
| 85 |
+
Derivative Works thereof in any medium, with or without modifications, and in
|
| 86 |
+
Source or Object form, provided that You meet the following conditions:
|
| 87 |
+
|
| 88 |
+
(a) You must give any other recipients of the Work or Derivative Works a copy
|
| 89 |
+
of this License; and
|
| 90 |
+
|
| 91 |
+
(b) You must cause any modified files to carry prominent notices stating that
|
| 92 |
+
You changed the files; and
|
| 93 |
+
|
| 94 |
+
(c) You must retain, in the Source form of any Derivative Works that You
|
| 95 |
+
distribute, all copyright, patent, trademark, and attribution notices from the
|
| 96 |
+
Source form of the Work, excluding those notices that do not pertain to any
|
| 97 |
+
part of the Derivative Works; and
|
| 98 |
+
|
| 99 |
+
(d) If the Work includes a "NOTICE" text file as part of its distribution, then
|
| 100 |
+
any Derivative Works that You distribute must include a readable copy of the
|
| 101 |
+
attribution notices contained within such NOTICE file, excluding those notices
|
| 102 |
+
that do not pertain to any part of the Derivative Works, in at least one of
|
| 103 |
+
the following places: within a NOTICE text file distributed as part of the
|
| 104 |
+
Derivative Works; within the Source form or documentation, if provided along
|
| 105 |
+
with the Derivative Works; or within a display generated by the Derivative
|
| 106 |
+
Works, if and wherever such third-party notices normally appear. The contents
|
| 107 |
+
of the NOTICE file are for informational purposes only and do not modify the
|
| 108 |
+
License. You may add Your own attribution notices within Derivative Works that
|
| 109 |
+
You distribute, alongside or as an addendum to the NOTICE text from the Work,
|
| 110 |
+
provided that such additional attribution notices cannot be construed as
|
| 111 |
+
modifying the License.
|
| 112 |
+
|
| 113 |
+
You may add Your own copyright statement to Your modifications and may provide
|
| 114 |
+
additional or different license terms and conditions for use, reproduction, or
|
| 115 |
+
distribution of Your modifications, or for any such Derivative Works as a
|
| 116 |
+
whole, provided Your use, reproduction, and distribution of the Work otherwise
|
| 117 |
+
complies with the conditions stated in this License.
|
| 118 |
+
|
| 119 |
+
5. Submission of Contributions. Unless You explicitly state otherwise, any
|
| 120 |
+
Contribution intentionally submitted for inclusion in the Work by You to the
|
| 121 |
+
Licensor shall be under the terms and conditions of this License, without any
|
| 122 |
+
additional terms or conditions. Notwithstanding the above, nothing herein shall
|
| 123 |
+
supersede or modify the terms of any separate license agreement you may have
|
| 124 |
+
executed with Licensor regarding such Contributions.
|
| 125 |
+
|
| 126 |
+
6. Trademarks. This License does not grant permission to use the trade names,
|
| 127 |
+
trademarks, service marks, or product names of the Licensor, except as required
|
| 128 |
+
for reasonable and customary use in describing the origin of the Work and
|
| 129 |
+
reproducing the content of the NOTICE file.
|
| 130 |
+
|
| 131 |
+
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
|
| 132 |
+
writing, Licensor provides the Work (and each Contributor provides its
|
| 133 |
+
Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
| 134 |
+
KIND, either express or implied, including, without limitation, any warranties
|
| 135 |
+
or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 136 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 137 |
+
appropriateness of using or redistributing the Work and assume any risks
|
| 138 |
+
associated with Your exercise of permissions under this License.
|
| 139 |
+
|
| 140 |
+
8. Limitation of Liability. In no event and under no legal theory, whether in
|
| 141 |
+
tort (including negligence), contract, or otherwise, unless required by
|
| 142 |
+
applicable law (such as deliberate and grossly negligent acts) or agreed to in
|
| 143 |
+
writing, shall any Contributor be liable to You for damages, including any
|
| 144 |
+
direct, indirect, special, incidental, or consequential damages of any character
|
| 145 |
+
arising as a result of this License or out of the use or inability to use the
|
| 146 |
+
Work (including but not limited to damages for loss of goodwill, work stoppage,
|
| 147 |
+
computer failure or malfunction, or any and all other commercial damages or
|
| 148 |
+
losses), even if such Contributor has been advised of the possibility of such
|
| 149 |
+
damages.
|
| 150 |
+
|
| 151 |
+
9. Accepting Warranty or Additional Liability. While redistributing the Work or
|
| 152 |
+
Derivative Works thereof, You may choose to offer, and charge a fee for,
|
| 153 |
+
acceptance of support, warranty, indemnity, or other liability obligations
|
| 154 |
+
and/or rights consistent with this License. However, in accepting such
|
| 155 |
+
obligations, You may act only on Your own behalf and on Your sole
|
| 156 |
+
responsibility, not on behalf of any other Contributor, and only if You agree
|
| 157 |
+
to indemnify, defend, and hold each Contributor harmless for any liability
|
| 158 |
+
incurred by, or claims asserted against, such Contributor by reason of your
|
| 159 |
+
accepting any such warranty or additional liability.
|
| 160 |
+
|
| 161 |
+
END OF TERMS AND CONDITIONS
|
README.md
CHANGED
|
@@ -1,10 +1,115 @@
|
|
| 1 |
---
|
| 2 |
-
title: Sheaf
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: blue
|
| 6 |
sdk: docker
|
|
|
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Sheaf-ADMM ICML 2026 Reproduction Executor
|
| 3 |
+
emoji: 🧩
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
pinned: false
|
| 9 |
+
license: apache-2.0
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# Sheaf-ADMM
|
| 13 |
+
|
| 14 |
+

|
| 15 |
+
|
| 16 |
+
Official JAX/Flax implementation of **Learning Multi-Agent Coordination via
|
| 17 |
+
Sheaf-ADMM** (ICML 2026).
|
| 18 |
+
|
| 19 |
+
[](https://arxiv.org/abs/2605.31005)
|
| 20 |
+
[](https://pub.sakana.ai/sheaf-admm/)
|
| 21 |
+
|
| 22 |
+
Sheaf-ADMM decomposes an input into overlapping local views, each processed by an
|
| 23 |
+
agent that solves a small convex subproblem parameterized by a neural encoder.
|
| 24 |
+
Agents coordinate through the Alternating Direction Method of Multipliers (ADMM),
|
| 25 |
+
with the inter-agent constraints specified by a *cellular sheaf* — which aspects
|
| 26 |
+
of neighboring solutions must agree. The optimization is unrolled for a fixed
|
| 27 |
+
number of iterations, so the whole pipeline is differentiable and every component
|
| 28 |
+
is trained end-to-end.
|
| 29 |
+
|
| 30 |
+
## Installation
|
| 31 |
+
|
| 32 |
+
This repository is intended to be run from a source checkout. Install
|
| 33 |
+
[uv](https://docs.astral.sh/uv/getting-started/installation/), then run:
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
uv sync
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
For CUDA experiments, install a JAX build matching your driver:
|
| 40 |
+
|
| 41 |
+
```bash
|
| 42 |
+
uv pip install -U "jax[cuda12]"
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
## Data
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
uv run python -m sheaf_admm.data.build_maze \
|
| 49 |
+
--height 19 \
|
| 50 |
+
--width 19 \
|
| 51 |
+
--train-size 10000 \
|
| 52 |
+
--test-size 1000 \
|
| 53 |
+
--min-path-length 18 \
|
| 54 |
+
--ood-sizes \
|
| 55 |
+
--output-dir datasets/maze_std3_19px_10k
|
| 56 |
+
|
| 57 |
+
uv run python -m sheaf_admm.data.build_mnist \
|
| 58 |
+
--output-dir datasets/mnist
|
| 59 |
+
|
| 60 |
+
uv run python -m sheaf_admm.data.build_sudoku \
|
| 61 |
+
--output-dir datasets/sudoku_easy
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
The MNIST and Sudoku builders download their source datasets on first run.
|
| 65 |
+
|
| 66 |
+
## Training
|
| 67 |
+
|
| 68 |
+
Each task has a Sheaf-ADMM model and a recurrent-MPNN baseline:
|
| 69 |
+
|
| 70 |
+
```bash
|
| 71 |
+
# Maze
|
| 72 |
+
uv run python scripts/train.py +experiment=maze_sheaf
|
| 73 |
+
uv run python scripts/train.py +experiment=maze_mpnn
|
| 74 |
+
|
| 75 |
+
# MNIST
|
| 76 |
+
uv run python scripts/train.py +experiment=mnist_sheaf
|
| 77 |
+
uv run python scripts/train.py +experiment=mnist_mpnn
|
| 78 |
+
|
| 79 |
+
# Sudoku
|
| 80 |
+
uv run python scripts/train.py +experiment=sudoku_sheaf
|
| 81 |
+
uv run python scripts/train.py +experiment=sudoku_sheaf_lora
|
| 82 |
+
uv run python scripts/train.py +experiment=sudoku_mpnn
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
Set `training.seed=42`, `123`, or `456` for the paper seeds. Set
|
| 86 |
+
`wandb.mode=online` to enable Weights & Biases logging. Checkpoints and
|
| 87 |
+
`history.json` are written to Hydra's run directory under `outputs/`.
|
| 88 |
+
|
| 89 |
+
## Visualization
|
| 90 |
+
|
| 91 |
+
The visualization script expects a Sheaf-ADMM checkpoint:
|
| 92 |
+
|
| 93 |
+
```bash
|
| 94 |
+
uv run python -m scripts.visualize \
|
| 95 |
+
--checkpoint outputs/<date>/<time>/checkpoint.pkl \
|
| 96 |
+
--out-dir /tmp/sheaf_admm_viz
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
## Checks
|
| 100 |
+
|
| 101 |
+
```bash
|
| 102 |
+
uv run python -m ruff check .
|
| 103 |
+
uv run python -m pytest -q
|
| 104 |
+
```
|
| 105 |
+
|
| 106 |
+
## Citation
|
| 107 |
+
|
| 108 |
+
```bibtex
|
| 109 |
+
@inproceedings{sheafadmm2026,
|
| 110 |
+
title = {Learning Multi-Agent Coordination via Sheaf-ADMM},
|
| 111 |
+
author = {Seely, Jeffrey and Cupia{\l}, Bart{\l}omiej and Jones, Llion},
|
| 112 |
+
booktitle = {International Conference on Machine Learning (ICML)},
|
| 113 |
+
year = {2026},
|
| 114 |
+
}
|
| 115 |
+
```
|
SCIENCE-SPEC.yaml
ADDED
|
@@ -0,0 +1,1653 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"authorities": {
|
| 3 |
+
"approved_protocol_sha256": "f4904e05e8abd051281926abec6774c6fe39b84160367f0bfde806b0669e0c2b",
|
| 4 |
+
"base_image": "python:3.12-slim@sha256:cab2dbf575e971934a81e4622f5aba17aa7929719bd7e31033a3a83b97fd0464",
|
| 5 |
+
"canonical_claims_sha256": "26ca2fa3697061cb71a91a0f687ab5ce99e908b7b50c009ee2efcc645695d489",
|
| 6 |
+
"challenge_dataset_revision": "81166abbeb76e5f79ff87e51061b5a0306507203",
|
| 7 |
+
"challenge_space_revision": "5bbcad2e9a7e8a7479f3563ac1fc6c768d4bb050",
|
| 8 |
+
"deadline_utc": "2026-08-03T11:59:00Z",
|
| 9 |
+
"dependency_lock_sha256": "e9dd209b20905a9a553c30ab1bab0259d2c665ee7805942744d9c5f198610bd7",
|
| 10 |
+
"execution_bucket": "Mindcraft/sheaf-admm-icml2026-runs",
|
| 11 |
+
"execution_image_space": "Mindcraft/sheaf-admm-icml2026-executor",
|
| 12 |
+
"input_bucket": "Mindcraft/sheaf-admm-icml2026-inputs",
|
| 13 |
+
"paper_sha256": "95d6de2011cdeaf4eeba6c7cc320146a530183da7f045457227cab3165b83a70",
|
| 14 |
+
"poster_commit": "e503c399b5427ca6cb712ccb080a758e9c19cf23",
|
| 15 |
+
"space_id": "Mindcraft/repro-learning-multi-agent-coordination-via-sheaf-admm",
|
| 16 |
+
"sudoku_dataset_revision": "4d5aa527a9fb9aacca0b0d5b8b77d569fa9afcaa",
|
| 17 |
+
"trace_mode": "none",
|
| 18 |
+
"trackio_logbook_autonote": 0,
|
| 19 |
+
"trackio_version": "0.33.0",
|
| 20 |
+
"trackio_wheel_sha256": "277340507ac46c02c06900c1d680129bdb528223c8110b0b6bc9326bb9f0891d",
|
| 21 |
+
"upstream_commit": "1e2b5d648361802234348b0b1a7fb3a222128e7d"
|
| 22 |
+
},
|
| 23 |
+
"budget_limits": {
|
| 24 |
+
"billing_quantum": "ceil(rate*1e6*ceil(seconds/60)/60)",
|
| 25 |
+
"gpu_retry_reserve_micro_usd": 10000000,
|
| 26 |
+
"minimum_unspent_balance_micro_usd": 10000000,
|
| 27 |
+
"normal_cap_micro_usd": 75000000
|
| 28 |
+
},
|
| 29 |
+
"data_rules": {
|
| 30 |
+
"c5": {
|
| 31 |
+
"K": 100,
|
| 32 |
+
"exact_figure6_protocol": "unreleased",
|
| 33 |
+
"examples_per_size": 1000,
|
| 34 |
+
"generator_seed_formula": "21005000+n",
|
| 35 |
+
"min_path_length_formula": "3*(n-1)/2",
|
| 36 |
+
"n19_replacements": true,
|
| 37 |
+
"overlap_rejection": "canonical wall/start/goal/path against training",
|
| 38 |
+
"realized_minimum": [
|
| 39 |
+
27,
|
| 40 |
+
33,
|
| 41 |
+
39,
|
| 42 |
+
45,
|
| 43 |
+
51,
|
| 44 |
+
57
|
| 45 |
+
],
|
| 46 |
+
"sizes": [
|
| 47 |
+
19,
|
| 48 |
+
23,
|
| 49 |
+
27,
|
| 50 |
+
31,
|
| 51 |
+
35,
|
| 52 |
+
39
|
| 53 |
+
],
|
| 54 |
+
"test_augmentation": false
|
| 55 |
+
},
|
| 56 |
+
"maze": {
|
| 57 |
+
"builder": "released_deterministic_DFS",
|
| 58 |
+
"height": 19,
|
| 59 |
+
"min_path_length": 18,
|
| 60 |
+
"ood_sizes": true,
|
| 61 |
+
"test_size": 1000,
|
| 62 |
+
"train_size": 10000,
|
| 63 |
+
"width": 19
|
| 64 |
+
},
|
| 65 |
+
"mnist": {
|
| 66 |
+
"agent_count": 81,
|
| 67 |
+
"clean": true,
|
| 68 |
+
"drop_agents": 24,
|
| 69 |
+
"drop_effect": [
|
| 70 |
+
"zero_3x3_pixels",
|
| 71 |
+
"remove_incident_sheaf_edges",
|
| 72 |
+
"exclude_removed_agents_from_vote"
|
| 73 |
+
],
|
| 74 |
+
"drop_fraction_label": "30%",
|
| 75 |
+
"mask_derivation": [
|
| 76 |
+
"dataset_revision",
|
| 77 |
+
"example_id",
|
| 78 |
+
"condition",
|
| 79 |
+
"master_seed"
|
| 80 |
+
],
|
| 81 |
+
"master_seed": 21005300,
|
| 82 |
+
"padding_pixels": 16,
|
| 83 |
+
"tier": "B_target_informed_nonconfirmatory"
|
| 84 |
+
},
|
| 85 |
+
"sudoku": {
|
| 86 |
+
"revision": "4d5aa527a9fb9aacca0b0d5b8b77d569fa9afcaa",
|
| 87 |
+
"source": "Ritvik19/Sudoku-Dataset",
|
| 88 |
+
"test_rows": [
|
| 89 |
+
50000,
|
| 90 |
+
52000
|
| 91 |
+
],
|
| 92 |
+
"test_split": "test_hard",
|
| 93 |
+
"train_augmentation": "eight_way",
|
| 94 |
+
"train_rows": [
|
| 95 |
+
0,
|
| 96 |
+
50000
|
| 97 |
+
]
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
"evaluators": {
|
| 101 |
+
"C5-EVAL-2X-GENERALIZATION-A": {
|
| 102 |
+
"physical_jobs": 1,
|
| 103 |
+
"shard": "deterministic_lpt_A"
|
| 104 |
+
},
|
| 105 |
+
"C5-EVAL-2X-GENERALIZATION-B": {
|
| 106 |
+
"physical_jobs": 1,
|
| 107 |
+
"shard": "deterministic_lpt_B"
|
| 108 |
+
},
|
| 109 |
+
"MAZE-EVAL-C2-C4B": {
|
| 110 |
+
"physical_jobs": 1,
|
| 111 |
+
"units": [
|
| 112 |
+
"imported_default_3",
|
| 113 |
+
"mpnn84_3",
|
| 114 |
+
"quadratic_3"
|
| 115 |
+
]
|
| 116 |
+
},
|
| 117 |
+
"MNIST-EVAL-C3": {
|
| 118 |
+
"conditions": [
|
| 119 |
+
"clean",
|
| 120 |
+
"pad16",
|
| 121 |
+
"drop30"
|
| 122 |
+
],
|
| 123 |
+
"physical_jobs": 1,
|
| 124 |
+
"units": [
|
| 125 |
+
"imported_sheaf_3",
|
| 126 |
+
"cnn_3"
|
| 127 |
+
]
|
| 128 |
+
},
|
| 129 |
+
"SUD-EVAL-C1-C4A": {
|
| 130 |
+
"physical_jobs": 1,
|
| 131 |
+
"units": [
|
| 132 |
+
"imported_sheaf_3",
|
| 133 |
+
"mpnn225_3",
|
| 134 |
+
"identity_3"
|
| 135 |
+
]
|
| 136 |
+
}
|
| 137 |
+
},
|
| 138 |
+
"failure_classes": [
|
| 139 |
+
"INVALID_INPUT",
|
| 140 |
+
"INCOMPLETE_OUTPUT",
|
| 141 |
+
"HASH_MISMATCH",
|
| 142 |
+
"CONFIG_MISMATCH",
|
| 143 |
+
"CAPABILITY_VIOLATION",
|
| 144 |
+
"READINESS_TIMEOUT",
|
| 145 |
+
"HEARTBEAT_TIMEOUT",
|
| 146 |
+
"INFRASTRUCTURE",
|
| 147 |
+
"CODE_PARITY",
|
| 148 |
+
"DATA_ASSERTION",
|
| 149 |
+
"BUDGET_GATE",
|
| 150 |
+
"DEADLINE_GATE",
|
| 151 |
+
"PRIVACY_FINDING",
|
| 152 |
+
"AMBIGUOUS_SUBMISSION"
|
| 153 |
+
],
|
| 154 |
+
"fixture_rules": {
|
| 155 |
+
"disjoint_from_final_generators": true,
|
| 156 |
+
"id_prefix": "smoke-",
|
| 157 |
+
"maze_seed_formula": "91005200+n",
|
| 158 |
+
"mnist_seed": 91005301,
|
| 159 |
+
"purpose": [
|
| 160 |
+
"compile",
|
| 161 |
+
"memory",
|
| 162 |
+
"timing",
|
| 163 |
+
"serialization",
|
| 164 |
+
"one_step_gradient"
|
| 165 |
+
],
|
| 166 |
+
"sudoku_seed": 91005101,
|
| 167 |
+
"verdict_metrics_forbidden": true
|
| 168 |
+
},
|
| 169 |
+
"format": 1,
|
| 170 |
+
"hardware_routes": {
|
| 171 |
+
"maze_c5": {
|
| 172 |
+
"fallback": {
|
| 173 |
+
"flavor": "l40sx1",
|
| 174 |
+
"usd_per_hour": "1.80"
|
| 175 |
+
},
|
| 176 |
+
"primary": {
|
| 177 |
+
"flavor": "l4x1",
|
| 178 |
+
"usd_per_hour": "0.80"
|
| 179 |
+
}
|
| 180 |
+
},
|
| 181 |
+
"mnist": {
|
| 182 |
+
"fallback": {
|
| 183 |
+
"flavor": "l40sx1",
|
| 184 |
+
"usd_per_hour": "1.80"
|
| 185 |
+
},
|
| 186 |
+
"primary": {
|
| 187 |
+
"flavor": "l4x1",
|
| 188 |
+
"usd_per_hour": "0.80"
|
| 189 |
+
}
|
| 190 |
+
},
|
| 191 |
+
"sudoku": {
|
| 192 |
+
"fallback": {
|
| 193 |
+
"flavor": "h200",
|
| 194 |
+
"usd_per_hour": "5.00"
|
| 195 |
+
},
|
| 196 |
+
"primary": {
|
| 197 |
+
"flavor": "a100-large",
|
| 198 |
+
"usd_per_hour": "2.50"
|
| 199 |
+
}
|
| 200 |
+
}
|
| 201 |
+
},
|
| 202 |
+
"imports": [
|
| 203 |
+
{
|
| 204 |
+
"checkpoint_sha256": "4444ca900b84911f778ade7fb4c682853308fac6008748ff644f5193122951b8",
|
| 205 |
+
"config_sha256": "831675683b8ca586616be95d7377a2ed1da776a04cc9457b5b4f855c5598e74a",
|
| 206 |
+
"ema_decay": 0.999,
|
| 207 |
+
"final_epoch": 19,
|
| 208 |
+
"neutral_alias": "mnist-sheaf-seed-42",
|
| 209 |
+
"seed": 42,
|
| 210 |
+
"task": "mnist"
|
| 211 |
+
},
|
| 212 |
+
{
|
| 213 |
+
"checkpoint_sha256": "34b693aac81e1ab880324726eb6db80e35e0683f3c997a2d082acd5c9393a5f6",
|
| 214 |
+
"config_sha256": "c1f902d7509c5a2dbd0130757b55607b51016e2a9039392da4b09b79c694c978",
|
| 215 |
+
"ema_decay": 0.999,
|
| 216 |
+
"final_epoch": 19,
|
| 217 |
+
"neutral_alias": "mnist-sheaf-seed-123",
|
| 218 |
+
"seed": 123,
|
| 219 |
+
"task": "mnist"
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"checkpoint_sha256": "f35206c24e79df75878e330ff01ff489ec29f82ae11eb3de04f401e2e4d2a3e5",
|
| 223 |
+
"config_sha256": "3a3c11e55551b9b820696abb7cf28ea94d4295e022c5da9525adfb802ee77d17",
|
| 224 |
+
"ema_decay": 0.999,
|
| 225 |
+
"final_epoch": 19,
|
| 226 |
+
"neutral_alias": "mnist-sheaf-seed-456",
|
| 227 |
+
"seed": 456,
|
| 228 |
+
"task": "mnist"
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"checkpoint_sha256": "7a8295f5af4f52c2ec5963fa35e7e3906ebce5ea433551bfc9017e4a9825de9c",
|
| 232 |
+
"config_sha256": "6e93a94469a1cec98dd2dc3ff9df91ce613ad99d18a5cae0388b8a42a610595a",
|
| 233 |
+
"ema_decay": 0.999,
|
| 234 |
+
"final_epoch": 49,
|
| 235 |
+
"neutral_alias": "maze-sheaf-seed-42",
|
| 236 |
+
"seed": 42,
|
| 237 |
+
"task": "maze"
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"checkpoint_sha256": "bd1369916039e3984911951217c7ae083efe6696f06f6cec9f2f5c009187a6f5",
|
| 241 |
+
"config_sha256": "8621e9687a39a5f28e5178561c4a6da9dbff09595dbc1340e1e97710c8af82d5",
|
| 242 |
+
"ema_decay": 0.999,
|
| 243 |
+
"final_epoch": 49,
|
| 244 |
+
"neutral_alias": "maze-sheaf-seed-123",
|
| 245 |
+
"seed": 123,
|
| 246 |
+
"task": "maze"
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"checkpoint_sha256": "a61191e0a3ec14aa245acc4b566dc7dd6caafba69365f9aa347dc2f906e95f38",
|
| 250 |
+
"config_sha256": "0566b34b360220bcc8e32cea6236884a9b7a693c68ca1cdc56861ab95ec7da13",
|
| 251 |
+
"ema_decay": 0.999,
|
| 252 |
+
"final_epoch": 49,
|
| 253 |
+
"neutral_alias": "maze-sheaf-seed-456",
|
| 254 |
+
"seed": 456,
|
| 255 |
+
"task": "maze"
|
| 256 |
+
},
|
| 257 |
+
{
|
| 258 |
+
"checkpoint_sha256": "53fb0f4c287bb419247956a3ab4492acb42cde6a3c3335ed1c18694c705d163e",
|
| 259 |
+
"config_sha256": "66ac9154bb1a15483211e0e8458cea4250181fdabc864c1748f89561535621fb",
|
| 260 |
+
"ema_decay": 0.999,
|
| 261 |
+
"final_epoch": 9,
|
| 262 |
+
"neutral_alias": "sudoku-sheaf-seed-42",
|
| 263 |
+
"seed": 42,
|
| 264 |
+
"task": "sudoku"
|
| 265 |
+
},
|
| 266 |
+
{
|
| 267 |
+
"checkpoint_sha256": "a16e792b46aeca8ca6dc9d9323217328ecf43e0ce32022870c5626186997511f",
|
| 268 |
+
"config_sha256": "f080a4055a676a5094204fc78de5e2f57c07dd2f67af8c55cf7615dcb818d47d",
|
| 269 |
+
"ema_decay": 0.999,
|
| 270 |
+
"final_epoch": 9,
|
| 271 |
+
"neutral_alias": "sudoku-sheaf-seed-123",
|
| 272 |
+
"seed": 123,
|
| 273 |
+
"task": "sudoku"
|
| 274 |
+
},
|
| 275 |
+
{
|
| 276 |
+
"checkpoint_sha256": "7a6937eedf75f553965c21b5dda5c4da8164489125159f6fc8465e9159b2b3af",
|
| 277 |
+
"config_sha256": "1649e4d4e047bcceefb9b13ebee6fa830eeac668445714457a2ed258563a9625",
|
| 278 |
+
"ema_decay": 0.999,
|
| 279 |
+
"final_epoch": 9,
|
| 280 |
+
"neutral_alias": "sudoku-sheaf-seed-456",
|
| 281 |
+
"seed": 456,
|
| 282 |
+
"task": "sudoku"
|
| 283 |
+
}
|
| 284 |
+
],
|
| 285 |
+
"lifecycle": {
|
| 286 |
+
"CPU_CANARY": {
|
| 287 |
+
"science_freeze_sha256": "NOT_APPLICABLE",
|
| 288 |
+
"science_spec_sha256": "NOT_APPLICABLE"
|
| 289 |
+
},
|
| 290 |
+
"CPU_IMPORT": {
|
| 291 |
+
"science_freeze_sha256": "NOT_APPLICABLE",
|
| 292 |
+
"science_spec_sha256": "EXACT"
|
| 293 |
+
},
|
| 294 |
+
"GPU_SMOKE": {
|
| 295 |
+
"science_freeze_sha256": "NOT_APPLICABLE",
|
| 296 |
+
"science_spec_sha256": "EXACT"
|
| 297 |
+
},
|
| 298 |
+
"SCIENTIFIC_EVAL": {
|
| 299 |
+
"control_mount": "/repro-control read-only",
|
| 300 |
+
"science_freeze_sha256": "EXACT",
|
| 301 |
+
"science_spec_sha256": "EXACT"
|
| 302 |
+
},
|
| 303 |
+
"SCIENTIFIC_TRAIN": {
|
| 304 |
+
"control_mount": "/repro-control read-only",
|
| 305 |
+
"science_freeze_sha256": "EXACT",
|
| 306 |
+
"science_spec_sha256": "EXACT"
|
| 307 |
+
}
|
| 308 |
+
},
|
| 309 |
+
"metrics": {
|
| 310 |
+
"c1": "Sudoku exact puzzle accuracy and parameter count",
|
| 311 |
+
"c2": "Maze exact puzzle accuracy and per-vertex latent dimension ratio",
|
| 312 |
+
"c3": "MNIST classification accuracy by condition",
|
| 313 |
+
"c4": "Sudoku exact puzzle accuracy and Maze exact puzzle accuracy",
|
| 314 |
+
"c5": "Maze exact puzzle accuracy by size",
|
| 315 |
+
"replication_unit": "training_seed",
|
| 316 |
+
"report": [
|
| 317 |
+
"every_seed",
|
| 318 |
+
"mean",
|
| 319 |
+
"sample_standard_deviation",
|
| 320 |
+
"student_t_interval",
|
| 321 |
+
"paper_value",
|
| 322 |
+
"unrounded_delta"
|
| 323 |
+
],
|
| 324 |
+
"same_seed_comparisons": "paired",
|
| 325 |
+
"t_critical": {
|
| 326 |
+
"90_df2": 2.919985580355516,
|
| 327 |
+
"95_df2": 4.302652729911275
|
| 328 |
+
}
|
| 329 |
+
},
|
| 330 |
+
"outcomes": {},
|
| 331 |
+
"protocol_status": "frozen_pre_mutation",
|
| 332 |
+
"reducers": {
|
| 333 |
+
"c1": {
|
| 334 |
+
"negative": "Sheaf_mean<85 or MPNN_mean>20 or U95(paired_gap)<=60",
|
| 335 |
+
"parameter_match": "relative_count_mismatch<=0.05",
|
| 336 |
+
"support": "Sheaf_mean>=85 and MPNN_mean<=20 and L95(paired_gap)>60"
|
| 337 |
+
},
|
| 338 |
+
"c2": {
|
| 339 |
+
"negative": "either mean<90 or interval wholly beyond equivalence bounds",
|
| 340 |
+
"ratio": "84/10=8.4x per-vertex latent dimension",
|
| 341 |
+
"support": "both means>=95 and paired_90_interval within [-2,2]"
|
| 342 |
+
},
|
| 343 |
+
"c3": {
|
| 344 |
+
"adequacy": "each mean>=98.5 and each seed>=98.0",
|
| 345 |
+
"negative": "either upper interval<=threshold",
|
| 346 |
+
"required_suffix": "DROPOUT_SEMANTICS_TARGET_SELECTED",
|
| 347 |
+
"support": "L95(pad_gap)>20 and L95(drop_gap)>10"
|
| 348 |
+
},
|
| 349 |
+
"c4_maze": {
|
| 350 |
+
"falsifies_collapse": "default_mean>=90 and quadratic_mean>=90 and U95(default-quadratic)<20",
|
| 351 |
+
"required_suffix": "PROMPT_MISSTATES_PAPER_TABLE",
|
| 352 |
+
"supports_direction": "default_mean>=90 and quadratic_mean<=60 and L95(default-quadratic)>20"
|
| 353 |
+
},
|
| 354 |
+
"c4_sudoku": {
|
| 355 |
+
"negative": "learned_mean<85 or identity_mean>15 or U95(gap)<=60",
|
| 356 |
+
"support": "learned_mean>=85 and identity_mean<=15 and L95(gap)>60"
|
| 357 |
+
},
|
| 358 |
+
"c5": {
|
| 359 |
+
"negative": "any mean<95",
|
| 360 |
+
"required_suffix": "INCONCLUSIVE_EXACT_DENSE_FIGURE6_CONFIG_UNRELEASED",
|
| 361 |
+
"support": "all six three-seed means>=95"
|
| 362 |
+
},
|
| 363 |
+
"precedence": [
|
| 364 |
+
"invalid_or_incomplete",
|
| 365 |
+
"supported_or_negative",
|
| 366 |
+
"ambiguous_or_inconclusive"
|
| 367 |
+
]
|
| 368 |
+
},
|
| 369 |
+
"registered_configs": {
|
| 370 |
+
"C1-SUD-MPNN225-123": {
|
| 371 |
+
"canonical_sha256": "703fe3f833c3b2910a92783710faee4893ddb333d008b4784288171e78fa56f5",
|
| 372 |
+
"config": {
|
| 373 |
+
"data": {
|
| 374 |
+
"dir": "/data/train/sudoku_easy",
|
| 375 |
+
"loader": "puzzle",
|
| 376 |
+
"train_split": "train",
|
| 377 |
+
"val_splits": []
|
| 378 |
+
},
|
| 379 |
+
"dtype": "float32",
|
| 380 |
+
"model": {
|
| 381 |
+
"comm_norm_type": "layernorm",
|
| 382 |
+
"d_e": 32,
|
| 383 |
+
"d_v": 225,
|
| 384 |
+
"dec_hidden_dims": [
|
| 385 |
+
256
|
| 386 |
+
],
|
| 387 |
+
"decoder_arch": "sudoku",
|
| 388 |
+
"enc_d_model": 128,
|
| 389 |
+
"enc_num_blocks": 2,
|
| 390 |
+
"encoder_arch": "sudoku",
|
| 391 |
+
"mpnn_aggregation": "max",
|
| 392 |
+
"mpnn_edge_type_mode": "slot",
|
| 393 |
+
"mpnn_graph_readout": "per_node",
|
| 394 |
+
"mpnn_message_dim": 32,
|
| 395 |
+
"num_classes": 10,
|
| 396 |
+
"num_directions": 9
|
| 397 |
+
},
|
| 398 |
+
"model_type": "mpnn",
|
| 399 |
+
"task": "sudoku",
|
| 400 |
+
"task_cfg": {},
|
| 401 |
+
"training": {
|
| 402 |
+
"K_eval": 100,
|
| 403 |
+
"K_train": 40,
|
| 404 |
+
"batch_size": 128,
|
| 405 |
+
"ema_decay": 0.999,
|
| 406 |
+
"epochs": 10,
|
| 407 |
+
"exit_on_nan": true,
|
| 408 |
+
"grad_clip": 1.0,
|
| 409 |
+
"loss_window": 4,
|
| 410 |
+
"lr": 0.0017,
|
| 411 |
+
"mpnn_eval_rounds": 50,
|
| 412 |
+
"mpnn_train_rounds": 20,
|
| 413 |
+
"seed": 123,
|
| 414 |
+
"train_iters_dist": "fixed",
|
| 415 |
+
"train_iters_min": 15,
|
| 416 |
+
"val_interval": 1,
|
| 417 |
+
"warmup_steps": 200,
|
| 418 |
+
"weight_decay": 1e-07
|
| 419 |
+
},
|
| 420 |
+
"wandb": {
|
| 421 |
+
"entity": null,
|
| 422 |
+
"group": null,
|
| 423 |
+
"mode": "disabled",
|
| 424 |
+
"name": null,
|
| 425 |
+
"project": "sheaf-admm",
|
| 426 |
+
"tags": []
|
| 427 |
+
}
|
| 428 |
+
},
|
| 429 |
+
"file": "control/registered-configs/C1-SUD-MPNN225-123.json"
|
| 430 |
+
},
|
| 431 |
+
"C1-SUD-MPNN225-42": {
|
| 432 |
+
"canonical_sha256": "a0a0fad0c015647c5b03f31542c7f01fd2648ad04c19360b82d6dc3e060e3af2",
|
| 433 |
+
"config": {
|
| 434 |
+
"data": {
|
| 435 |
+
"dir": "/data/train/sudoku_easy",
|
| 436 |
+
"loader": "puzzle",
|
| 437 |
+
"train_split": "train",
|
| 438 |
+
"val_splits": []
|
| 439 |
+
},
|
| 440 |
+
"dtype": "float32",
|
| 441 |
+
"model": {
|
| 442 |
+
"comm_norm_type": "layernorm",
|
| 443 |
+
"d_e": 32,
|
| 444 |
+
"d_v": 225,
|
| 445 |
+
"dec_hidden_dims": [
|
| 446 |
+
256
|
| 447 |
+
],
|
| 448 |
+
"decoder_arch": "sudoku",
|
| 449 |
+
"enc_d_model": 128,
|
| 450 |
+
"enc_num_blocks": 2,
|
| 451 |
+
"encoder_arch": "sudoku",
|
| 452 |
+
"mpnn_aggregation": "max",
|
| 453 |
+
"mpnn_edge_type_mode": "slot",
|
| 454 |
+
"mpnn_graph_readout": "per_node",
|
| 455 |
+
"mpnn_message_dim": 32,
|
| 456 |
+
"num_classes": 10,
|
| 457 |
+
"num_directions": 9
|
| 458 |
+
},
|
| 459 |
+
"model_type": "mpnn",
|
| 460 |
+
"task": "sudoku",
|
| 461 |
+
"task_cfg": {},
|
| 462 |
+
"training": {
|
| 463 |
+
"K_eval": 100,
|
| 464 |
+
"K_train": 40,
|
| 465 |
+
"batch_size": 128,
|
| 466 |
+
"ema_decay": 0.999,
|
| 467 |
+
"epochs": 10,
|
| 468 |
+
"exit_on_nan": true,
|
| 469 |
+
"grad_clip": 1.0,
|
| 470 |
+
"loss_window": 4,
|
| 471 |
+
"lr": 0.0017,
|
| 472 |
+
"mpnn_eval_rounds": 50,
|
| 473 |
+
"mpnn_train_rounds": 20,
|
| 474 |
+
"seed": 42,
|
| 475 |
+
"train_iters_dist": "fixed",
|
| 476 |
+
"train_iters_min": 15,
|
| 477 |
+
"val_interval": 1,
|
| 478 |
+
"warmup_steps": 200,
|
| 479 |
+
"weight_decay": 1e-07
|
| 480 |
+
},
|
| 481 |
+
"wandb": {
|
| 482 |
+
"entity": null,
|
| 483 |
+
"group": null,
|
| 484 |
+
"mode": "disabled",
|
| 485 |
+
"name": null,
|
| 486 |
+
"project": "sheaf-admm",
|
| 487 |
+
"tags": []
|
| 488 |
+
}
|
| 489 |
+
},
|
| 490 |
+
"file": "control/registered-configs/C1-SUD-MPNN225-42.json"
|
| 491 |
+
},
|
| 492 |
+
"C1-SUD-MPNN225-456": {
|
| 493 |
+
"canonical_sha256": "b1e99a06aafa6fa0255a8c4c4141b5e7da37a72f21e0388fe10ab45528454a44",
|
| 494 |
+
"config": {
|
| 495 |
+
"data": {
|
| 496 |
+
"dir": "/data/train/sudoku_easy",
|
| 497 |
+
"loader": "puzzle",
|
| 498 |
+
"train_split": "train",
|
| 499 |
+
"val_splits": []
|
| 500 |
+
},
|
| 501 |
+
"dtype": "float32",
|
| 502 |
+
"model": {
|
| 503 |
+
"comm_norm_type": "layernorm",
|
| 504 |
+
"d_e": 32,
|
| 505 |
+
"d_v": 225,
|
| 506 |
+
"dec_hidden_dims": [
|
| 507 |
+
256
|
| 508 |
+
],
|
| 509 |
+
"decoder_arch": "sudoku",
|
| 510 |
+
"enc_d_model": 128,
|
| 511 |
+
"enc_num_blocks": 2,
|
| 512 |
+
"encoder_arch": "sudoku",
|
| 513 |
+
"mpnn_aggregation": "max",
|
| 514 |
+
"mpnn_edge_type_mode": "slot",
|
| 515 |
+
"mpnn_graph_readout": "per_node",
|
| 516 |
+
"mpnn_message_dim": 32,
|
| 517 |
+
"num_classes": 10,
|
| 518 |
+
"num_directions": 9
|
| 519 |
+
},
|
| 520 |
+
"model_type": "mpnn",
|
| 521 |
+
"task": "sudoku",
|
| 522 |
+
"task_cfg": {},
|
| 523 |
+
"training": {
|
| 524 |
+
"K_eval": 100,
|
| 525 |
+
"K_train": 40,
|
| 526 |
+
"batch_size": 128,
|
| 527 |
+
"ema_decay": 0.999,
|
| 528 |
+
"epochs": 10,
|
| 529 |
+
"exit_on_nan": true,
|
| 530 |
+
"grad_clip": 1.0,
|
| 531 |
+
"loss_window": 4,
|
| 532 |
+
"lr": 0.0017,
|
| 533 |
+
"mpnn_eval_rounds": 50,
|
| 534 |
+
"mpnn_train_rounds": 20,
|
| 535 |
+
"seed": 456,
|
| 536 |
+
"train_iters_dist": "fixed",
|
| 537 |
+
"train_iters_min": 15,
|
| 538 |
+
"val_interval": 1,
|
| 539 |
+
"warmup_steps": 200,
|
| 540 |
+
"weight_decay": 1e-07
|
| 541 |
+
},
|
| 542 |
+
"wandb": {
|
| 543 |
+
"entity": null,
|
| 544 |
+
"group": null,
|
| 545 |
+
"mode": "disabled",
|
| 546 |
+
"name": null,
|
| 547 |
+
"project": "sheaf-admm",
|
| 548 |
+
"tags": []
|
| 549 |
+
}
|
| 550 |
+
},
|
| 551 |
+
"file": "control/registered-configs/C1-SUD-MPNN225-456.json"
|
| 552 |
+
},
|
| 553 |
+
"C2-MAZE-MPNN84-123": {
|
| 554 |
+
"canonical_sha256": "545effae66a4ab9a6c802d32eaa28b06bf664a3618cac556aadd42a39285b40b",
|
| 555 |
+
"config": {
|
| 556 |
+
"data": {
|
| 557 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 558 |
+
"loader": "puzzle",
|
| 559 |
+
"train_split": "train",
|
| 560 |
+
"val_splits": []
|
| 561 |
+
},
|
| 562 |
+
"dtype": "float32",
|
| 563 |
+
"model": {
|
| 564 |
+
"comm_norm_type": "layernorm",
|
| 565 |
+
"d_e": 42,
|
| 566 |
+
"d_v": 84,
|
| 567 |
+
"dec_hidden_dim": 256,
|
| 568 |
+
"decoder_arch": "mlp_concat_v2",
|
| 569 |
+
"enc_hidden_dim": 256,
|
| 570 |
+
"encoder_arch": "mlp_v2",
|
| 571 |
+
"mpnn_aggregation": "max",
|
| 572 |
+
"mpnn_edge_type_mode": "spatial",
|
| 573 |
+
"mpnn_graph_readout": "per_node",
|
| 574 |
+
"mpnn_message_dim": 42,
|
| 575 |
+
"num_classes": 6,
|
| 576 |
+
"num_directions": 8
|
| 577 |
+
},
|
| 578 |
+
"model_type": "mpnn",
|
| 579 |
+
"task": "maze",
|
| 580 |
+
"task_cfg": {
|
| 581 |
+
"connectivity": 8,
|
| 582 |
+
"num_classes": 6,
|
| 583 |
+
"patch_size": 3,
|
| 584 |
+
"stride": 2
|
| 585 |
+
},
|
| 586 |
+
"training": {
|
| 587 |
+
"K_eval": 100,
|
| 588 |
+
"K_train": 40,
|
| 589 |
+
"batch_size": 128,
|
| 590 |
+
"ema_decay": 0.999,
|
| 591 |
+
"epochs": 50,
|
| 592 |
+
"exit_on_nan": true,
|
| 593 |
+
"grad_clip": 1.0,
|
| 594 |
+
"loss_window": 4,
|
| 595 |
+
"lr": 0.0003,
|
| 596 |
+
"mpnn_eval_rounds": 100,
|
| 597 |
+
"mpnn_train_rounds": 40,
|
| 598 |
+
"seed": 123,
|
| 599 |
+
"train_iters_dist": "fixed",
|
| 600 |
+
"train_iters_min": 15,
|
| 601 |
+
"val_interval": 5,
|
| 602 |
+
"warmup_steps": 200,
|
| 603 |
+
"weight_decay": 1e-06
|
| 604 |
+
},
|
| 605 |
+
"wandb": {
|
| 606 |
+
"entity": null,
|
| 607 |
+
"group": null,
|
| 608 |
+
"mode": "disabled",
|
| 609 |
+
"name": null,
|
| 610 |
+
"project": "sheaf-admm",
|
| 611 |
+
"tags": []
|
| 612 |
+
}
|
| 613 |
+
},
|
| 614 |
+
"file": "control/registered-configs/C2-MAZE-MPNN84-123.json"
|
| 615 |
+
},
|
| 616 |
+
"C2-MAZE-MPNN84-42": {
|
| 617 |
+
"canonical_sha256": "f826d5b2c392645ceabd97ed219310e4c8dd8278dce3c935b893ef6ec67c598d",
|
| 618 |
+
"config": {
|
| 619 |
+
"data": {
|
| 620 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 621 |
+
"loader": "puzzle",
|
| 622 |
+
"train_split": "train",
|
| 623 |
+
"val_splits": []
|
| 624 |
+
},
|
| 625 |
+
"dtype": "float32",
|
| 626 |
+
"model": {
|
| 627 |
+
"comm_norm_type": "layernorm",
|
| 628 |
+
"d_e": 42,
|
| 629 |
+
"d_v": 84,
|
| 630 |
+
"dec_hidden_dim": 256,
|
| 631 |
+
"decoder_arch": "mlp_concat_v2",
|
| 632 |
+
"enc_hidden_dim": 256,
|
| 633 |
+
"encoder_arch": "mlp_v2",
|
| 634 |
+
"mpnn_aggregation": "max",
|
| 635 |
+
"mpnn_edge_type_mode": "spatial",
|
| 636 |
+
"mpnn_graph_readout": "per_node",
|
| 637 |
+
"mpnn_message_dim": 42,
|
| 638 |
+
"num_classes": 6,
|
| 639 |
+
"num_directions": 8
|
| 640 |
+
},
|
| 641 |
+
"model_type": "mpnn",
|
| 642 |
+
"task": "maze",
|
| 643 |
+
"task_cfg": {
|
| 644 |
+
"connectivity": 8,
|
| 645 |
+
"num_classes": 6,
|
| 646 |
+
"patch_size": 3,
|
| 647 |
+
"stride": 2
|
| 648 |
+
},
|
| 649 |
+
"training": {
|
| 650 |
+
"K_eval": 100,
|
| 651 |
+
"K_train": 40,
|
| 652 |
+
"batch_size": 128,
|
| 653 |
+
"ema_decay": 0.999,
|
| 654 |
+
"epochs": 50,
|
| 655 |
+
"exit_on_nan": true,
|
| 656 |
+
"grad_clip": 1.0,
|
| 657 |
+
"loss_window": 4,
|
| 658 |
+
"lr": 0.0003,
|
| 659 |
+
"mpnn_eval_rounds": 100,
|
| 660 |
+
"mpnn_train_rounds": 40,
|
| 661 |
+
"seed": 42,
|
| 662 |
+
"train_iters_dist": "fixed",
|
| 663 |
+
"train_iters_min": 15,
|
| 664 |
+
"val_interval": 5,
|
| 665 |
+
"warmup_steps": 200,
|
| 666 |
+
"weight_decay": 1e-06
|
| 667 |
+
},
|
| 668 |
+
"wandb": {
|
| 669 |
+
"entity": null,
|
| 670 |
+
"group": null,
|
| 671 |
+
"mode": "disabled",
|
| 672 |
+
"name": null,
|
| 673 |
+
"project": "sheaf-admm",
|
| 674 |
+
"tags": []
|
| 675 |
+
}
|
| 676 |
+
},
|
| 677 |
+
"file": "control/registered-configs/C2-MAZE-MPNN84-42.json"
|
| 678 |
+
},
|
| 679 |
+
"C2-MAZE-MPNN84-456": {
|
| 680 |
+
"canonical_sha256": "2fb6db8a94a03033c21b698aeb700ebf2013abf803e4d34e23a778e3a5ab8d09",
|
| 681 |
+
"config": {
|
| 682 |
+
"data": {
|
| 683 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 684 |
+
"loader": "puzzle",
|
| 685 |
+
"train_split": "train",
|
| 686 |
+
"val_splits": []
|
| 687 |
+
},
|
| 688 |
+
"dtype": "float32",
|
| 689 |
+
"model": {
|
| 690 |
+
"comm_norm_type": "layernorm",
|
| 691 |
+
"d_e": 42,
|
| 692 |
+
"d_v": 84,
|
| 693 |
+
"dec_hidden_dim": 256,
|
| 694 |
+
"decoder_arch": "mlp_concat_v2",
|
| 695 |
+
"enc_hidden_dim": 256,
|
| 696 |
+
"encoder_arch": "mlp_v2",
|
| 697 |
+
"mpnn_aggregation": "max",
|
| 698 |
+
"mpnn_edge_type_mode": "spatial",
|
| 699 |
+
"mpnn_graph_readout": "per_node",
|
| 700 |
+
"mpnn_message_dim": 42,
|
| 701 |
+
"num_classes": 6,
|
| 702 |
+
"num_directions": 8
|
| 703 |
+
},
|
| 704 |
+
"model_type": "mpnn",
|
| 705 |
+
"task": "maze",
|
| 706 |
+
"task_cfg": {
|
| 707 |
+
"connectivity": 8,
|
| 708 |
+
"num_classes": 6,
|
| 709 |
+
"patch_size": 3,
|
| 710 |
+
"stride": 2
|
| 711 |
+
},
|
| 712 |
+
"training": {
|
| 713 |
+
"K_eval": 100,
|
| 714 |
+
"K_train": 40,
|
| 715 |
+
"batch_size": 128,
|
| 716 |
+
"ema_decay": 0.999,
|
| 717 |
+
"epochs": 50,
|
| 718 |
+
"exit_on_nan": true,
|
| 719 |
+
"grad_clip": 1.0,
|
| 720 |
+
"loss_window": 4,
|
| 721 |
+
"lr": 0.0003,
|
| 722 |
+
"mpnn_eval_rounds": 100,
|
| 723 |
+
"mpnn_train_rounds": 40,
|
| 724 |
+
"seed": 456,
|
| 725 |
+
"train_iters_dist": "fixed",
|
| 726 |
+
"train_iters_min": 15,
|
| 727 |
+
"val_interval": 5,
|
| 728 |
+
"warmup_steps": 200,
|
| 729 |
+
"weight_decay": 1e-06
|
| 730 |
+
},
|
| 731 |
+
"wandb": {
|
| 732 |
+
"entity": null,
|
| 733 |
+
"group": null,
|
| 734 |
+
"mode": "disabled",
|
| 735 |
+
"name": null,
|
| 736 |
+
"project": "sheaf-admm",
|
| 737 |
+
"tags": []
|
| 738 |
+
}
|
| 739 |
+
},
|
| 740 |
+
"file": "control/registered-configs/C2-MAZE-MPNN84-456.json"
|
| 741 |
+
},
|
| 742 |
+
"C3-MNIST-CNN-123": {
|
| 743 |
+
"canonical_sha256": "528044762e54a8f02036c84c34836a857518c98b5aef2fec60d5352b04d89ef8",
|
| 744 |
+
"config": {
|
| 745 |
+
"data": {
|
| 746 |
+
"augmentation": false,
|
| 747 |
+
"dir": "/data/train/mnist",
|
| 748 |
+
"examples": 60000,
|
| 749 |
+
"input_range": [
|
| 750 |
+
0,
|
| 751 |
+
1
|
| 752 |
+
],
|
| 753 |
+
"loader": "image",
|
| 754 |
+
"normalization": false,
|
| 755 |
+
"train_split": "train",
|
| 756 |
+
"val_splits": []
|
| 757 |
+
},
|
| 758 |
+
"dtype": "float32",
|
| 759 |
+
"model": {
|
| 760 |
+
"layout": "NHWC/HWIO/NHWC",
|
| 761 |
+
"parameter_count": 65642,
|
| 762 |
+
"post_pool_sizes": [
|
| 763 |
+
14,
|
| 764 |
+
30
|
| 765 |
+
],
|
| 766 |
+
"pre_pool_sizes": [
|
| 767 |
+
28,
|
| 768 |
+
60
|
| 769 |
+
]
|
| 770 |
+
},
|
| 771 |
+
"model_type": "mnist_cnn_repro",
|
| 772 |
+
"task": "mnist",
|
| 773 |
+
"task_cfg": {},
|
| 774 |
+
"training": {
|
| 775 |
+
"batch_size": 128,
|
| 776 |
+
"betas": [
|
| 777 |
+
0.9,
|
| 778 |
+
0.999
|
| 779 |
+
],
|
| 780 |
+
"ema_decay": 0.999,
|
| 781 |
+
"epoch_permutation": "fold_in(PRNGKey(seed),epoch)",
|
| 782 |
+
"epochs": 20,
|
| 783 |
+
"epsilon": 1e-08,
|
| 784 |
+
"final_batch": "pad_to_128_mask_normalize",
|
| 785 |
+
"grad_clip": 1.0,
|
| 786 |
+
"lr": 0.001,
|
| 787 |
+
"schedule": "linear_to_task_lr_then_constant",
|
| 788 |
+
"seed": 123,
|
| 789 |
+
"selection": "fixed_final_epoch",
|
| 790 |
+
"warmup_steps": 200,
|
| 791 |
+
"weight_decay": 1e-07
|
| 792 |
+
},
|
| 793 |
+
"wandb": {
|
| 794 |
+
"mode": "disabled"
|
| 795 |
+
}
|
| 796 |
+
},
|
| 797 |
+
"file": "control/registered-configs/C3-MNIST-CNN-123.json"
|
| 798 |
+
},
|
| 799 |
+
"C3-MNIST-CNN-42": {
|
| 800 |
+
"canonical_sha256": "442cbeddd56b3eba9dc239e84403285d135bca4868d58a315fe67fcd97b1f4f6",
|
| 801 |
+
"config": {
|
| 802 |
+
"data": {
|
| 803 |
+
"augmentation": false,
|
| 804 |
+
"dir": "/data/train/mnist",
|
| 805 |
+
"examples": 60000,
|
| 806 |
+
"input_range": [
|
| 807 |
+
0,
|
| 808 |
+
1
|
| 809 |
+
],
|
| 810 |
+
"loader": "image",
|
| 811 |
+
"normalization": false,
|
| 812 |
+
"train_split": "train",
|
| 813 |
+
"val_splits": []
|
| 814 |
+
},
|
| 815 |
+
"dtype": "float32",
|
| 816 |
+
"model": {
|
| 817 |
+
"layout": "NHWC/HWIO/NHWC",
|
| 818 |
+
"parameter_count": 65642,
|
| 819 |
+
"post_pool_sizes": [
|
| 820 |
+
14,
|
| 821 |
+
30
|
| 822 |
+
],
|
| 823 |
+
"pre_pool_sizes": [
|
| 824 |
+
28,
|
| 825 |
+
60
|
| 826 |
+
]
|
| 827 |
+
},
|
| 828 |
+
"model_type": "mnist_cnn_repro",
|
| 829 |
+
"task": "mnist",
|
| 830 |
+
"task_cfg": {},
|
| 831 |
+
"training": {
|
| 832 |
+
"batch_size": 128,
|
| 833 |
+
"betas": [
|
| 834 |
+
0.9,
|
| 835 |
+
0.999
|
| 836 |
+
],
|
| 837 |
+
"ema_decay": 0.999,
|
| 838 |
+
"epoch_permutation": "fold_in(PRNGKey(seed),epoch)",
|
| 839 |
+
"epochs": 20,
|
| 840 |
+
"epsilon": 1e-08,
|
| 841 |
+
"final_batch": "pad_to_128_mask_normalize",
|
| 842 |
+
"grad_clip": 1.0,
|
| 843 |
+
"lr": 0.001,
|
| 844 |
+
"schedule": "linear_to_task_lr_then_constant",
|
| 845 |
+
"seed": 42,
|
| 846 |
+
"selection": "fixed_final_epoch",
|
| 847 |
+
"warmup_steps": 200,
|
| 848 |
+
"weight_decay": 1e-07
|
| 849 |
+
},
|
| 850 |
+
"wandb": {
|
| 851 |
+
"mode": "disabled"
|
| 852 |
+
}
|
| 853 |
+
},
|
| 854 |
+
"file": "control/registered-configs/C3-MNIST-CNN-42.json"
|
| 855 |
+
},
|
| 856 |
+
"C3-MNIST-CNN-456": {
|
| 857 |
+
"canonical_sha256": "2ba04eca4a4bec45d8cadb2c2363efe34b0ab731698ac12948f82c5ef288a099",
|
| 858 |
+
"config": {
|
| 859 |
+
"data": {
|
| 860 |
+
"augmentation": false,
|
| 861 |
+
"dir": "/data/train/mnist",
|
| 862 |
+
"examples": 60000,
|
| 863 |
+
"input_range": [
|
| 864 |
+
0,
|
| 865 |
+
1
|
| 866 |
+
],
|
| 867 |
+
"loader": "image",
|
| 868 |
+
"normalization": false,
|
| 869 |
+
"train_split": "train",
|
| 870 |
+
"val_splits": []
|
| 871 |
+
},
|
| 872 |
+
"dtype": "float32",
|
| 873 |
+
"model": {
|
| 874 |
+
"layout": "NHWC/HWIO/NHWC",
|
| 875 |
+
"parameter_count": 65642,
|
| 876 |
+
"post_pool_sizes": [
|
| 877 |
+
14,
|
| 878 |
+
30
|
| 879 |
+
],
|
| 880 |
+
"pre_pool_sizes": [
|
| 881 |
+
28,
|
| 882 |
+
60
|
| 883 |
+
]
|
| 884 |
+
},
|
| 885 |
+
"model_type": "mnist_cnn_repro",
|
| 886 |
+
"task": "mnist",
|
| 887 |
+
"task_cfg": {},
|
| 888 |
+
"training": {
|
| 889 |
+
"batch_size": 128,
|
| 890 |
+
"betas": [
|
| 891 |
+
0.9,
|
| 892 |
+
0.999
|
| 893 |
+
],
|
| 894 |
+
"ema_decay": 0.999,
|
| 895 |
+
"epoch_permutation": "fold_in(PRNGKey(seed),epoch)",
|
| 896 |
+
"epochs": 20,
|
| 897 |
+
"epsilon": 1e-08,
|
| 898 |
+
"final_batch": "pad_to_128_mask_normalize",
|
| 899 |
+
"grad_clip": 1.0,
|
| 900 |
+
"lr": 0.001,
|
| 901 |
+
"schedule": "linear_to_task_lr_then_constant",
|
| 902 |
+
"seed": 456,
|
| 903 |
+
"selection": "fixed_final_epoch",
|
| 904 |
+
"warmup_steps": 200,
|
| 905 |
+
"weight_decay": 1e-07
|
| 906 |
+
},
|
| 907 |
+
"wandb": {
|
| 908 |
+
"mode": "disabled"
|
| 909 |
+
}
|
| 910 |
+
},
|
| 911 |
+
"file": "control/registered-configs/C3-MNIST-CNN-456.json"
|
| 912 |
+
},
|
| 913 |
+
"C4-MAZE-QUADRATIC-123": {
|
| 914 |
+
"canonical_sha256": "5f7b04835bc32b012a5e2d5e12e2b1512c3c95343ac7d8ac97a77a46178f723e",
|
| 915 |
+
"config": {
|
| 916 |
+
"data": {
|
| 917 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 918 |
+
"loader": "puzzle",
|
| 919 |
+
"train_split": "train",
|
| 920 |
+
"val_splits": []
|
| 921 |
+
},
|
| 922 |
+
"dtype": "float32",
|
| 923 |
+
"model": {
|
| 924 |
+
"cg_iters": 5,
|
| 925 |
+
"comm_norm_type": "layernorm",
|
| 926 |
+
"d_e": 5,
|
| 927 |
+
"d_v": 10,
|
| 928 |
+
"dec_hidden_dim": 256,
|
| 929 |
+
"decoder_arch": "mlp_concat_v2",
|
| 930 |
+
"enc_hidden_dim": 256,
|
| 931 |
+
"encoder_arch": "mlp_v2",
|
| 932 |
+
"gamma": 5.0,
|
| 933 |
+
"lora_init_style": "standard",
|
| 934 |
+
"lora_rank": 4,
|
| 935 |
+
"num_classes": 6,
|
| 936 |
+
"num_directions": 8,
|
| 937 |
+
"objective_mode": "quadratic",
|
| 938 |
+
"rho_init": 0.25,
|
| 939 |
+
"rm_init": "orthonormal",
|
| 940 |
+
"rm_mode": "context",
|
| 941 |
+
"rm_sharing": "directional",
|
| 942 |
+
"tikhonov_eps": 1e-05,
|
| 943 |
+
"x_solver": "diagonal_prox",
|
| 944 |
+
"z_mode": "prox",
|
| 945 |
+
"z_solver": "unrolled_cg"
|
| 946 |
+
},
|
| 947 |
+
"model_type": "sheaf",
|
| 948 |
+
"task": "maze",
|
| 949 |
+
"task_cfg": {
|
| 950 |
+
"connectivity": 8,
|
| 951 |
+
"num_classes": 6,
|
| 952 |
+
"patch_size": 3,
|
| 953 |
+
"stride": 2
|
| 954 |
+
},
|
| 955 |
+
"training": {
|
| 956 |
+
"K_eval": 100,
|
| 957 |
+
"K_train": 40,
|
| 958 |
+
"batch_size": 128,
|
| 959 |
+
"ema_decay": 0.999,
|
| 960 |
+
"epochs": 50,
|
| 961 |
+
"exit_on_nan": true,
|
| 962 |
+
"grad_clip": 1.0,
|
| 963 |
+
"loss_window": 4,
|
| 964 |
+
"lr": 0.0003,
|
| 965 |
+
"mpnn_eval_rounds": 100,
|
| 966 |
+
"mpnn_train_rounds": 40,
|
| 967 |
+
"seed": 123,
|
| 968 |
+
"train_iters_dist": "uniform",
|
| 969 |
+
"train_iters_min": 15,
|
| 970 |
+
"val_interval": 5,
|
| 971 |
+
"warmup_steps": 200,
|
| 972 |
+
"weight_decay": 1e-06
|
| 973 |
+
},
|
| 974 |
+
"wandb": {
|
| 975 |
+
"entity": null,
|
| 976 |
+
"group": null,
|
| 977 |
+
"mode": "disabled",
|
| 978 |
+
"name": null,
|
| 979 |
+
"project": "sheaf-admm",
|
| 980 |
+
"tags": []
|
| 981 |
+
}
|
| 982 |
+
},
|
| 983 |
+
"file": "control/registered-configs/C4-MAZE-QUADRATIC-123.json"
|
| 984 |
+
},
|
| 985 |
+
"C4-MAZE-QUADRATIC-42": {
|
| 986 |
+
"canonical_sha256": "87eb0690dbe06c6c70044866181311c9f2c0f7f724e75063464c501c3ef533db",
|
| 987 |
+
"config": {
|
| 988 |
+
"data": {
|
| 989 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 990 |
+
"loader": "puzzle",
|
| 991 |
+
"train_split": "train",
|
| 992 |
+
"val_splits": []
|
| 993 |
+
},
|
| 994 |
+
"dtype": "float32",
|
| 995 |
+
"model": {
|
| 996 |
+
"cg_iters": 5,
|
| 997 |
+
"comm_norm_type": "layernorm",
|
| 998 |
+
"d_e": 5,
|
| 999 |
+
"d_v": 10,
|
| 1000 |
+
"dec_hidden_dim": 256,
|
| 1001 |
+
"decoder_arch": "mlp_concat_v2",
|
| 1002 |
+
"enc_hidden_dim": 256,
|
| 1003 |
+
"encoder_arch": "mlp_v2",
|
| 1004 |
+
"gamma": 5.0,
|
| 1005 |
+
"lora_init_style": "standard",
|
| 1006 |
+
"lora_rank": 4,
|
| 1007 |
+
"num_classes": 6,
|
| 1008 |
+
"num_directions": 8,
|
| 1009 |
+
"objective_mode": "quadratic",
|
| 1010 |
+
"rho_init": 0.25,
|
| 1011 |
+
"rm_init": "orthonormal",
|
| 1012 |
+
"rm_mode": "context",
|
| 1013 |
+
"rm_sharing": "directional",
|
| 1014 |
+
"tikhonov_eps": 1e-05,
|
| 1015 |
+
"x_solver": "diagonal_prox",
|
| 1016 |
+
"z_mode": "prox",
|
| 1017 |
+
"z_solver": "unrolled_cg"
|
| 1018 |
+
},
|
| 1019 |
+
"model_type": "sheaf",
|
| 1020 |
+
"task": "maze",
|
| 1021 |
+
"task_cfg": {
|
| 1022 |
+
"connectivity": 8,
|
| 1023 |
+
"num_classes": 6,
|
| 1024 |
+
"patch_size": 3,
|
| 1025 |
+
"stride": 2
|
| 1026 |
+
},
|
| 1027 |
+
"training": {
|
| 1028 |
+
"K_eval": 100,
|
| 1029 |
+
"K_train": 40,
|
| 1030 |
+
"batch_size": 128,
|
| 1031 |
+
"ema_decay": 0.999,
|
| 1032 |
+
"epochs": 50,
|
| 1033 |
+
"exit_on_nan": true,
|
| 1034 |
+
"grad_clip": 1.0,
|
| 1035 |
+
"loss_window": 4,
|
| 1036 |
+
"lr": 0.0003,
|
| 1037 |
+
"mpnn_eval_rounds": 100,
|
| 1038 |
+
"mpnn_train_rounds": 40,
|
| 1039 |
+
"seed": 42,
|
| 1040 |
+
"train_iters_dist": "uniform",
|
| 1041 |
+
"train_iters_min": 15,
|
| 1042 |
+
"val_interval": 5,
|
| 1043 |
+
"warmup_steps": 200,
|
| 1044 |
+
"weight_decay": 1e-06
|
| 1045 |
+
},
|
| 1046 |
+
"wandb": {
|
| 1047 |
+
"entity": null,
|
| 1048 |
+
"group": null,
|
| 1049 |
+
"mode": "disabled",
|
| 1050 |
+
"name": null,
|
| 1051 |
+
"project": "sheaf-admm",
|
| 1052 |
+
"tags": []
|
| 1053 |
+
}
|
| 1054 |
+
},
|
| 1055 |
+
"file": "control/registered-configs/C4-MAZE-QUADRATIC-42.json"
|
| 1056 |
+
},
|
| 1057 |
+
"C4-MAZE-QUADRATIC-456": {
|
| 1058 |
+
"canonical_sha256": "14a74656730c7fcc84be1e1cbda241310e68a92cc34f714820b7a5f32f18997e",
|
| 1059 |
+
"config": {
|
| 1060 |
+
"data": {
|
| 1061 |
+
"dir": "/data/train/maze_std3_19px_10k",
|
| 1062 |
+
"loader": "puzzle",
|
| 1063 |
+
"train_split": "train",
|
| 1064 |
+
"val_splits": []
|
| 1065 |
+
},
|
| 1066 |
+
"dtype": "float32",
|
| 1067 |
+
"model": {
|
| 1068 |
+
"cg_iters": 5,
|
| 1069 |
+
"comm_norm_type": "layernorm",
|
| 1070 |
+
"d_e": 5,
|
| 1071 |
+
"d_v": 10,
|
| 1072 |
+
"dec_hidden_dim": 256,
|
| 1073 |
+
"decoder_arch": "mlp_concat_v2",
|
| 1074 |
+
"enc_hidden_dim": 256,
|
| 1075 |
+
"encoder_arch": "mlp_v2",
|
| 1076 |
+
"gamma": 5.0,
|
| 1077 |
+
"lora_init_style": "standard",
|
| 1078 |
+
"lora_rank": 4,
|
| 1079 |
+
"num_classes": 6,
|
| 1080 |
+
"num_directions": 8,
|
| 1081 |
+
"objective_mode": "quadratic",
|
| 1082 |
+
"rho_init": 0.25,
|
| 1083 |
+
"rm_init": "orthonormal",
|
| 1084 |
+
"rm_mode": "context",
|
| 1085 |
+
"rm_sharing": "directional",
|
| 1086 |
+
"tikhonov_eps": 1e-05,
|
| 1087 |
+
"x_solver": "diagonal_prox",
|
| 1088 |
+
"z_mode": "prox",
|
| 1089 |
+
"z_solver": "unrolled_cg"
|
| 1090 |
+
},
|
| 1091 |
+
"model_type": "sheaf",
|
| 1092 |
+
"task": "maze",
|
| 1093 |
+
"task_cfg": {
|
| 1094 |
+
"connectivity": 8,
|
| 1095 |
+
"num_classes": 6,
|
| 1096 |
+
"patch_size": 3,
|
| 1097 |
+
"stride": 2
|
| 1098 |
+
},
|
| 1099 |
+
"training": {
|
| 1100 |
+
"K_eval": 100,
|
| 1101 |
+
"K_train": 40,
|
| 1102 |
+
"batch_size": 128,
|
| 1103 |
+
"ema_decay": 0.999,
|
| 1104 |
+
"epochs": 50,
|
| 1105 |
+
"exit_on_nan": true,
|
| 1106 |
+
"grad_clip": 1.0,
|
| 1107 |
+
"loss_window": 4,
|
| 1108 |
+
"lr": 0.0003,
|
| 1109 |
+
"mpnn_eval_rounds": 100,
|
| 1110 |
+
"mpnn_train_rounds": 40,
|
| 1111 |
+
"seed": 456,
|
| 1112 |
+
"train_iters_dist": "uniform",
|
| 1113 |
+
"train_iters_min": 15,
|
| 1114 |
+
"val_interval": 5,
|
| 1115 |
+
"warmup_steps": 200,
|
| 1116 |
+
"weight_decay": 1e-06
|
| 1117 |
+
},
|
| 1118 |
+
"wandb": {
|
| 1119 |
+
"entity": null,
|
| 1120 |
+
"group": null,
|
| 1121 |
+
"mode": "disabled",
|
| 1122 |
+
"name": null,
|
| 1123 |
+
"project": "sheaf-admm",
|
| 1124 |
+
"tags": []
|
| 1125 |
+
}
|
| 1126 |
+
},
|
| 1127 |
+
"file": "control/registered-configs/C4-MAZE-QUADRATIC-456.json"
|
| 1128 |
+
},
|
| 1129 |
+
"C4-SUD-IDENTITY-123": {
|
| 1130 |
+
"canonical_sha256": "3bbb5dec4ca83cbab0379208b94d19b570eb97e2e411259208734c7467ef06ef",
|
| 1131 |
+
"config": {
|
| 1132 |
+
"data": {
|
| 1133 |
+
"dir": "/data/train/sudoku_easy",
|
| 1134 |
+
"loader": "puzzle",
|
| 1135 |
+
"train_split": "train",
|
| 1136 |
+
"val_splits": []
|
| 1137 |
+
},
|
| 1138 |
+
"dtype": "float32",
|
| 1139 |
+
"model": {
|
| 1140 |
+
"cg_iters": 5,
|
| 1141 |
+
"comm_norm_type": "layernorm",
|
| 1142 |
+
"d_e": 32,
|
| 1143 |
+
"d_v": 288,
|
| 1144 |
+
"dec_hidden_dims": [
|
| 1145 |
+
256
|
| 1146 |
+
],
|
| 1147 |
+
"decoder_arch": "sudoku",
|
| 1148 |
+
"enc_d_model": 128,
|
| 1149 |
+
"enc_num_blocks": 2,
|
| 1150 |
+
"encoder_arch": "sudoku",
|
| 1151 |
+
"gamma": 2.0,
|
| 1152 |
+
"num_classes": 10,
|
| 1153 |
+
"num_directions": 9,
|
| 1154 |
+
"objective_mode": "non_negative",
|
| 1155 |
+
"rho_init": 0.25,
|
| 1156 |
+
"rm_constant": true,
|
| 1157 |
+
"rm_init": "identity",
|
| 1158 |
+
"rm_mode": "fixed",
|
| 1159 |
+
"rm_sharing": "sudoku",
|
| 1160 |
+
"x_solver": "diagonal_prox",
|
| 1161 |
+
"z_mode": "prox",
|
| 1162 |
+
"z_solver": "unrolled_cg"
|
| 1163 |
+
},
|
| 1164 |
+
"model_type": "sheaf",
|
| 1165 |
+
"task": "sudoku",
|
| 1166 |
+
"task_cfg": {},
|
| 1167 |
+
"training": {
|
| 1168 |
+
"K_eval": 50,
|
| 1169 |
+
"K_train": 20,
|
| 1170 |
+
"batch_size": 128,
|
| 1171 |
+
"ema_decay": 0.999,
|
| 1172 |
+
"epochs": 10,
|
| 1173 |
+
"exit_on_nan": true,
|
| 1174 |
+
"grad_clip": 1.0,
|
| 1175 |
+
"loss_window": 2,
|
| 1176 |
+
"lr": 0.0017,
|
| 1177 |
+
"mpnn_eval_rounds": 100,
|
| 1178 |
+
"mpnn_train_rounds": 40,
|
| 1179 |
+
"seed": 123,
|
| 1180 |
+
"train_iters_dist": "fixed",
|
| 1181 |
+
"train_iters_min": 15,
|
| 1182 |
+
"val_interval": 1,
|
| 1183 |
+
"warmup_steps": 200,
|
| 1184 |
+
"weight_decay": 1e-07
|
| 1185 |
+
},
|
| 1186 |
+
"wandb": {
|
| 1187 |
+
"entity": null,
|
| 1188 |
+
"group": null,
|
| 1189 |
+
"mode": "disabled",
|
| 1190 |
+
"name": null,
|
| 1191 |
+
"project": "sheaf-admm",
|
| 1192 |
+
"tags": []
|
| 1193 |
+
}
|
| 1194 |
+
},
|
| 1195 |
+
"file": "control/registered-configs/C4-SUD-IDENTITY-123.json"
|
| 1196 |
+
},
|
| 1197 |
+
"C4-SUD-IDENTITY-42": {
|
| 1198 |
+
"canonical_sha256": "d6af26f0bfbcb23770bcd838ee12b937401c2dcc060dff601ff32de262e70f85",
|
| 1199 |
+
"config": {
|
| 1200 |
+
"data": {
|
| 1201 |
+
"dir": "/data/train/sudoku_easy",
|
| 1202 |
+
"loader": "puzzle",
|
| 1203 |
+
"train_split": "train",
|
| 1204 |
+
"val_splits": []
|
| 1205 |
+
},
|
| 1206 |
+
"dtype": "float32",
|
| 1207 |
+
"model": {
|
| 1208 |
+
"cg_iters": 5,
|
| 1209 |
+
"comm_norm_type": "layernorm",
|
| 1210 |
+
"d_e": 32,
|
| 1211 |
+
"d_v": 288,
|
| 1212 |
+
"dec_hidden_dims": [
|
| 1213 |
+
256
|
| 1214 |
+
],
|
| 1215 |
+
"decoder_arch": "sudoku",
|
| 1216 |
+
"enc_d_model": 128,
|
| 1217 |
+
"enc_num_blocks": 2,
|
| 1218 |
+
"encoder_arch": "sudoku",
|
| 1219 |
+
"gamma": 2.0,
|
| 1220 |
+
"num_classes": 10,
|
| 1221 |
+
"num_directions": 9,
|
| 1222 |
+
"objective_mode": "non_negative",
|
| 1223 |
+
"rho_init": 0.25,
|
| 1224 |
+
"rm_constant": true,
|
| 1225 |
+
"rm_init": "identity",
|
| 1226 |
+
"rm_mode": "fixed",
|
| 1227 |
+
"rm_sharing": "sudoku",
|
| 1228 |
+
"x_solver": "diagonal_prox",
|
| 1229 |
+
"z_mode": "prox",
|
| 1230 |
+
"z_solver": "unrolled_cg"
|
| 1231 |
+
},
|
| 1232 |
+
"model_type": "sheaf",
|
| 1233 |
+
"task": "sudoku",
|
| 1234 |
+
"task_cfg": {},
|
| 1235 |
+
"training": {
|
| 1236 |
+
"K_eval": 50,
|
| 1237 |
+
"K_train": 20,
|
| 1238 |
+
"batch_size": 128,
|
| 1239 |
+
"ema_decay": 0.999,
|
| 1240 |
+
"epochs": 10,
|
| 1241 |
+
"exit_on_nan": true,
|
| 1242 |
+
"grad_clip": 1.0,
|
| 1243 |
+
"loss_window": 2,
|
| 1244 |
+
"lr": 0.0017,
|
| 1245 |
+
"mpnn_eval_rounds": 100,
|
| 1246 |
+
"mpnn_train_rounds": 40,
|
| 1247 |
+
"seed": 42,
|
| 1248 |
+
"train_iters_dist": "fixed",
|
| 1249 |
+
"train_iters_min": 15,
|
| 1250 |
+
"val_interval": 1,
|
| 1251 |
+
"warmup_steps": 200,
|
| 1252 |
+
"weight_decay": 1e-07
|
| 1253 |
+
},
|
| 1254 |
+
"wandb": {
|
| 1255 |
+
"entity": null,
|
| 1256 |
+
"group": null,
|
| 1257 |
+
"mode": "disabled",
|
| 1258 |
+
"name": null,
|
| 1259 |
+
"project": "sheaf-admm",
|
| 1260 |
+
"tags": []
|
| 1261 |
+
}
|
| 1262 |
+
},
|
| 1263 |
+
"file": "control/registered-configs/C4-SUD-IDENTITY-42.json"
|
| 1264 |
+
},
|
| 1265 |
+
"C4-SUD-IDENTITY-456": {
|
| 1266 |
+
"canonical_sha256": "265179cbf9101405f1b04175bc3d247a719ceb8a415d11b84345a240365cbd49",
|
| 1267 |
+
"config": {
|
| 1268 |
+
"data": {
|
| 1269 |
+
"dir": "/data/train/sudoku_easy",
|
| 1270 |
+
"loader": "puzzle",
|
| 1271 |
+
"train_split": "train",
|
| 1272 |
+
"val_splits": []
|
| 1273 |
+
},
|
| 1274 |
+
"dtype": "float32",
|
| 1275 |
+
"model": {
|
| 1276 |
+
"cg_iters": 5,
|
| 1277 |
+
"comm_norm_type": "layernorm",
|
| 1278 |
+
"d_e": 32,
|
| 1279 |
+
"d_v": 288,
|
| 1280 |
+
"dec_hidden_dims": [
|
| 1281 |
+
256
|
| 1282 |
+
],
|
| 1283 |
+
"decoder_arch": "sudoku",
|
| 1284 |
+
"enc_d_model": 128,
|
| 1285 |
+
"enc_num_blocks": 2,
|
| 1286 |
+
"encoder_arch": "sudoku",
|
| 1287 |
+
"gamma": 2.0,
|
| 1288 |
+
"num_classes": 10,
|
| 1289 |
+
"num_directions": 9,
|
| 1290 |
+
"objective_mode": "non_negative",
|
| 1291 |
+
"rho_init": 0.25,
|
| 1292 |
+
"rm_constant": true,
|
| 1293 |
+
"rm_init": "identity",
|
| 1294 |
+
"rm_mode": "fixed",
|
| 1295 |
+
"rm_sharing": "sudoku",
|
| 1296 |
+
"x_solver": "diagonal_prox",
|
| 1297 |
+
"z_mode": "prox",
|
| 1298 |
+
"z_solver": "unrolled_cg"
|
| 1299 |
+
},
|
| 1300 |
+
"model_type": "sheaf",
|
| 1301 |
+
"task": "sudoku",
|
| 1302 |
+
"task_cfg": {},
|
| 1303 |
+
"training": {
|
| 1304 |
+
"K_eval": 50,
|
| 1305 |
+
"K_train": 20,
|
| 1306 |
+
"batch_size": 128,
|
| 1307 |
+
"ema_decay": 0.999,
|
| 1308 |
+
"epochs": 10,
|
| 1309 |
+
"exit_on_nan": true,
|
| 1310 |
+
"grad_clip": 1.0,
|
| 1311 |
+
"loss_window": 2,
|
| 1312 |
+
"lr": 0.0017,
|
| 1313 |
+
"mpnn_eval_rounds": 100,
|
| 1314 |
+
"mpnn_train_rounds": 40,
|
| 1315 |
+
"seed": 456,
|
| 1316 |
+
"train_iters_dist": "fixed",
|
| 1317 |
+
"train_iters_min": 15,
|
| 1318 |
+
"val_interval": 1,
|
| 1319 |
+
"warmup_steps": 200,
|
| 1320 |
+
"weight_decay": 1e-07
|
| 1321 |
+
},
|
| 1322 |
+
"wandb": {
|
| 1323 |
+
"entity": null,
|
| 1324 |
+
"group": null,
|
| 1325 |
+
"mode": "disabled",
|
| 1326 |
+
"name": null,
|
| 1327 |
+
"project": "sheaf-admm",
|
| 1328 |
+
"tags": []
|
| 1329 |
+
}
|
| 1330 |
+
},
|
| 1331 |
+
"file": "control/registered-configs/C4-SUD-IDENTITY-456.json"
|
| 1332 |
+
}
|
| 1333 |
+
},
|
| 1334 |
+
"run_identities": [
|
| 1335 |
+
"C1-SUD-MPNN225-42",
|
| 1336 |
+
"C1-SUD-MPNN225-123",
|
| 1337 |
+
"C1-SUD-MPNN225-456",
|
| 1338 |
+
"C2-MAZE-MPNN84-42",
|
| 1339 |
+
"C2-MAZE-MPNN84-123",
|
| 1340 |
+
"C2-MAZE-MPNN84-456",
|
| 1341 |
+
"C3-MNIST-CNN-42",
|
| 1342 |
+
"C3-MNIST-CNN-123",
|
| 1343 |
+
"C3-MNIST-CNN-456",
|
| 1344 |
+
"C4-SUD-IDENTITY-42",
|
| 1345 |
+
"C4-SUD-IDENTITY-123",
|
| 1346 |
+
"C4-SUD-IDENTITY-456",
|
| 1347 |
+
"C4-MAZE-QUADRATIC-42",
|
| 1348 |
+
"C4-MAZE-QUADRATIC-123",
|
| 1349 |
+
"C4-MAZE-QUADRATIC-456"
|
| 1350 |
+
],
|
| 1351 |
+
"submission_limits": {
|
| 1352 |
+
"all_returned_ids_max": 29,
|
| 1353 |
+
"cpu_retry_ids": 1,
|
| 1354 |
+
"cpu_returned_ids_max": 3,
|
| 1355 |
+
"gpu_retry_ids": 3,
|
| 1356 |
+
"gpu_returned_ids_max": 26,
|
| 1357 |
+
"primary_cpu_ids": 2,
|
| 1358 |
+
"primary_gpu_ids": 23,
|
| 1359 |
+
"target_scientific_concurrency": 6
|
| 1360 |
+
},
|
| 1361 |
+
"title": "Learning Multi-Agent Coordination via Sheaf-ADMM",
|
| 1362 |
+
"training_common": {
|
| 1363 |
+
"batch_size": 128,
|
| 1364 |
+
"dtype": "float32",
|
| 1365 |
+
"early_stopping": false,
|
| 1366 |
+
"ema_decay": 0.999,
|
| 1367 |
+
"evaluation_parameters": "EMA",
|
| 1368 |
+
"global_grad_clip": 1.0,
|
| 1369 |
+
"jax_default_matmul_precision": "highest",
|
| 1370 |
+
"optimizer": "AdamW",
|
| 1371 |
+
"schedule": "linear_to_task_lr_then_constant",
|
| 1372 |
+
"selection": "fixed_final_epoch",
|
| 1373 |
+
"training_mount_content": [
|
| 1374 |
+
"registered_task_train_split"
|
| 1375 |
+
],
|
| 1376 |
+
"validation_selection": false,
|
| 1377 |
+
"validation_splits": [],
|
| 1378 |
+
"warmup_steps": 200
|
| 1379 |
+
},
|
| 1380 |
+
"training_families": {
|
| 1381 |
+
"C1-SUD-MPNN225": {
|
| 1382 |
+
"base": "sudoku_mpnn",
|
| 1383 |
+
"data": {
|
| 1384 |
+
"dir": "datasets/sudoku_easy",
|
| 1385 |
+
"loader": "puzzle",
|
| 1386 |
+
"train_split": "train",
|
| 1387 |
+
"val_splits": []
|
| 1388 |
+
},
|
| 1389 |
+
"model": {
|
| 1390 |
+
"comm_norm_type": "layernorm",
|
| 1391 |
+
"d_e": 32,
|
| 1392 |
+
"d_v": 225,
|
| 1393 |
+
"dec_hidden_dims": [
|
| 1394 |
+
256
|
| 1395 |
+
],
|
| 1396 |
+
"decoder_arch": "sudoku",
|
| 1397 |
+
"enc_d_model": 128,
|
| 1398 |
+
"enc_num_blocks": 2,
|
| 1399 |
+
"encoder_arch": "sudoku",
|
| 1400 |
+
"mpnn_aggregation": "max",
|
| 1401 |
+
"mpnn_edge_type_mode": "slot",
|
| 1402 |
+
"mpnn_graph_readout": "per_node",
|
| 1403 |
+
"mpnn_message_dim": 32,
|
| 1404 |
+
"num_classes": 10,
|
| 1405 |
+
"num_directions": 9
|
| 1406 |
+
},
|
| 1407 |
+
"model_type": "mpnn",
|
| 1408 |
+
"paper_row_reconstruction": true,
|
| 1409 |
+
"seeds": [
|
| 1410 |
+
42,
|
| 1411 |
+
123,
|
| 1412 |
+
456
|
| 1413 |
+
],
|
| 1414 |
+
"task": "sudoku",
|
| 1415 |
+
"task_cfg": {},
|
| 1416 |
+
"training": {
|
| 1417 |
+
"epochs": 10,
|
| 1418 |
+
"lr": 0.0017,
|
| 1419 |
+
"mpnn_eval_rounds": 50,
|
| 1420 |
+
"mpnn_train_rounds": 20,
|
| 1421 |
+
"weight_decay": 1e-07
|
| 1422 |
+
}
|
| 1423 |
+
},
|
| 1424 |
+
"C2-MAZE-MPNN84": {
|
| 1425 |
+
"base": "maze_mpnn",
|
| 1426 |
+
"data": {
|
| 1427 |
+
"dir": "datasets/maze_std3_19px_10k",
|
| 1428 |
+
"loader": "puzzle",
|
| 1429 |
+
"train_split": "train",
|
| 1430 |
+
"val_splits": []
|
| 1431 |
+
},
|
| 1432 |
+
"model": {
|
| 1433 |
+
"comm_norm_type": "layernorm",
|
| 1434 |
+
"d_e": 42,
|
| 1435 |
+
"d_v": 84,
|
| 1436 |
+
"dec_hidden_dim": 256,
|
| 1437 |
+
"decoder_arch": "mlp_concat_v2",
|
| 1438 |
+
"enc_hidden_dim": 256,
|
| 1439 |
+
"encoder_arch": "mlp_v2",
|
| 1440 |
+
"mpnn_aggregation": "max",
|
| 1441 |
+
"mpnn_edge_type_mode": "spatial",
|
| 1442 |
+
"mpnn_graph_readout": "per_node",
|
| 1443 |
+
"mpnn_message_dim": 42,
|
| 1444 |
+
"num_classes": 6,
|
| 1445 |
+
"num_directions": 8
|
| 1446 |
+
},
|
| 1447 |
+
"model_type": "mpnn",
|
| 1448 |
+
"seeds": [
|
| 1449 |
+
42,
|
| 1450 |
+
123,
|
| 1451 |
+
456
|
| 1452 |
+
],
|
| 1453 |
+
"task": "maze",
|
| 1454 |
+
"task_cfg": {
|
| 1455 |
+
"connectivity": 8,
|
| 1456 |
+
"num_classes": 6,
|
| 1457 |
+
"patch_size": 3,
|
| 1458 |
+
"stride": 2
|
| 1459 |
+
},
|
| 1460 |
+
"training": {
|
| 1461 |
+
"epochs": 50,
|
| 1462 |
+
"lr": 0.0003,
|
| 1463 |
+
"mpnn_eval_rounds": 100,
|
| 1464 |
+
"mpnn_train_rounds": 40,
|
| 1465 |
+
"train_round_distribution": "fixed",
|
| 1466 |
+
"weight_decay": 1e-06
|
| 1467 |
+
}
|
| 1468 |
+
},
|
| 1469 |
+
"C3-MNIST-CNN": {
|
| 1470 |
+
"data": {
|
| 1471 |
+
"augmentation": false,
|
| 1472 |
+
"input_range": [
|
| 1473 |
+
0,
|
| 1474 |
+
1
|
| 1475 |
+
],
|
| 1476 |
+
"normalization": false,
|
| 1477 |
+
"val_splits": []
|
| 1478 |
+
},
|
| 1479 |
+
"model": {
|
| 1480 |
+
"bias_init": "zeros",
|
| 1481 |
+
"kernel_init": "glorot_uniform",
|
| 1482 |
+
"layers": [
|
| 1483 |
+
"conv3x3-1-32-same-relu",
|
| 1484 |
+
"conv3x3-32-32-same-relu",
|
| 1485 |
+
"maxpool2x2-stride2-valid",
|
| 1486 |
+
"conv3x3-32-64-same-relu",
|
| 1487 |
+
"conv3x3-64-64-same-relu",
|
| 1488 |
+
"global-spatial-mean",
|
| 1489 |
+
"dense64-10"
|
| 1490 |
+
],
|
| 1491 |
+
"layout": "NHWC/HWIO/NHWC",
|
| 1492 |
+
"parameter_count": 65642,
|
| 1493 |
+
"post_pool_sizes": [
|
| 1494 |
+
14,
|
| 1495 |
+
30
|
| 1496 |
+
],
|
| 1497 |
+
"pre_pool_sizes": [
|
| 1498 |
+
28,
|
| 1499 |
+
60
|
| 1500 |
+
]
|
| 1501 |
+
},
|
| 1502 |
+
"model_type": "mnist_cnn_repro",
|
| 1503 |
+
"seeds": [
|
| 1504 |
+
42,
|
| 1505 |
+
123,
|
| 1506 |
+
456
|
| 1507 |
+
],
|
| 1508 |
+
"task": "mnist",
|
| 1509 |
+
"training": {
|
| 1510 |
+
"betas": [
|
| 1511 |
+
0.9,
|
| 1512 |
+
0.999
|
| 1513 |
+
],
|
| 1514 |
+
"epoch_permutation": "fold_in(PRNGKey(seed),epoch)",
|
| 1515 |
+
"epochs": 20,
|
| 1516 |
+
"epsilon": 1e-08,
|
| 1517 |
+
"examples": 60000,
|
| 1518 |
+
"final_batch": "pad_to_128_mask_normalize",
|
| 1519 |
+
"kernel_only_weight_decay": 1e-07,
|
| 1520 |
+
"loss": "mean_sparse_categorical_cross_entropy",
|
| 1521 |
+
"lr": 0.001
|
| 1522 |
+
}
|
| 1523 |
+
},
|
| 1524 |
+
"C4-MAZE-QUADRATIC": {
|
| 1525 |
+
"base": "maze_sheaf",
|
| 1526 |
+
"data": {
|
| 1527 |
+
"dir": "datasets/maze_std3_19px_10k",
|
| 1528 |
+
"loader": "puzzle",
|
| 1529 |
+
"train_split": "train",
|
| 1530 |
+
"val_splits": []
|
| 1531 |
+
},
|
| 1532 |
+
"forbidden_heads": [
|
| 1533 |
+
"l1",
|
| 1534 |
+
"lower_bound",
|
| 1535 |
+
"upper_bound"
|
| 1536 |
+
],
|
| 1537 |
+
"heads": [
|
| 1538 |
+
"positive_q_diag",
|
| 1539 |
+
"q"
|
| 1540 |
+
],
|
| 1541 |
+
"model": {
|
| 1542 |
+
"cg_iters": 5,
|
| 1543 |
+
"comm_norm_type": "layernorm",
|
| 1544 |
+
"d_e": 5,
|
| 1545 |
+
"d_v": 10,
|
| 1546 |
+
"dec_hidden_dim": 256,
|
| 1547 |
+
"decoder_arch": "mlp_concat_v2",
|
| 1548 |
+
"enc_hidden_dim": 256,
|
| 1549 |
+
"encoder_arch": "mlp_v2",
|
| 1550 |
+
"gamma": 5,
|
| 1551 |
+
"lora_init_style": "standard",
|
| 1552 |
+
"lora_rank": 4,
|
| 1553 |
+
"num_classes": 6,
|
| 1554 |
+
"num_directions": 8,
|
| 1555 |
+
"objective_mode": "quadratic",
|
| 1556 |
+
"rho_init": 0.25,
|
| 1557 |
+
"rm_init": "orthonormal",
|
| 1558 |
+
"rm_mode": "context",
|
| 1559 |
+
"rm_sharing": "directional",
|
| 1560 |
+
"tikhonov_eps": 1e-05,
|
| 1561 |
+
"x_solver": "diagonal_prox",
|
| 1562 |
+
"z_mode": "prox",
|
| 1563 |
+
"z_solver": "unrolled_cg"
|
| 1564 |
+
},
|
| 1565 |
+
"model_type": "sheaf",
|
| 1566 |
+
"overrides": {
|
| 1567 |
+
"data.val_splits": [],
|
| 1568 |
+
"model.objective_mode": "quadratic"
|
| 1569 |
+
},
|
| 1570 |
+
"seeds": [
|
| 1571 |
+
42,
|
| 1572 |
+
123,
|
| 1573 |
+
456
|
| 1574 |
+
],
|
| 1575 |
+
"task": "maze",
|
| 1576 |
+
"task_cfg": {
|
| 1577 |
+
"connectivity": 8,
|
| 1578 |
+
"num_classes": 6,
|
| 1579 |
+
"patch_size": 3,
|
| 1580 |
+
"stride": 2
|
| 1581 |
+
},
|
| 1582 |
+
"training": {
|
| 1583 |
+
"K_eval": 100,
|
| 1584 |
+
"K_train": 40,
|
| 1585 |
+
"epochs": 50,
|
| 1586 |
+
"loss_window": 4,
|
| 1587 |
+
"lr": 0.0003,
|
| 1588 |
+
"train_K_distribution": "uniform_inclusive_15_40",
|
| 1589 |
+
"train_iters_min": 15,
|
| 1590 |
+
"weight_decay": 1e-06
|
| 1591 |
+
}
|
| 1592 |
+
},
|
| 1593 |
+
"C4-SUD-IDENTITY": {
|
| 1594 |
+
"base": "sudoku_sheaf",
|
| 1595 |
+
"constant_map": "F=[I_32,0,...,0] for every slot and endpoint",
|
| 1596 |
+
"data": {
|
| 1597 |
+
"dir": "datasets/sudoku_easy",
|
| 1598 |
+
"loader": "puzzle",
|
| 1599 |
+
"train_split": "train",
|
| 1600 |
+
"val_splits": []
|
| 1601 |
+
},
|
| 1602 |
+
"model": {
|
| 1603 |
+
"cg_iters": 5,
|
| 1604 |
+
"comm_norm_type": "layernorm",
|
| 1605 |
+
"d_e": 32,
|
| 1606 |
+
"d_v": 288,
|
| 1607 |
+
"dec_hidden_dims": [
|
| 1608 |
+
256
|
| 1609 |
+
],
|
| 1610 |
+
"decoder_arch": "sudoku",
|
| 1611 |
+
"enc_d_model": 128,
|
| 1612 |
+
"enc_num_blocks": 2,
|
| 1613 |
+
"encoder_arch": "sudoku",
|
| 1614 |
+
"gamma": 2.0,
|
| 1615 |
+
"num_classes": 10,
|
| 1616 |
+
"num_directions": 9,
|
| 1617 |
+
"objective_mode": "non_negative",
|
| 1618 |
+
"rho_init": 0.25,
|
| 1619 |
+
"rm_constant": true,
|
| 1620 |
+
"rm_init": "identity",
|
| 1621 |
+
"rm_mode": "fixed",
|
| 1622 |
+
"rm_sharing": "sudoku",
|
| 1623 |
+
"x_solver": "diagonal_prox",
|
| 1624 |
+
"z_mode": "prox",
|
| 1625 |
+
"z_solver": "unrolled_cg"
|
| 1626 |
+
},
|
| 1627 |
+
"model_type": "sheaf",
|
| 1628 |
+
"overrides": {
|
| 1629 |
+
"data.val_splits": [],
|
| 1630 |
+
"model.rm_constant": true,
|
| 1631 |
+
"model.rm_init": "identity"
|
| 1632 |
+
},
|
| 1633 |
+
"pairing_requirement": "control and intervention common leaves and counters bit-identical before replacement",
|
| 1634 |
+
"parameter_tree_requirement": "no restriction-map leaf",
|
| 1635 |
+
"seeds": [
|
| 1636 |
+
42,
|
| 1637 |
+
123,
|
| 1638 |
+
456
|
| 1639 |
+
],
|
| 1640 |
+
"task": "sudoku",
|
| 1641 |
+
"task_cfg": {},
|
| 1642 |
+
"training": {
|
| 1643 |
+
"K_eval": 50,
|
| 1644 |
+
"K_train": 20,
|
| 1645 |
+
"epochs": 10,
|
| 1646 |
+
"loss_window": 2,
|
| 1647 |
+
"lr": 0.0017,
|
| 1648 |
+
"train_iters_dist": "fixed",
|
| 1649 |
+
"weight_decay": 1e-07
|
| 1650 |
+
}
|
| 1651 |
+
}
|
| 1652 |
+
}
|
| 1653 |
+
}
|
assets/fig2.png
ADDED
|
Git LFS Details
|
configs/config.yaml
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Base config. Shared optimization settings live here (the paper's shared HPs);
|
| 2 |
+
# per-task architecture + solver settings come from configs/experiment/*.yaml,
|
| 3 |
+
# selected with `+experiment=<name>`.
|
| 4 |
+
defaults:
|
| 5 |
+
- _self_
|
| 6 |
+
|
| 7 |
+
task: maze # maze | mnist | sudoku (selects the task hook)
|
| 8 |
+
model_type: sheaf # sheaf | mpnn
|
| 9 |
+
dtype: float32 # registered reproduction runs forbid mixed precision
|
| 10 |
+
|
| 11 |
+
wandb:
|
| 12 |
+
project: sheaf-admm
|
| 13 |
+
entity: null # null -> your default wandb entity
|
| 14 |
+
name: null # null -> wandb auto-name
|
| 15 |
+
group: null # e.g. group seeds of one experiment together
|
| 16 |
+
tags: []
|
| 17 |
+
mode: disabled # online | offline | disabled
|
| 18 |
+
|
| 19 |
+
training:
|
| 20 |
+
seed: 42 # paper sweeps seeds {42, 123, 456}
|
| 21 |
+
lr: 3.0e-4
|
| 22 |
+
weight_decay: 1.0e-6
|
| 23 |
+
epochs: 50
|
| 24 |
+
batch_size: 128
|
| 25 |
+
warmup_steps: 200 # linear warmup -> constant LR
|
| 26 |
+
grad_clip: 1.0 # global-norm clip
|
| 27 |
+
ema_decay: 0.999 # EMA of params, used at eval
|
| 28 |
+
exit_on_nan: true
|
| 29 |
+
val_interval: 5
|
| 30 |
+
|
| 31 |
+
# ADMM horizon (Sheaf): K_train iterations at train, K_eval at eval.
|
| 32 |
+
K_train: 40
|
| 33 |
+
K_eval: 100
|
| 34 |
+
loss_window: 4 # average CE over the final w iterates
|
| 35 |
+
train_iters_dist: fixed # fixed | uniform (Maze resamples K ~ U[train_iters_min, K_train])
|
| 36 |
+
train_iters_min: 15
|
| 37 |
+
|
| 38 |
+
# MPNN baseline message-passing rounds (the K analog for the recurrent MPNN).
|
| 39 |
+
mpnn_train_rounds: 40
|
| 40 |
+
mpnn_eval_rounds: 100
|
| 41 |
+
|
| 42 |
+
data:
|
| 43 |
+
dir: ??? # set per experiment
|
| 44 |
+
train_split: train
|
| 45 |
+
val_splits: [test]
|
| 46 |
+
loader: puzzle # puzzle (maze/sudoku) | image (mnist)
|
| 47 |
+
|
| 48 |
+
# kwargs forwarded to the task hook (sheaf_admm.training.make_task)
|
| 49 |
+
task_cfg: {}
|
| 50 |
+
|
| 51 |
+
# sheaf_admm.models.ModelConfig fields (set per experiment)
|
| 52 |
+
model: {}
|
configs/experiment/maze_mpnn.yaml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# Table 1 baseline: recurrent-MPNN, parameter-matched (d_v=84, ~182K), max aggregation.
|
| 3 |
+
# (CM variant: set model.d_v=10, model.d_e=5, model.mpnn_message_dim=5 -> ~49K.)
|
| 4 |
+
task: maze
|
| 5 |
+
model_type: mpnn
|
| 6 |
+
|
| 7 |
+
training:
|
| 8 |
+
lr: 3.0e-4
|
| 9 |
+
weight_decay: 1.0e-6
|
| 10 |
+
epochs: 50
|
| 11 |
+
mpnn_train_rounds: 40
|
| 12 |
+
mpnn_eval_rounds: 100
|
| 13 |
+
val_interval: 5
|
| 14 |
+
|
| 15 |
+
data:
|
| 16 |
+
dir: datasets/maze_std3_19px_10k
|
| 17 |
+
train_split: train
|
| 18 |
+
val_splits: [test, test_ood_2x, test_ood_2xW, test_ood_4x, test_ood_4xW]
|
| 19 |
+
loader: puzzle
|
| 20 |
+
|
| 21 |
+
task_cfg:
|
| 22 |
+
patch_size: 3
|
| 23 |
+
stride: 2
|
| 24 |
+
connectivity: 8
|
| 25 |
+
num_classes: 6
|
| 26 |
+
|
| 27 |
+
model:
|
| 28 |
+
num_classes: 6
|
| 29 |
+
d_v: 84
|
| 30 |
+
d_e: 42
|
| 31 |
+
encoder_arch: mlp_v2
|
| 32 |
+
enc_hidden_dim: 256
|
| 33 |
+
comm_norm_type: layernorm
|
| 34 |
+
decoder_arch: mlp_concat_v2
|
| 35 |
+
dec_hidden_dim: 256
|
| 36 |
+
mpnn_message_dim: 42
|
| 37 |
+
mpnn_aggregation: max # the strongest aggregation in the sweep
|
| 38 |
+
mpnn_edge_type_mode: spatial # 8 grid directions
|
| 39 |
+
num_directions: 8
|
| 40 |
+
mpnn_graph_readout: per_node
|
configs/experiment/maze_sheaf.yaml
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# Table 1 anchor: Sheaf-ADMM maze (d_v=10, ~182K params -> 99.9 / 98.1 / 4.5).
|
| 3 |
+
task: maze
|
| 4 |
+
model_type: sheaf
|
| 5 |
+
|
| 6 |
+
training:
|
| 7 |
+
lr: 3.0e-4
|
| 8 |
+
weight_decay: 1.0e-6
|
| 9 |
+
epochs: 50
|
| 10 |
+
K_train: 40
|
| 11 |
+
K_eval: 100
|
| 12 |
+
loss_window: 4
|
| 13 |
+
train_iters_dist: uniform # K ~ U[15, 40] per step
|
| 14 |
+
train_iters_min: 15
|
| 15 |
+
val_interval: 5
|
| 16 |
+
|
| 17 |
+
data:
|
| 18 |
+
dir: datasets/maze_std3_19px_10k
|
| 19 |
+
train_split: train
|
| 20 |
+
val_splits: [test, test_ood_2x, test_ood_2xW, test_ood_4x, test_ood_4xW]
|
| 21 |
+
loader: puzzle
|
| 22 |
+
|
| 23 |
+
task_cfg:
|
| 24 |
+
patch_size: 3
|
| 25 |
+
stride: 2
|
| 26 |
+
connectivity: 8
|
| 27 |
+
num_classes: 6
|
| 28 |
+
|
| 29 |
+
model:
|
| 30 |
+
num_classes: 6
|
| 31 |
+
d_v: 10
|
| 32 |
+
d_e: 5
|
| 33 |
+
encoder_arch: mlp_v2
|
| 34 |
+
enc_hidden_dim: 256
|
| 35 |
+
comm_norm_type: layernorm
|
| 36 |
+
objective_mode: l1box_diag # closed-form L1 + box x-update
|
| 37 |
+
x_solver: diagonal_prox
|
| 38 |
+
z_solver: unrolled_cg
|
| 39 |
+
z_mode: prox # soft consensus
|
| 40 |
+
gamma: 5.0
|
| 41 |
+
cg_iters: 5
|
| 42 |
+
tikhonov_eps: 1.0e-5
|
| 43 |
+
rm_sharing: directional # one base map per grid direction (8-way, see num_directions)
|
| 44 |
+
rm_init: orthonormal
|
| 45 |
+
rm_mode: context # LoRA-modulated restriction maps
|
| 46 |
+
lora_rank: 4
|
| 47 |
+
lora_init_style: standard
|
| 48 |
+
num_directions: 8
|
| 49 |
+
rho_init: 0.25
|
| 50 |
+
decoder_arch: mlp_concat_v2
|
| 51 |
+
dec_hidden_dim: 256
|
configs/experiment/mnist_mpnn.yaml
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# MNIST recurrent-MPNN baseline (graph-level mean-pool readout).
|
| 3 |
+
task: mnist
|
| 4 |
+
model_type: mpnn
|
| 5 |
+
|
| 6 |
+
training:
|
| 7 |
+
lr: 1.0e-3
|
| 8 |
+
weight_decay: 1.0e-7
|
| 9 |
+
epochs: 20
|
| 10 |
+
mpnn_train_rounds: 20
|
| 11 |
+
mpnn_eval_rounds: 50
|
| 12 |
+
val_interval: 5
|
| 13 |
+
|
| 14 |
+
data:
|
| 15 |
+
dir: datasets/mnist
|
| 16 |
+
train_split: train
|
| 17 |
+
val_splits: [test]
|
| 18 |
+
loader: image
|
| 19 |
+
|
| 20 |
+
task_cfg:
|
| 21 |
+
patch_size: 3
|
| 22 |
+
stride: 3
|
| 23 |
+
connectivity: 8
|
| 24 |
+
num_classes: 10
|
| 25 |
+
|
| 26 |
+
model:
|
| 27 |
+
num_classes: 10
|
| 28 |
+
d_v: 32
|
| 29 |
+
d_e: 24
|
| 30 |
+
encoder_arch: mlp
|
| 31 |
+
enc_hidden_dim: 256
|
| 32 |
+
comm_norm_type: layernorm
|
| 33 |
+
decoder_arch: classification
|
| 34 |
+
dec_hidden_dims: [128]
|
| 35 |
+
mpnn_message_dim: 24
|
| 36 |
+
mpnn_aggregation: mean
|
| 37 |
+
mpnn_edge_type_mode: spatial
|
| 38 |
+
num_directions: 8
|
| 39 |
+
mpnn_graph_readout: graph
|
configs/experiment/mnist_sheaf.yaml
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# MNIST classification (clean ~98.5%): 28x28 image, non-overlapping 3x3 patches
|
| 3 |
+
# (stride 3 -> 81 agents). Global (shared) restriction map, hard consensus (Fz=0).
|
| 4 |
+
task: mnist
|
| 5 |
+
model_type: sheaf
|
| 6 |
+
|
| 7 |
+
training:
|
| 8 |
+
lr: 1.0e-3
|
| 9 |
+
weight_decay: 1.0e-7
|
| 10 |
+
epochs: 20
|
| 11 |
+
K_train: 20
|
| 12 |
+
K_eval: 100
|
| 13 |
+
loss_window: 2
|
| 14 |
+
train_iters_dist: fixed
|
| 15 |
+
val_interval: 5
|
| 16 |
+
|
| 17 |
+
data:
|
| 18 |
+
dir: datasets/mnist
|
| 19 |
+
train_split: train
|
| 20 |
+
val_splits: [test]
|
| 21 |
+
loader: image
|
| 22 |
+
|
| 23 |
+
task_cfg:
|
| 24 |
+
patch_size: 3
|
| 25 |
+
stride: 3 # paper Table 4 stride
|
| 26 |
+
connectivity: 8
|
| 27 |
+
num_classes: 10
|
| 28 |
+
|
| 29 |
+
model:
|
| 30 |
+
num_classes: 10
|
| 31 |
+
d_v: 32
|
| 32 |
+
d_e: 24
|
| 33 |
+
encoder_arch: mlp
|
| 34 |
+
enc_hidden_dim: 256
|
| 35 |
+
comm_norm_type: layernorm
|
| 36 |
+
objective_mode: lasso
|
| 37 |
+
l1_weight: 0.006337180166370117
|
| 38 |
+
x_solver: diagonal_prox
|
| 39 |
+
z_solver: unrolled_cg
|
| 40 |
+
z_mode: project # hard consensus (Fz = 0)
|
| 41 |
+
cg_iters: 5
|
| 42 |
+
tikhonov_eps: 1.0e-5
|
| 43 |
+
rm_sharing: global # one shared restriction map
|
| 44 |
+
rm_init: orthonormal
|
| 45 |
+
rm_mode: context
|
| 46 |
+
lora_rank: 8
|
| 47 |
+
lora_init_style: legacy
|
| 48 |
+
num_directions: 8
|
| 49 |
+
rho_init: 0.12
|
| 50 |
+
decoder_arch: classification
|
| 51 |
+
dec_linear_head: true
|
| 52 |
+
dec_readout_mode: x_only
|
configs/experiment/sudoku_mpnn.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# Recurrent-MPNN Sudoku baseline.
|
| 3 |
+
task: sudoku
|
| 4 |
+
model_type: mpnn
|
| 5 |
+
|
| 6 |
+
training:
|
| 7 |
+
lr: 3.0e-4
|
| 8 |
+
weight_decay: 1.0e-7
|
| 9 |
+
epochs: 5
|
| 10 |
+
mpnn_train_rounds: 20
|
| 11 |
+
mpnn_eval_rounds: 50
|
| 12 |
+
val_interval: 1
|
| 13 |
+
|
| 14 |
+
data:
|
| 15 |
+
dir: datasets/sudoku_easy
|
| 16 |
+
train_split: train
|
| 17 |
+
val_splits: [test_hard]
|
| 18 |
+
loader: puzzle
|
| 19 |
+
|
| 20 |
+
task_cfg: {}
|
| 21 |
+
|
| 22 |
+
model:
|
| 23 |
+
num_classes: 10
|
| 24 |
+
d_v: 504
|
| 25 |
+
d_e: 32
|
| 26 |
+
encoder_arch: sudoku
|
| 27 |
+
enc_d_model: 128
|
| 28 |
+
enc_num_blocks: 2
|
| 29 |
+
comm_norm_type: layernorm
|
| 30 |
+
decoder_arch: sudoku
|
| 31 |
+
dec_hidden_dims: [256]
|
| 32 |
+
mpnn_message_dim: 32
|
| 33 |
+
mpnn_aggregation: max
|
| 34 |
+
mpnn_edge_type_mode: slot # 9 shared-cell slots
|
| 35 |
+
num_directions: 9
|
| 36 |
+
mpnn_graph_readout: per_node
|
configs/experiment/sudoku_sheaf.yaml
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# Sheaf-ADMM Sudoku with fixed restriction maps.
|
| 3 |
+
task: sudoku
|
| 4 |
+
model_type: sheaf
|
| 5 |
+
|
| 6 |
+
training:
|
| 7 |
+
lr: 1.7e-3
|
| 8 |
+
weight_decay: 1.0e-7
|
| 9 |
+
epochs: 10
|
| 10 |
+
K_train: 20
|
| 11 |
+
K_eval: 50
|
| 12 |
+
loss_window: 2
|
| 13 |
+
train_iters_dist: fixed
|
| 14 |
+
val_interval: 1
|
| 15 |
+
|
| 16 |
+
data:
|
| 17 |
+
dir: datasets/sudoku_easy
|
| 18 |
+
train_split: train
|
| 19 |
+
val_splits: [test_hard]
|
| 20 |
+
loader: puzzle
|
| 21 |
+
|
| 22 |
+
task_cfg: {} # 27 row/col/box agents are derived from the multigraph
|
| 23 |
+
|
| 24 |
+
model:
|
| 25 |
+
num_classes: 10
|
| 26 |
+
d_v: 288 # 9 cells x 32-dim blocks
|
| 27 |
+
d_e: 32
|
| 28 |
+
encoder_arch: sudoku
|
| 29 |
+
enc_d_model: 128
|
| 30 |
+
enc_num_blocks: 2
|
| 31 |
+
comm_norm_type: layernorm
|
| 32 |
+
objective_mode: non_negative # closed-form x = max((rho(z-y)-q)/(D+rho), 0)
|
| 33 |
+
x_solver: diagonal_prox
|
| 34 |
+
z_solver: unrolled_cg
|
| 35 |
+
z_mode: prox
|
| 36 |
+
gamma: 2.0
|
| 37 |
+
cg_iters: 5
|
| 38 |
+
rm_sharing: sudoku # 9 base maps = selectors onto 9 disjoint 32-d blocks
|
| 39 |
+
rm_init: soft_slice
|
| 40 |
+
rm_mode: fixed
|
| 41 |
+
num_directions: 9
|
| 42 |
+
rho_init: 0.25
|
| 43 |
+
decoder_arch: sudoku
|
| 44 |
+
dec_hidden_dims: [256]
|
configs/experiment/sudoku_sheaf_lora.yaml
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# @package _global_
|
| 2 |
+
# Sheaf-ADMM Sudoku with LoRA-modulated restriction maps.
|
| 3 |
+
task: sudoku
|
| 4 |
+
model_type: sheaf
|
| 5 |
+
|
| 6 |
+
training:
|
| 7 |
+
lr: 1.7e-3
|
| 8 |
+
weight_decay: 1.0e-7
|
| 9 |
+
epochs: 10
|
| 10 |
+
K_train: 20
|
| 11 |
+
K_eval: 50
|
| 12 |
+
loss_window: 2
|
| 13 |
+
train_iters_dist: fixed
|
| 14 |
+
val_interval: 1
|
| 15 |
+
|
| 16 |
+
data:
|
| 17 |
+
dir: datasets/sudoku_easy
|
| 18 |
+
train_split: train
|
| 19 |
+
val_splits: [test_hard]
|
| 20 |
+
loader: puzzle
|
| 21 |
+
|
| 22 |
+
task_cfg: {}
|
| 23 |
+
|
| 24 |
+
model:
|
| 25 |
+
num_classes: 10
|
| 26 |
+
d_v: 288
|
| 27 |
+
d_e: 32
|
| 28 |
+
encoder_arch: sudoku
|
| 29 |
+
enc_d_model: 128
|
| 30 |
+
enc_num_blocks: 2
|
| 31 |
+
comm_norm_type: layernorm
|
| 32 |
+
objective_mode: non_negative
|
| 33 |
+
x_solver: diagonal_prox
|
| 34 |
+
z_solver: unrolled_cg
|
| 35 |
+
z_mode: prox
|
| 36 |
+
gamma: 2.0
|
| 37 |
+
cg_iters: 5
|
| 38 |
+
rm_sharing: sudoku
|
| 39 |
+
rm_init: soft_slice
|
| 40 |
+
rm_mode: context # LoRA modulation on
|
| 41 |
+
lora_rank: 4
|
| 42 |
+
lora_init_style: standard
|
| 43 |
+
num_directions: 9
|
| 44 |
+
rho_init: 0.25
|
| 45 |
+
decoder_arch: sudoku
|
| 46 |
+
dec_hidden_dims: [256]
|
pyproject.toml
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "sheaf-admm"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "Learning multi-agent coordination via sheaf-constrained ADMM (JAX/Flax reference implementation)."
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.12,<3.13"
|
| 7 |
+
license = { text = "Apache-2.0" }
|
| 8 |
+
authors = [
|
| 9 |
+
{ name = "Jeffrey Seely" },
|
| 10 |
+
{ name = "Bartłomiej Cupiał" },
|
| 11 |
+
{ name = "Llion Jones" },
|
| 12 |
+
]
|
| 13 |
+
dependencies = [
|
| 14 |
+
"jax==0.10.1; sys_platform != 'linux'",
|
| 15 |
+
"jax[cuda12]==0.10.1; sys_platform == 'linux'",
|
| 16 |
+
"flax==0.12.7",
|
| 17 |
+
"optax==0.2.8",
|
| 18 |
+
"numpy==2.4.6",
|
| 19 |
+
"hydra-core==1.3.2",
|
| 20 |
+
"omegaconf==2.3.0",
|
| 21 |
+
# dataset construction — required to build MNIST / Sudoku / maze locally
|
| 22 |
+
"ml-datasets==0.2.1",
|
| 23 |
+
"datasets==4.8.5",
|
| 24 |
+
"huggingface-hub==1.17.0",
|
| 25 |
+
# figures / artifacts
|
| 26 |
+
"matplotlib==3.10.9",
|
| 27 |
+
"pillow==12.2.0",
|
| 28 |
+
# experiment tracking
|
| 29 |
+
"wandb==0.27.0",
|
| 30 |
+
]
|
| 31 |
+
|
| 32 |
+
[build-system]
|
| 33 |
+
requires = ["hatchling"]
|
| 34 |
+
build-backend = "hatchling.build"
|
| 35 |
+
|
| 36 |
+
[tool.hatch.build.targets.wheel]
|
| 37 |
+
packages = ["src/sheaf_admm"]
|
| 38 |
+
|
| 39 |
+
[tool.hatch.build.targets.sdist]
|
| 40 |
+
include = [
|
| 41 |
+
"/assets",
|
| 42 |
+
"/configs",
|
| 43 |
+
"/scripts",
|
| 44 |
+
"/src",
|
| 45 |
+
"/tests",
|
| 46 |
+
"/LICENSE",
|
| 47 |
+
"/README.md",
|
| 48 |
+
"/pyproject.toml",
|
| 49 |
+
"/uv.lock",
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
+
[dependency-groups]
|
| 53 |
+
dev = [
|
| 54 |
+
"pytest==9.0.3",
|
| 55 |
+
"ruff==0.15.15",
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
[tool.pytest.ini_options]
|
| 59 |
+
testpaths = ["tests"]
|
| 60 |
+
markers = [
|
| 61 |
+
"slow: end-to-end tests that train a tiny model (deselect with -m 'not slow')",
|
| 62 |
+
]
|
| 63 |
+
|
| 64 |
+
[tool.ruff]
|
| 65 |
+
line-length = 100
|
| 66 |
+
target-version = "py312"
|
| 67 |
+
|
| 68 |
+
[tool.ruff.lint]
|
| 69 |
+
# Keep math unicode (ρ, σ, γ, η, λ) — they read better than ASCII in this codebase.
|
| 70 |
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
| 71 |
+
ignore = ["E741", "RUF002", "RUF003", "B008"]
|
scripts/aggregate_attempts.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Accept a complete attempt or verify the complete accepted identity set."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.aggregation import accept_attempt, load_complete_units
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def main() -> int:
|
| 17 |
+
parser = argparse.ArgumentParser()
|
| 18 |
+
parser.add_argument("--registry", type=Path, required=True)
|
| 19 |
+
sub = parser.add_subparsers(dest="action", required=True)
|
| 20 |
+
accept = sub.add_parser("accept")
|
| 21 |
+
accept.add_argument("--logical-id", required=True)
|
| 22 |
+
accept.add_argument("--attempt-id", required=True)
|
| 23 |
+
accept.add_argument("--prefix", type=Path, required=True)
|
| 24 |
+
verify = sub.add_parser("verify")
|
| 25 |
+
verify.add_argument("--required-json", type=Path, required=True)
|
| 26 |
+
args = parser.parse_args()
|
| 27 |
+
if args.action == "accept":
|
| 28 |
+
row = accept_attempt(
|
| 29 |
+
args.registry,
|
| 30 |
+
logical_id=args.logical_id,
|
| 31 |
+
attempt_id=args.attempt_id,
|
| 32 |
+
prefix=args.prefix,
|
| 33 |
+
)
|
| 34 |
+
print(json.dumps(row, sort_keys=True))
|
| 35 |
+
else:
|
| 36 |
+
required = set(json.loads(args.required_json.read_text()))
|
| 37 |
+
print(json.dumps(load_complete_units(args.registry, required), sort_keys=True))
|
| 38 |
+
return 0
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
raise SystemExit(main())
|
scripts/build_preupload_privacy.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Assemble the exact multi-scope PRE_UPLOAD_PRIVACY receipt, failing incomplete."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import subprocess
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 13 |
+
|
| 14 |
+
from repro_control.hashing import atomic_write_json, compact_json_bytes, sha256_bytes, sha256_file
|
| 15 |
+
from repro_control.privacy import SCANNER_VERSION, scan_paths
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def main() -> int:
|
| 19 |
+
parser = argparse.ArgumentParser()
|
| 20 |
+
parser.add_argument("--checkout", type=Path, required=True)
|
| 21 |
+
parser.add_argument("--package-manifest", type=Path, required=True)
|
| 22 |
+
parser.add_argument("--image-contract", type=Path, required=True)
|
| 23 |
+
parser.add_argument("--intended-remote-map", type=Path, required=True)
|
| 24 |
+
parser.add_argument("--output", type=Path, default=Path("PRE_UPLOAD_PRIVACY.json"))
|
| 25 |
+
args = parser.parse_args()
|
| 26 |
+
checkout = args.checkout.resolve()
|
| 27 |
+
tracked = subprocess.run(
|
| 28 |
+
["git", "ls-files", "-z"],
|
| 29 |
+
cwd=checkout,
|
| 30 |
+
check=True,
|
| 31 |
+
capture_output=True,
|
| 32 |
+
).stdout.split(b"\0")
|
| 33 |
+
tracked_paths = [checkout / raw.decode() for raw in tracked if raw]
|
| 34 |
+
source = scan_paths(
|
| 35 |
+
tracked_paths,
|
| 36 |
+
relative_to=checkout,
|
| 37 |
+
intended_remote_map=json.loads(args.intended_remote_map.read_text()),
|
| 38 |
+
)
|
| 39 |
+
object_lines = subprocess.run(
|
| 40 |
+
["git", "rev-list", "--objects", "--all"],
|
| 41 |
+
cwd=checkout,
|
| 42 |
+
check=True,
|
| 43 |
+
capture_output=True,
|
| 44 |
+
text=True,
|
| 45 |
+
).stdout.splitlines()
|
| 46 |
+
git_objects = sorted(line.split()[0] for line in object_lines if line)
|
| 47 |
+
trackio = checkout / ".trackio"
|
| 48 |
+
trackio_scan = (
|
| 49 |
+
scan_paths([trackio], relative_to=checkout, intended_remote_map={})
|
| 50 |
+
if trackio.is_dir()
|
| 51 |
+
else None
|
| 52 |
+
)
|
| 53 |
+
image = json.loads(args.image_contract.read_text())
|
| 54 |
+
package_manifest = json.loads(args.package_manifest.read_text())
|
| 55 |
+
blockers = []
|
| 56 |
+
if trackio_scan is None:
|
| 57 |
+
blockers.append(".trackio scaffold does not exist")
|
| 58 |
+
if not image.get("resolved_and_read_back"):
|
| 59 |
+
blockers.append("immutable image config/history/layers are unresolved")
|
| 60 |
+
if source["findings"]:
|
| 61 |
+
blockers.append("source privacy findings exist")
|
| 62 |
+
if trackio_scan and trackio_scan["findings"]:
|
| 63 |
+
blockers.append(".trackio privacy findings exist")
|
| 64 |
+
receipt = {
|
| 65 |
+
"format": 1,
|
| 66 |
+
"scanner_version": SCANNER_VERSION,
|
| 67 |
+
"rules": source["rules"],
|
| 68 |
+
"source": source,
|
| 69 |
+
"git_object_ids": git_objects,
|
| 70 |
+
"git_object_set_sha256": sha256_bytes(compact_json_bytes(git_objects)),
|
| 71 |
+
"trackio": trackio_scan,
|
| 72 |
+
"image_contract_sha256": sha256_file(args.image_contract),
|
| 73 |
+
"image": image,
|
| 74 |
+
"private_input_manifest_sha256": sha256_file(args.package_manifest),
|
| 75 |
+
"private_input_payload_root_sha256": package_manifest["payload_root_sha256"],
|
| 76 |
+
"intended_remote_map_sha256": sha256_file(args.intended_remote_map),
|
| 77 |
+
"complete": not blockers,
|
| 78 |
+
"blockers": blockers,
|
| 79 |
+
"exit_code": 0 if not blockers else 2,
|
| 80 |
+
}
|
| 81 |
+
atomic_write_json(args.output, receipt)
|
| 82 |
+
print(json.dumps({"complete": receipt["complete"], "blockers": blockers}))
|
| 83 |
+
return receipt["exit_code"]
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
if __name__ == "__main__":
|
| 87 |
+
raise SystemExit(main())
|
scripts/build_registered_data.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build sealed MNIST mask banks or registered C5 long-path shards."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.data import (
|
| 14 |
+
build_mnist_mask_bank,
|
| 15 |
+
generate_c5_size,
|
| 16 |
+
write_mnist_mask_bank,
|
| 17 |
+
)
|
| 18 |
+
from repro_control.hashing import atomic_write_json
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def main() -> int:
|
| 22 |
+
parser = argparse.ArgumentParser()
|
| 23 |
+
sub = parser.add_subparsers(dest="command", required=True)
|
| 24 |
+
masks = sub.add_parser("mnist-masks")
|
| 25 |
+
masks.add_argument("--dataset-revision", required=True)
|
| 26 |
+
masks.add_argument("--example-ids-json", type=Path, required=True)
|
| 27 |
+
masks.add_argument("--output", type=Path, required=True)
|
| 28 |
+
c5 = sub.add_parser("c5")
|
| 29 |
+
c5.add_argument("--size", type=int, required=True)
|
| 30 |
+
c5.add_argument("--training-identities-json", type=Path, required=True)
|
| 31 |
+
c5.add_argument("--examples", type=int, default=1000)
|
| 32 |
+
c5.add_argument("--output-npz", type=Path, required=True)
|
| 33 |
+
c5.add_argument("--output-manifest", type=Path, required=True)
|
| 34 |
+
args = parser.parse_args()
|
| 35 |
+
if args.command == "mnist-masks":
|
| 36 |
+
ids = json.loads(args.example_ids_json.read_text())
|
| 37 |
+
bank = build_mnist_mask_bank(args.dataset_revision, ids)
|
| 38 |
+
write_mnist_mask_bank(args.output, bank)
|
| 39 |
+
print(bank["bank_sha256"])
|
| 40 |
+
return 0
|
| 41 |
+
import numpy as np
|
| 42 |
+
|
| 43 |
+
identities = frozenset(json.loads(args.training_identities_json.read_text()))
|
| 44 |
+
arrays, manifest = generate_c5_size(
|
| 45 |
+
args.size,
|
| 46 |
+
training_identities=identities,
|
| 47 |
+
examples=args.examples,
|
| 48 |
+
)
|
| 49 |
+
args.output_npz.parent.mkdir(parents=True, exist_ok=True)
|
| 50 |
+
np.savez_compressed(args.output_npz, **arrays)
|
| 51 |
+
atomic_write_json(args.output_manifest, manifest)
|
| 52 |
+
print(manifest["identity_root_sha256"])
|
| 53 |
+
return 0
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
if __name__ == "__main__":
|
| 57 |
+
raise SystemExit(main())
|
scripts/evaluate_repro.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run one fused verdict evaluator and seal its complete result units."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.archives import safe_extract_tar_gz
|
| 14 |
+
from repro_control.artifacts import finalize_attempt, verify_read_back
|
| 15 |
+
from repro_control.hashing import atomic_write_json, sha256_file
|
| 16 |
+
from repro_control.heartbeat import heartbeat, marker
|
| 17 |
+
from repro_control.runtime import (
|
| 18 |
+
assert_evaluation_isolated,
|
| 19 |
+
configure_scientific_runtime,
|
| 20 |
+
load_job_manifest,
|
| 21 |
+
require_control_freeze,
|
| 22 |
+
verify_science_spec,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def main() -> int:
|
| 27 |
+
parser = argparse.ArgumentParser()
|
| 28 |
+
parser.add_argument("--science-spec", type=Path, required=True)
|
| 29 |
+
parser.add_argument("--job-manifest", type=Path, required=True)
|
| 30 |
+
parser.add_argument("--control-dir", type=Path, required=True)
|
| 31 |
+
parser.add_argument("--freeze-sha256", required=True)
|
| 32 |
+
parser.add_argument("--unit-manifest", type=Path)
|
| 33 |
+
parser.add_argument("--output-dir", type=Path)
|
| 34 |
+
parser.add_argument("--eval-archive", type=Path)
|
| 35 |
+
parser.add_argument("--eval-destination", type=Path, default=Path("/data/eval"))
|
| 36 |
+
parser.add_argument("--imports-archive", type=Path)
|
| 37 |
+
parser.add_argument("--imports-destination", type=Path, default=Path("/imports"))
|
| 38 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 39 |
+
args = parser.parse_args()
|
| 40 |
+
spec = verify_science_spec(args.science_spec)
|
| 41 |
+
manifest = load_job_manifest(args.job_manifest, freeze_sha256=args.freeze_sha256)
|
| 42 |
+
assert_evaluation_isolated(manifest)
|
| 43 |
+
if manifest["logical_id"] not in spec["evaluators"]:
|
| 44 |
+
raise SystemExit("unregistered evaluator logical identity")
|
| 45 |
+
require_control_freeze(
|
| 46 |
+
args.control_dir,
|
| 47 |
+
args.freeze_sha256,
|
| 48 |
+
manifest["science_spec_sha256"],
|
| 49 |
+
)
|
| 50 |
+
configure_scientific_runtime()
|
| 51 |
+
if args.dry_run:
|
| 52 |
+
print(
|
| 53 |
+
json.dumps(
|
| 54 |
+
{
|
| 55 |
+
"contract_verified": True,
|
| 56 |
+
"logical_id": manifest["logical_id"],
|
| 57 |
+
"outcomes": {},
|
| 58 |
+
}
|
| 59 |
+
)
|
| 60 |
+
)
|
| 61 |
+
return 0
|
| 62 |
+
if args.unit_manifest is None or args.output_dir is None:
|
| 63 |
+
raise SystemExit("--unit-manifest and --output-dir are required for execution")
|
| 64 |
+
if sha256_file(args.unit_manifest) != manifest["hashes"]["config"]:
|
| 65 |
+
raise SystemExit("evaluator unit manifest hash mismatch")
|
| 66 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 67 |
+
if args.eval_archive:
|
| 68 |
+
safe_extract_tar_gz(args.eval_archive, args.eval_destination)
|
| 69 |
+
if args.imports_archive:
|
| 70 |
+
safe_extract_tar_gz(args.imports_archive, args.imports_destination)
|
| 71 |
+
marker("GPU_READY", manifest["logical_id"])
|
| 72 |
+
from repro_control.evaluation import evaluate_units
|
| 73 |
+
|
| 74 |
+
with heartbeat(f"SCIENTIFIC_EVAL:{manifest['logical_id']}"):
|
| 75 |
+
evaluate_units(args.unit_manifest, args.output_dir / "results.json")
|
| 76 |
+
atomic_write_json(
|
| 77 |
+
args.output_dir / "evaluation-receipt.json",
|
| 78 |
+
{
|
| 79 |
+
"format": 1,
|
| 80 |
+
"logical_id": manifest["logical_id"],
|
| 81 |
+
"attempt_id": manifest["attempt_id"],
|
| 82 |
+
"unit_manifest_sha256": sha256_file(args.unit_manifest),
|
| 83 |
+
"results_sha256": sha256_file(args.output_dir / "results.json"),
|
| 84 |
+
"outcomes": {},
|
| 85 |
+
},
|
| 86 |
+
)
|
| 87 |
+
finalize_attempt(
|
| 88 |
+
args.output_dir,
|
| 89 |
+
logical_id=manifest["logical_id"],
|
| 90 |
+
attempt_id=manifest["attempt_id"],
|
| 91 |
+
expected_outputs=manifest["expected_outputs"],
|
| 92 |
+
)
|
| 93 |
+
verify_read_back(
|
| 94 |
+
args.output_dir,
|
| 95 |
+
logical_id=manifest["logical_id"],
|
| 96 |
+
attempt_id=manifest["attempt_id"],
|
| 97 |
+
)
|
| 98 |
+
marker("DONE", f"{manifest['logical_id']} {manifest['attempt_id']}")
|
| 99 |
+
return 0
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
if __name__ == "__main__":
|
| 103 |
+
raise SystemExit(main())
|
scripts/freeze_science_spec.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Materialize exact registered configs and propagate the immutable science-spec hash."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
+
import re
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.configs import config_sha256, registered_config_for
|
| 14 |
+
from repro_control.hashing import atomic_write_json, sha256_file
|
| 15 |
+
|
| 16 |
+
SUDOKU_REVISION = "4d5aa527a9fb9aacca0b0d5b8b77d569fa9afcaa"
|
| 17 |
+
TRACKIO_WHEEL_SHA256 = "277340507ac46c02c06900c1d680129bdb528223c8110b0b6bc9326bb9f0891d"
|
| 18 |
+
BASE_IMAGE = (
|
| 19 |
+
"python:3.12-slim@"
|
| 20 |
+
"sha256:cab2dbf575e971934a81e4622f5aba17aa7929719bd7e31033a3a83b97fd0464"
|
| 21 |
+
)
|
| 22 |
+
IMAGE_SPACE_ID = "Mindcraft/sheaf-admm-icml2026-executor"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _replace_spec_hash(path: Path, old: str, new: str) -> None:
|
| 26 |
+
text = path.read_text()
|
| 27 |
+
if old not in text:
|
| 28 |
+
raise RuntimeError(f"old science-spec hash not present in {path}")
|
| 29 |
+
path.write_text(text.replace(old, new))
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def main() -> int:
|
| 33 |
+
root = Path(__file__).resolve().parents[1]
|
| 34 |
+
spec_path = root / "SCIENCE-SPEC.yaml"
|
| 35 |
+
old_hash = sha256_file(spec_path)
|
| 36 |
+
spec = json.loads(spec_path.read_text())
|
| 37 |
+
if spec.get("outcomes") != {} or len(spec.get("run_identities", [])) != 15:
|
| 38 |
+
raise RuntimeError("science spec is not in the pre-outcome fifteen-run state")
|
| 39 |
+
spec["authorities"].update(
|
| 40 |
+
{
|
| 41 |
+
"sudoku_dataset_revision": SUDOKU_REVISION,
|
| 42 |
+
"dependency_lock_sha256": sha256_file(root / "uv.lock"),
|
| 43 |
+
"trackio_wheel_sha256": TRACKIO_WHEEL_SHA256,
|
| 44 |
+
"base_image": BASE_IMAGE,
|
| 45 |
+
"execution_image_space": IMAGE_SPACE_ID,
|
| 46 |
+
}
|
| 47 |
+
)
|
| 48 |
+
spec["data_rules"]["sudoku"]["revision"] = SUDOKU_REVISION
|
| 49 |
+
registered = {}
|
| 50 |
+
config_root = root / "control" / "registered-configs"
|
| 51 |
+
for logical_id in spec["run_identities"]:
|
| 52 |
+
config = registered_config_for(root, logical_id)
|
| 53 |
+
digest = config_sha256(config)
|
| 54 |
+
atomic_write_json(config_root / f"{logical_id}.json", config)
|
| 55 |
+
if sha256_file(config_root / f"{logical_id}.json") == digest:
|
| 56 |
+
raise RuntimeError("file-byte hash must remain distinct from canonical config hash")
|
| 57 |
+
registered[logical_id] = {
|
| 58 |
+
"canonical_sha256": digest,
|
| 59 |
+
"file": f"control/registered-configs/{logical_id}.json",
|
| 60 |
+
"config": config,
|
| 61 |
+
}
|
| 62 |
+
spec["registered_configs"] = registered
|
| 63 |
+
atomic_write_json(spec_path, spec)
|
| 64 |
+
new_hash = sha256_file(spec_path)
|
| 65 |
+
if new_hash == old_hash:
|
| 66 |
+
raise RuntimeError("science spec did not change")
|
| 67 |
+
|
| 68 |
+
constants_path = root / "src" / "repro_control" / "constants.py"
|
| 69 |
+
constants = constants_path.read_text()
|
| 70 |
+
constants = re.sub(
|
| 71 |
+
r'SCIENCE_SPEC_SHA256 = "[0-9a-f]{64}"',
|
| 72 |
+
f'SCIENCE_SPEC_SHA256 = "{new_hash}"',
|
| 73 |
+
constants,
|
| 74 |
+
count=1,
|
| 75 |
+
)
|
| 76 |
+
constants_path.write_text(constants)
|
| 77 |
+
_replace_spec_hash(root / "Dockerfile", old_hash, new_hash)
|
| 78 |
+
|
| 79 |
+
image_contract_path = root / "IMAGE-SOURCE-CONTRACT.json"
|
| 80 |
+
image_contract = json.loads(image_contract_path.read_text())
|
| 81 |
+
image_contract.update(
|
| 82 |
+
base_image_digest=BASE_IMAGE.rsplit("@", 1)[1],
|
| 83 |
+
science_spec_sha256=new_hash,
|
| 84 |
+
)
|
| 85 |
+
atomic_write_json(image_contract_path, image_contract)
|
| 86 |
+
|
| 87 |
+
freeze_path = root / "control" / "SCIENCE-FREEZE.template.yaml"
|
| 88 |
+
freeze = json.loads(freeze_path.read_text())
|
| 89 |
+
freeze["science_spec_sha256"] = new_hash
|
| 90 |
+
atomic_write_json(freeze_path, freeze)
|
| 91 |
+
|
| 92 |
+
inventory_path = root / "PUBLICATION-INVENTORY.json"
|
| 93 |
+
inventory = json.loads(inventory_path.read_text())
|
| 94 |
+
image_rows = [
|
| 95 |
+
row for row in inventory["entries"] if row["asset_or_surface_type"] == "execution_image"
|
| 96 |
+
]
|
| 97 |
+
if len(image_rows) != 1:
|
| 98 |
+
raise RuntimeError("publication inventory must have exactly one execution image row")
|
| 99 |
+
image_rows[0]["full_url_or_expected_repo"] = (
|
| 100 |
+
f"https://huggingface.co/spaces/{IMAGE_SPACE_ID}"
|
| 101 |
+
)
|
| 102 |
+
atomic_write_json(inventory_path, inventory)
|
| 103 |
+
|
| 104 |
+
status_path = root / "LOCAL-IMPLEMENTATION-STATUS.md"
|
| 105 |
+
if status_path.exists() and old_hash in status_path.read_text():
|
| 106 |
+
_replace_spec_hash(status_path, old_hash, new_hash)
|
| 107 |
+
print(
|
| 108 |
+
json.dumps(
|
| 109 |
+
{
|
| 110 |
+
"old_science_spec_sha256": old_hash,
|
| 111 |
+
"science_spec_sha256": new_hash,
|
| 112 |
+
"registered_config_count": len(registered),
|
| 113 |
+
"sudoku_revision": SUDOKU_REVISION,
|
| 114 |
+
},
|
| 115 |
+
sort_keys=True,
|
| 116 |
+
)
|
| 117 |
+
)
|
| 118 |
+
return 0
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
if __name__ == "__main__":
|
| 122 |
+
raise SystemExit(main())
|
scripts/import_data.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build and seal the private train/eval/import input archives on an HF CPU Job."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import shutil
|
| 9 |
+
import sys
|
| 10 |
+
import tempfile
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 14 |
+
|
| 15 |
+
from repro_control.archives import create_deterministic_tar_gz
|
| 16 |
+
from repro_control.artifacts import finalize_attempt, verify_read_back
|
| 17 |
+
from repro_control.checkpoints import load_neutral_checkpoint
|
| 18 |
+
from repro_control.data import (
|
| 19 |
+
C5_SIZES,
|
| 20 |
+
build_mnist_mask_bank,
|
| 21 |
+
canonical_maze_identity,
|
| 22 |
+
generate_c5_size,
|
| 23 |
+
validate_smoke_fixture_registry,
|
| 24 |
+
write_mnist_mask_bank,
|
| 25 |
+
write_puzzle_split,
|
| 26 |
+
)
|
| 27 |
+
from repro_control.hashing import (
|
| 28 |
+
atomic_write_json,
|
| 29 |
+
canonical_root,
|
| 30 |
+
file_entry,
|
| 31 |
+
sha256_file,
|
| 32 |
+
)
|
| 33 |
+
from repro_control.heartbeat import heartbeat, marker
|
| 34 |
+
from repro_control.runtime import load_job_manifest, verify_science_spec
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _verify_handoff(handoff: Path, *, verify_checkpoints: bool) -> list[dict]:
|
| 38 |
+
package_manifest = json.loads((handoff / "PACKAGE-MANIFEST.json").read_text())
|
| 39 |
+
for row in package_manifest["entries"]:
|
| 40 |
+
path = handoff / row["path"]
|
| 41 |
+
if path.stat().st_size != row["bytes"] or sha256_file(path) != row["sha256"]:
|
| 42 |
+
raise SystemExit(f"handoff mismatch: {row['path']}")
|
| 43 |
+
imports = json.loads((handoff / "IMPORTS.json").read_text())
|
| 44 |
+
if len(imports) != 9:
|
| 45 |
+
raise SystemExit("IMPORTS.json must contain exactly nine rows")
|
| 46 |
+
validate_smoke_fixture_registry(handoff / "SMOKE-FIXTURES.json")
|
| 47 |
+
if verify_checkpoints:
|
| 48 |
+
for row in imports:
|
| 49 |
+
load_neutral_checkpoint(
|
| 50 |
+
handoff / "checkpoints" / row["neutral_alias"] / "checkpoint.pkl",
|
| 51 |
+
handoff / "configs" / f"{row['neutral_alias']}.json",
|
| 52 |
+
row,
|
| 53 |
+
)
|
| 54 |
+
return imports
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _tree_manifest(root: Path) -> dict:
|
| 58 |
+
entries = [
|
| 59 |
+
file_entry(path, relative_to=root)
|
| 60 |
+
for path in sorted(root.rglob("*"))
|
| 61 |
+
if path.is_file()
|
| 62 |
+
]
|
| 63 |
+
return {"format": 1, "entries": entries, "root_sha256": canonical_root(entries)}
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _copy_split(source_dataset: Path, split: str, target_dataset: Path) -> None:
|
| 67 |
+
target_dataset.mkdir(parents=True, exist_ok=True)
|
| 68 |
+
shutil.copytree(source_dataset / split, target_dataset / split)
|
| 69 |
+
for name in ("metadata.json", "config.json"):
|
| 70 |
+
source = source_dataset / name
|
| 71 |
+
if source.is_file():
|
| 72 |
+
shutil.copy2(source, target_dataset / name)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _maze_training_identities(dataset_root: Path) -> frozenset[str]:
|
| 76 |
+
import numpy as np
|
| 77 |
+
|
| 78 |
+
inputs = np.load(dataset_root / "train/all__inputs.npy", mmap_mode="r")
|
| 79 |
+
labels = np.load(dataset_root / "train/all__labels.npy", mmap_mode="r")
|
| 80 |
+
return frozenset(
|
| 81 |
+
canonical_maze_identity(input_row, label_row)
|
| 82 |
+
for input_row, label_row in zip(inputs, labels, strict=True)
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _build_registered_inputs(
|
| 87 |
+
handoff: Path,
|
| 88 |
+
workspace: Path,
|
| 89 |
+
*,
|
| 90 |
+
sudoku_revision: str,
|
| 91 |
+
) -> dict:
|
| 92 |
+
from sheaf_admm.data.build_maze import MazeConfig
|
| 93 |
+
from sheaf_admm.data.build_maze import build as build_maze
|
| 94 |
+
from sheaf_admm.data.build_mnist import MNISTConfig
|
| 95 |
+
from sheaf_admm.data.build_mnist import build as build_mnist
|
| 96 |
+
from sheaf_admm.data.build_sudoku import SudokuConfig
|
| 97 |
+
from sheaf_admm.data.build_sudoku import build as build_sudoku
|
| 98 |
+
|
| 99 |
+
build_root = workspace / "build"
|
| 100 |
+
maze_root = build_root / "maze_std3_19px_10k"
|
| 101 |
+
mnist_root = build_root / "mnist"
|
| 102 |
+
sudoku_root = build_root / "sudoku_easy"
|
| 103 |
+
|
| 104 |
+
marker("HEARTBEAT", "CPU_IMPORT:build-maze")
|
| 105 |
+
build_maze(
|
| 106 |
+
MazeConfig(
|
| 107 |
+
height=19,
|
| 108 |
+
width=19,
|
| 109 |
+
train_size=10_000,
|
| 110 |
+
test_size=1_000,
|
| 111 |
+
min_path_length=18,
|
| 112 |
+
train_augment=True,
|
| 113 |
+
test_augment=False,
|
| 114 |
+
seed=0,
|
| 115 |
+
output_dir=maze_root,
|
| 116 |
+
),
|
| 117 |
+
ood_sizes=False,
|
| 118 |
+
)
|
| 119 |
+
marker("HEARTBEAT", "CPU_IMPORT:build-mnist")
|
| 120 |
+
build_mnist(
|
| 121 |
+
MNISTConfig(
|
| 122 |
+
padding=0,
|
| 123 |
+
seed=0,
|
| 124 |
+
normalize=True,
|
| 125 |
+
gen_robustness=False,
|
| 126 |
+
output_dir=mnist_root,
|
| 127 |
+
)
|
| 128 |
+
)
|
| 129 |
+
marker("HEARTBEAT", "CPU_IMPORT:build-sudoku")
|
| 130 |
+
build_sudoku(
|
| 131 |
+
SudokuConfig(
|
| 132 |
+
dataset_revision=sudoku_revision,
|
| 133 |
+
seed=0,
|
| 134 |
+
train_size=50_000,
|
| 135 |
+
test_size=2_000,
|
| 136 |
+
difficulty_max=2.0,
|
| 137 |
+
train_augment=True,
|
| 138 |
+
test_augment=False,
|
| 139 |
+
output_dir=sudoku_root,
|
| 140 |
+
)
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
train_root = workspace / "train"
|
| 144 |
+
eval_root = workspace / "eval"
|
| 145 |
+
imports_root = workspace / "imports"
|
| 146 |
+
_copy_split(maze_root, "train", train_root / "maze_std3_19px_10k")
|
| 147 |
+
_copy_split(mnist_root, "train", train_root / "mnist")
|
| 148 |
+
_copy_split(sudoku_root, "train", train_root / "sudoku_easy")
|
| 149 |
+
_copy_split(maze_root, "test", eval_root / "maze_std3_19px_10k")
|
| 150 |
+
_copy_split(mnist_root, "test", eval_root / "mnist")
|
| 151 |
+
_copy_split(sudoku_root, "test_hard", eval_root / "sudoku_easy")
|
| 152 |
+
|
| 153 |
+
test_count = 10_000
|
| 154 |
+
mask_bank = build_mnist_mask_bank(
|
| 155 |
+
f"ml-datasets-0.2.1:{sha256_file(mnist_root / 'test/images.npy')}",
|
| 156 |
+
range(test_count),
|
| 157 |
+
)
|
| 158 |
+
write_mnist_mask_bank(eval_root / "mnist/drop30-mask-bank.json", mask_bank)
|
| 159 |
+
|
| 160 |
+
marker("HEARTBEAT", "CPU_IMPORT:build-c5")
|
| 161 |
+
training_identities = _maze_training_identities(maze_root)
|
| 162 |
+
c5_receipts = {}
|
| 163 |
+
for size in C5_SIZES:
|
| 164 |
+
arrays, c5_manifest = generate_c5_size(
|
| 165 |
+
size,
|
| 166 |
+
training_identities=training_identities,
|
| 167 |
+
examples=1000,
|
| 168 |
+
)
|
| 169 |
+
c5_root = eval_root / "c5" / f"{size}x{size}"
|
| 170 |
+
write_puzzle_split(
|
| 171 |
+
c5_root,
|
| 172 |
+
"test",
|
| 173 |
+
arrays["inputs"],
|
| 174 |
+
arrays["labels"],
|
| 175 |
+
height=size,
|
| 176 |
+
width=size,
|
| 177 |
+
)
|
| 178 |
+
atomic_write_json(c5_root / "generation-manifest.json", c5_manifest)
|
| 179 |
+
c5_receipts[str(size)] = c5_manifest
|
| 180 |
+
|
| 181 |
+
shutil.copytree(handoff / "checkpoints", imports_root / "checkpoints")
|
| 182 |
+
shutil.copytree(handoff / "configs", imports_root / "configs")
|
| 183 |
+
shutil.copy2(handoff / "IMPORTS.json", imports_root / "IMPORTS.json")
|
| 184 |
+
shutil.copy2(handoff / "RIGHTS.json", imports_root / "RIGHTS.json")
|
| 185 |
+
manifests = {
|
| 186 |
+
"train": _tree_manifest(train_root),
|
| 187 |
+
"eval": _tree_manifest(eval_root),
|
| 188 |
+
"imports": _tree_manifest(imports_root),
|
| 189 |
+
}
|
| 190 |
+
for name, value in manifests.items():
|
| 191 |
+
atomic_write_json(workspace / f"{name}-tree-manifest.json", value)
|
| 192 |
+
return {
|
| 193 |
+
"tree_manifests": manifests,
|
| 194 |
+
"mnist_mask_bank_sha256": mask_bank["bank_sha256"],
|
| 195 |
+
"maze_training_identity_count": len(training_identities),
|
| 196 |
+
"c5": c5_receipts,
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def main() -> int:
|
| 201 |
+
parser = argparse.ArgumentParser()
|
| 202 |
+
parser.add_argument("--science-spec", type=Path, required=True)
|
| 203 |
+
parser.add_argument("--job-manifest", type=Path, required=True)
|
| 204 |
+
parser.add_argument("--handoff", type=Path, required=True)
|
| 205 |
+
parser.add_argument("--output-dir", type=Path, required=True)
|
| 206 |
+
parser.add_argument("--sudoku-revision", required=True)
|
| 207 |
+
parser.add_argument("--verify-checkpoints", action="store_true")
|
| 208 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 209 |
+
args = parser.parse_args()
|
| 210 |
+
spec = verify_science_spec(args.science_spec)
|
| 211 |
+
manifest = load_job_manifest(args.job_manifest)
|
| 212 |
+
if manifest["job_class"] != "CPU_IMPORT":
|
| 213 |
+
raise SystemExit("CPU import entrypoint requires a CPU_IMPORT manifest")
|
| 214 |
+
imports = _verify_handoff(args.handoff, verify_checkpoints=args.verify_checkpoints)
|
| 215 |
+
if args.dry_run:
|
| 216 |
+
print(
|
| 217 |
+
json.dumps(
|
| 218 |
+
{
|
| 219 |
+
"contract_verified": True,
|
| 220 |
+
"verified_import_rows": len(imports),
|
| 221 |
+
"outcomes": {},
|
| 222 |
+
}
|
| 223 |
+
)
|
| 224 |
+
)
|
| 225 |
+
return 0
|
| 226 |
+
if args.sudoku_revision != spec["data_rules"]["sudoku"]["revision"]:
|
| 227 |
+
raise SystemExit("Sudoku revision differs from the frozen science spec")
|
| 228 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 229 |
+
marker("CPU_READY", "CPU_IMPORT")
|
| 230 |
+
with (
|
| 231 |
+
heartbeat("CPU_IMPORT"),
|
| 232 |
+
tempfile.TemporaryDirectory(prefix="sheaf-import-") as temporary,
|
| 233 |
+
):
|
| 234 |
+
workspace = Path(temporary)
|
| 235 |
+
build_receipt = _build_registered_inputs(
|
| 236 |
+
args.handoff,
|
| 237 |
+
workspace,
|
| 238 |
+
sudoku_revision=args.sudoku_revision,
|
| 239 |
+
)
|
| 240 |
+
archive_hashes = {}
|
| 241 |
+
for name in ("train", "eval", "imports"):
|
| 242 |
+
archive = args.output_dir / f"{name}-data.tar.gz"
|
| 243 |
+
archive_hashes[name] = create_deterministic_tar_gz(workspace / name, archive)
|
| 244 |
+
shutil.copy2(
|
| 245 |
+
workspace / f"{name}-tree-manifest.json",
|
| 246 |
+
args.output_dir / f"{name}-tree-manifest.json",
|
| 247 |
+
)
|
| 248 |
+
receipt = {
|
| 249 |
+
"format": 1,
|
| 250 |
+
"logical_id": manifest["logical_id"],
|
| 251 |
+
"attempt_id": manifest["attempt_id"],
|
| 252 |
+
"verified_import_rows": len(imports),
|
| 253 |
+
"sudoku_revision": args.sudoku_revision,
|
| 254 |
+
"archives": archive_hashes,
|
| 255 |
+
"build": build_receipt,
|
| 256 |
+
"outcomes": {},
|
| 257 |
+
}
|
| 258 |
+
atomic_write_json(args.output_dir / "import-receipt.json", receipt)
|
| 259 |
+
finalize_attempt(
|
| 260 |
+
args.output_dir,
|
| 261 |
+
logical_id=manifest["logical_id"],
|
| 262 |
+
attempt_id=manifest["attempt_id"],
|
| 263 |
+
expected_outputs=manifest["expected_outputs"],
|
| 264 |
+
)
|
| 265 |
+
verify_read_back(
|
| 266 |
+
args.output_dir,
|
| 267 |
+
logical_id=manifest["logical_id"],
|
| 268 |
+
attempt_id=manifest["attempt_id"],
|
| 269 |
+
)
|
| 270 |
+
marker("DONE", f"{manifest['logical_id']} {manifest['attempt_id']}")
|
| 271 |
+
return 0
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
if __name__ == "__main__":
|
| 275 |
+
raise SystemExit(main())
|
scripts/launch_job.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Prepare, submit, reconcile, poll, or cancel one durable direct HF Job attempt."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from datetime import UTC, datetime
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 13 |
+
|
| 14 |
+
from repro_control.hf_provider import HFJobsProvider
|
| 15 |
+
from repro_control.launcher import (
|
| 16 |
+
AmbiguousSubmission,
|
| 17 |
+
BudgetSnapshot,
|
| 18 |
+
DirectLauncher,
|
| 19 |
+
LaunchError,
|
| 20 |
+
LauncherState,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _launcher(args) -> DirectLauncher:
|
| 25 |
+
provider = HFJobsProvider(namespace=args.namespace) if args.action != "prepare" else None
|
| 26 |
+
return DirectLauncher(
|
| 27 |
+
args.state,
|
| 28 |
+
provider=provider,
|
| 29 |
+
enable_submit=args.enable_submit,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def main() -> int:
|
| 34 |
+
parser = argparse.ArgumentParser()
|
| 35 |
+
parser.add_argument("--state", type=Path, required=True)
|
| 36 |
+
parser.add_argument("--namespace", default="Mindcraft")
|
| 37 |
+
parser.add_argument(
|
| 38 |
+
"--enable-submit",
|
| 39 |
+
action="store_true",
|
| 40 |
+
help="Required for submit/cancel; omitted by default to keep the CLI non-mutating.",
|
| 41 |
+
)
|
| 42 |
+
sub = parser.add_subparsers(dest="action", required=True)
|
| 43 |
+
prepare = sub.add_parser("prepare")
|
| 44 |
+
prepare.add_argument("--request", type=Path, required=True)
|
| 45 |
+
prepare.add_argument("--budget-snapshot", type=Path, required=True)
|
| 46 |
+
prepare.add_argument("--guarded-critical-path-seconds", type=int, required=True)
|
| 47 |
+
submit = sub.add_parser("submit")
|
| 48 |
+
submit.add_argument("--attempt-id", required=True)
|
| 49 |
+
reconcile = sub.add_parser("reconcile")
|
| 50 |
+
reconcile.add_argument("--attempt-id", required=True)
|
| 51 |
+
poll = sub.add_parser("poll")
|
| 52 |
+
poll.add_argument("--attempt-id", required=True)
|
| 53 |
+
cancel = sub.add_parser("cancel")
|
| 54 |
+
cancel.add_argument("--attempt-id", required=True)
|
| 55 |
+
sub.add_parser("show")
|
| 56 |
+
args = parser.parse_args()
|
| 57 |
+
launcher = _launcher(args)
|
| 58 |
+
try:
|
| 59 |
+
if args.action == "prepare":
|
| 60 |
+
request = json.loads(args.request.read_text())
|
| 61 |
+
budget = BudgetSnapshot(**json.loads(args.budget_snapshot.read_text()))
|
| 62 |
+
row = launcher.prepare(
|
| 63 |
+
request,
|
| 64 |
+
budget=budget,
|
| 65 |
+
now=datetime.now(UTC),
|
| 66 |
+
guarded_critical_path_seconds=args.guarded_critical_path_seconds,
|
| 67 |
+
)
|
| 68 |
+
elif args.action == "submit":
|
| 69 |
+
if not args.enable_submit:
|
| 70 |
+
raise LaunchError("submit requires --enable-submit")
|
| 71 |
+
row = launcher.submit_prepared(args.attempt_id)
|
| 72 |
+
elif args.action == "reconcile":
|
| 73 |
+
row = launcher.reconcile_ambiguous(args.attempt_id)
|
| 74 |
+
elif args.action == "poll":
|
| 75 |
+
row = launcher.poll(args.attempt_id, now=datetime.now(UTC))
|
| 76 |
+
elif args.action == "cancel":
|
| 77 |
+
if not args.enable_submit:
|
| 78 |
+
raise LaunchError("cancel requires --enable-submit")
|
| 79 |
+
launcher.cancel(args.attempt_id)
|
| 80 |
+
row = next(
|
| 81 |
+
item
|
| 82 |
+
for item in LauncherState.load(args.state).attempts
|
| 83 |
+
if item["attempt_id"] == args.attempt_id
|
| 84 |
+
)
|
| 85 |
+
else:
|
| 86 |
+
print(
|
| 87 |
+
json.dumps(
|
| 88 |
+
{
|
| 89 |
+
"format": 1,
|
| 90 |
+
"attempts": LauncherState.load(args.state).attempts,
|
| 91 |
+
},
|
| 92 |
+
sort_keys=True,
|
| 93 |
+
)
|
| 94 |
+
)
|
| 95 |
+
return 0
|
| 96 |
+
except AmbiguousSubmission as exc:
|
| 97 |
+
print(str(exc), file=sys.stderr)
|
| 98 |
+
return 3
|
| 99 |
+
except (LaunchError, ValueError, KeyError) as exc:
|
| 100 |
+
print(str(exc), file=sys.stderr)
|
| 101 |
+
return 2
|
| 102 |
+
print(json.dumps(row, sort_keys=True))
|
| 103 |
+
return 0
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
raise SystemExit(main())
|
scripts/reduce_results.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Apply one exhaustive registered reducer to accepted complete numeric inputs."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import dataclasses
|
| 8 |
+
import json
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 13 |
+
|
| 14 |
+
from repro_control.reducers import (
|
| 15 |
+
reduce_c1,
|
| 16 |
+
reduce_c2,
|
| 17 |
+
reduce_c3,
|
| 18 |
+
reduce_c4_maze,
|
| 19 |
+
reduce_c4_sudoku,
|
| 20 |
+
reduce_c5,
|
| 21 |
+
)
|
| 22 |
+
from repro_control.hashing import atomic_write_json
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def plain(value):
|
| 26 |
+
if dataclasses.is_dataclass(value):
|
| 27 |
+
return {field.name: plain(getattr(value, field.name)) for field in dataclasses.fields(value)}
|
| 28 |
+
if isinstance(value, dict):
|
| 29 |
+
return {str(key): plain(item) for key, item in value.items()}
|
| 30 |
+
if isinstance(value, (list, tuple)):
|
| 31 |
+
return [plain(item) for item in value]
|
| 32 |
+
return value
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def main() -> int:
|
| 36 |
+
parser = argparse.ArgumentParser()
|
| 37 |
+
parser.add_argument("--claim", choices=("c1", "c2", "c3", "c4-sudoku", "c4-maze", "c5"), required=True)
|
| 38 |
+
parser.add_argument("--input", type=Path, required=True)
|
| 39 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 40 |
+
args = parser.parse_args()
|
| 41 |
+
value = json.loads(args.input.read_text())
|
| 42 |
+
reducers = {
|
| 43 |
+
"c1": reduce_c1,
|
| 44 |
+
"c2": reduce_c2,
|
| 45 |
+
"c3": reduce_c3,
|
| 46 |
+
"c4-sudoku": reduce_c4_sudoku,
|
| 47 |
+
"c4-maze": reduce_c4_maze,
|
| 48 |
+
"c5": reduce_c5,
|
| 49 |
+
}
|
| 50 |
+
result = reducers[args.claim](**value)
|
| 51 |
+
atomic_write_json(args.output, plain(result))
|
| 52 |
+
return 0
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
if __name__ == "__main__":
|
| 56 |
+
raise SystemExit(main())
|
scripts/replace_scaffold_cells.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Replace the two official pinned scaffold cells in place."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 11 |
+
|
| 12 |
+
from repro_control.scaffold import replace_scaffold_file
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def main() -> int:
|
| 16 |
+
parser = argparse.ArgumentParser()
|
| 17 |
+
parser.add_argument(
|
| 18 |
+
"--scaffold-export",
|
| 19 |
+
type=Path,
|
| 20 |
+
required=True,
|
| 21 |
+
help="Official Executive-summary page.md or a synthetic JSON export",
|
| 22 |
+
)
|
| 23 |
+
parser.add_argument("--summary", type=Path, required=True)
|
| 24 |
+
parser.add_argument("--poster", type=Path, required=True)
|
| 25 |
+
parser.add_argument("--output", type=Path)
|
| 26 |
+
args = parser.parse_args()
|
| 27 |
+
replace_scaffold_file(
|
| 28 |
+
args.scaffold_export,
|
| 29 |
+
summary_path=args.summary,
|
| 30 |
+
poster_path=args.poster,
|
| 31 |
+
output_path=args.output,
|
| 32 |
+
)
|
| 33 |
+
return 0
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
if __name__ == "__main__":
|
| 37 |
+
raise SystemExit(main())
|
scripts/scan_privacy.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Write a deterministic privacy receipt for explicitly selected local roots."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.privacy import scan_paths, write_receipt
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def main() -> int:
|
| 17 |
+
parser = argparse.ArgumentParser()
|
| 18 |
+
parser.add_argument("--checkout", type=Path, required=True)
|
| 19 |
+
parser.add_argument("--root", type=Path, action="append", required=True)
|
| 20 |
+
parser.add_argument("--intended-remote-map", type=Path, required=True)
|
| 21 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 22 |
+
args = parser.parse_args()
|
| 23 |
+
mapping = json.loads(args.intended_remote_map.read_text())
|
| 24 |
+
receipt = scan_paths(args.root, relative_to=args.checkout, intended_remote_map=mapping)
|
| 25 |
+
write_receipt(args.output, receipt, require_clean=False)
|
| 26 |
+
print(json.dumps({"exit_code": receipt["exit_code"], "findings": len(receipt["findings"])}))
|
| 27 |
+
return receipt["exit_code"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
if __name__ == "__main__":
|
| 31 |
+
raise SystemExit(main())
|
scripts/smoke_repro.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run a real, verdict-isolated admission workload and seal its receipt."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 12 |
+
|
| 13 |
+
from repro_control.artifacts import finalize_attempt, verify_read_back
|
| 14 |
+
from repro_control.hashing import atomic_write_json
|
| 15 |
+
from repro_control.heartbeat import heartbeat, marker
|
| 16 |
+
from repro_control.runtime import assert_smoke_isolated, load_job_manifest, verify_science_spec
|
| 17 |
+
from repro_control.smoke_runtime import (
|
| 18 |
+
fixture_hashes,
|
| 19 |
+
orchestrate_family,
|
| 20 |
+
run_c5_probe_child,
|
| 21 |
+
run_family_child,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def main() -> int:
|
| 26 |
+
parser = argparse.ArgumentParser()
|
| 27 |
+
parser.add_argument("--science-spec", type=Path)
|
| 28 |
+
parser.add_argument("--job-manifest", type=Path)
|
| 29 |
+
parser.add_argument("--family", choices=("sudoku", "maze-c5", "mnist"))
|
| 30 |
+
parser.add_argument("--project-root", type=Path, default=Path(__file__).resolve().parents[1])
|
| 31 |
+
parser.add_argument("--handoff", type=Path)
|
| 32 |
+
parser.add_argument("--output-dir", type=Path)
|
| 33 |
+
parser.add_argument("--steps-warmup", type=int, default=5)
|
| 34 |
+
parser.add_argument("--steps-measured", type=int, default=20)
|
| 35 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 36 |
+
parser.add_argument("--allow-cpu-preflight", action="store_true")
|
| 37 |
+
parser.add_argument("--internal-family-child", choices=("sudoku", "maze-c5", "mnist"))
|
| 38 |
+
parser.add_argument("--internal-c5-probe-child", action="store_true")
|
| 39 |
+
parser.add_argument("--size", type=int)
|
| 40 |
+
parser.add_argument("--batch-size", type=int)
|
| 41 |
+
parser.add_argument("--timing", action="store_true")
|
| 42 |
+
args = parser.parse_args()
|
| 43 |
+
|
| 44 |
+
if args.internal_family_child:
|
| 45 |
+
value = run_family_child(
|
| 46 |
+
args.project_root,
|
| 47 |
+
args.handoff,
|
| 48 |
+
args.internal_family_child,
|
| 49 |
+
warmup_steps=args.steps_warmup,
|
| 50 |
+
measured_steps=args.steps_measured,
|
| 51 |
+
allow_cpu=args.allow_cpu_preflight,
|
| 52 |
+
)
|
| 53 |
+
print(json.dumps(value))
|
| 54 |
+
return 0
|
| 55 |
+
if args.internal_c5_probe_child:
|
| 56 |
+
value = run_c5_probe_child(
|
| 57 |
+
args.handoff,
|
| 58 |
+
size=args.size,
|
| 59 |
+
batch_size=args.batch_size,
|
| 60 |
+
timing=args.timing,
|
| 61 |
+
allow_cpu=args.allow_cpu_preflight,
|
| 62 |
+
)
|
| 63 |
+
print(json.dumps(value))
|
| 64 |
+
return 0
|
| 65 |
+
|
| 66 |
+
if not all((args.science_spec, args.job_manifest, args.family, args.handoff)):
|
| 67 |
+
parser.error("top-level smoke requires spec, manifest, family, and handoff")
|
| 68 |
+
verify_science_spec(args.science_spec)
|
| 69 |
+
manifest = load_job_manifest(args.job_manifest)
|
| 70 |
+
assert_smoke_isolated(manifest)
|
| 71 |
+
if (args.steps_warmup, args.steps_measured) != (5, 20):
|
| 72 |
+
raise SystemExit("registered smoke requires 5 warm-up and 20 measured steps")
|
| 73 |
+
if args.dry_run:
|
| 74 |
+
print(
|
| 75 |
+
json.dumps(
|
| 76 |
+
{
|
| 77 |
+
"contract_verified": True,
|
| 78 |
+
"family": args.family,
|
| 79 |
+
"verdict_metrics_emitted": False,
|
| 80 |
+
"outcomes": {},
|
| 81 |
+
}
|
| 82 |
+
)
|
| 83 |
+
)
|
| 84 |
+
return 0
|
| 85 |
+
if args.output_dir is None:
|
| 86 |
+
raise SystemExit("--output-dir is required for execution")
|
| 87 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 88 |
+
marker("CPU_PREFLIGHT_READY" if args.allow_cpu_preflight else "GPU_READY", args.family)
|
| 89 |
+
with heartbeat(f"GPU_SMOKE:{args.family}"):
|
| 90 |
+
result = orchestrate_family(
|
| 91 |
+
Path(__file__).resolve(),
|
| 92 |
+
args.project_root,
|
| 93 |
+
args.handoff,
|
| 94 |
+
args.family,
|
| 95 |
+
warmup_steps=args.steps_warmup,
|
| 96 |
+
measured_steps=args.steps_measured,
|
| 97 |
+
allow_cpu=args.allow_cpu_preflight,
|
| 98 |
+
)
|
| 99 |
+
receipt = {
|
| 100 |
+
"format": 1,
|
| 101 |
+
"logical_id": manifest["logical_id"],
|
| 102 |
+
"attempt_id": manifest["attempt_id"],
|
| 103 |
+
"family": args.family,
|
| 104 |
+
"fixture_hashes": fixture_hashes(args.handoff),
|
| 105 |
+
"warmup_steps": 5,
|
| 106 |
+
"measured_steps": 20,
|
| 107 |
+
"result": result,
|
| 108 |
+
"verdict_metrics_emitted": False,
|
| 109 |
+
"outcomes": {},
|
| 110 |
+
}
|
| 111 |
+
atomic_write_json(args.output_dir / "smoke-receipt.json", receipt)
|
| 112 |
+
finalize_attempt(
|
| 113 |
+
args.output_dir,
|
| 114 |
+
logical_id=manifest["logical_id"],
|
| 115 |
+
attempt_id=manifest["attempt_id"],
|
| 116 |
+
expected_outputs=manifest["expected_outputs"],
|
| 117 |
+
)
|
| 118 |
+
verify_read_back(
|
| 119 |
+
args.output_dir,
|
| 120 |
+
logical_id=manifest["logical_id"],
|
| 121 |
+
attempt_id=manifest["attempt_id"],
|
| 122 |
+
)
|
| 123 |
+
marker("DONE", f"{manifest['logical_id']} {manifest['attempt_id']}")
|
| 124 |
+
return 0
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
if __name__ == "__main__":
|
| 128 |
+
raise SystemExit(main())
|
scripts/train.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hydra training entrypoint for Sheaf-ADMM (and the MPNN baseline).
|
| 2 |
+
|
| 3 |
+
One entrypoint for every task (maze / mnist / sudoku) and both model families
|
| 4 |
+
(``model_type=sheaf|mpnn``); the task and HPs come entirely from config.
|
| 5 |
+
|
| 6 |
+
python scripts/train.py +experiment=maze_sheaf
|
| 7 |
+
python scripts/train.py +experiment=sudoku_sheaf training.seed=123
|
| 8 |
+
python scripts/train.py +experiment=mnist_sheaf
|
| 9 |
+
|
| 10 |
+
Runs do not log by default. Set ``wandb.mode=online`` to enable Weights & Biases.
|
| 11 |
+
Importing ``sheaf_admm`` pins ``float32`` matmul precision to ``highest`` (the
|
| 12 |
+
paper setting) before any compilation.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import hydra
|
| 22 |
+
import jax
|
| 23 |
+
import numpy as np
|
| 24 |
+
import wandb
|
| 25 |
+
from hydra.core.hydra_config import HydraConfig
|
| 26 |
+
from omegaconf import DictConfig, OmegaConf
|
| 27 |
+
|
| 28 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 29 |
+
if str(PROJECT_ROOT / "src") not in sys.path:
|
| 30 |
+
sys.path.insert(0, str(PROJECT_ROOT / "src"))
|
| 31 |
+
|
| 32 |
+
import sheaf_admm as _sheaf_admm # noqa: F401,E402 (sets matmul precision on import)
|
| 33 |
+
from repro_control.checkpoints import save_checkpoint_atomic # noqa: E402
|
| 34 |
+
from repro_control.configs import ( # noqa: E402
|
| 35 |
+
config_sha256,
|
| 36 |
+
load_json_config,
|
| 37 |
+
verify_before_optimizer,
|
| 38 |
+
)
|
| 39 |
+
from repro_control.hashing import atomic_write_json # noqa: E402
|
| 40 |
+
from repro_control.interventions import assert_common_bit_identical # noqa: E402
|
| 41 |
+
from sheaf_admm.data import ImageDataset, PuzzleDataset # noqa: E402
|
| 42 |
+
from sheaf_admm.models import model_config_from_dict # noqa: E402
|
| 43 |
+
from sheaf_admm.training import ( # noqa: E402
|
| 44 |
+
build_model,
|
| 45 |
+
create_train_state,
|
| 46 |
+
evaluate,
|
| 47 |
+
make_task,
|
| 48 |
+
make_train_step,
|
| 49 |
+
sample_k,
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _puzzle_batch(batch):
|
| 54 |
+
out = {"inputs": np.asarray(batch["inputs"]), "labels": np.asarray(batch["labels"])}
|
| 55 |
+
for key in ("height", "width"):
|
| 56 |
+
if key in batch:
|
| 57 |
+
out[key] = batch[key]
|
| 58 |
+
return out
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _train_batches(cfg: DictConfig, epoch: int):
|
| 62 |
+
"""Yield ``{inputs/images, labels}`` batches for one training epoch."""
|
| 63 |
+
d = cfg.data
|
| 64 |
+
if d.loader == "puzzle":
|
| 65 |
+
ds = PuzzleDataset(d.dir, d.train_split)
|
| 66 |
+
for _set, batch in ds.iter_train_batches(
|
| 67 |
+
cfg.training.batch_size, seed=cfg.training.seed + epoch
|
| 68 |
+
):
|
| 69 |
+
yield _puzzle_batch(batch)
|
| 70 |
+
else: # image
|
| 71 |
+
ds = ImageDataset(d.dir, d.train_split)
|
| 72 |
+
for batch in ds.iter_batches(
|
| 73 |
+
cfg.training.batch_size, shuffle=True, seed=cfg.training.seed + epoch
|
| 74 |
+
):
|
| 75 |
+
yield {"images": np.asarray(batch["images"]), "labels": np.asarray(batch["labels"])}
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _val_batches(cfg: DictConfig, split: str):
|
| 79 |
+
d = cfg.data
|
| 80 |
+
if d.loader == "puzzle":
|
| 81 |
+
ds = PuzzleDataset(d.dir, split)
|
| 82 |
+
for _set, batch in ds.iter_test_batches(cfg.training.batch_size):
|
| 83 |
+
yield _puzzle_batch(batch)
|
| 84 |
+
else:
|
| 85 |
+
ds = ImageDataset(d.dir, split)
|
| 86 |
+
for batch in ds.iter_batches(cfg.training.batch_size, shuffle=False):
|
| 87 |
+
yield {"images": np.asarray(batch["images"]), "labels": np.asarray(batch["labels"])}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
@hydra.main(config_path="../configs", config_name="config", version_base=None)
|
| 91 |
+
def main(cfg: DictConfig) -> None:
|
| 92 |
+
sys.stdout.reconfigure(line_buffering=True) # flush per line so sbatch log-tailing works live
|
| 93 |
+
expected_path = os.environ.get("REPRO_EXPECTED_CONFIG_PATH", "")
|
| 94 |
+
expected_sha256 = os.environ.get("REPRO_EXPECTED_CONFIG_SHA256", "")
|
| 95 |
+
if bool(expected_path) != bool(expected_sha256):
|
| 96 |
+
raise RuntimeError("both registered config environment variables are required together")
|
| 97 |
+
if expected_path:
|
| 98 |
+
resolved = OmegaConf.to_container(cfg, resolve=True)
|
| 99 |
+
expected = load_json_config(Path(expected_path))
|
| 100 |
+
verify_before_optimizer(resolved, expected, expected_sha256=expected_sha256)
|
| 101 |
+
if config_sha256(expected) != expected_sha256:
|
| 102 |
+
raise RuntimeError("registered config file hash mismatch")
|
| 103 |
+
print(f"CONFIG_VERIFIED {expected_sha256}", flush=True)
|
| 104 |
+
print(OmegaConf.to_yaml(cfg))
|
| 105 |
+
t = cfg.training
|
| 106 |
+
run = wandb.init(
|
| 107 |
+
project=cfg.wandb.project,
|
| 108 |
+
entity=cfg.wandb.entity,
|
| 109 |
+
name=cfg.wandb.name,
|
| 110 |
+
group=cfg.wandb.group,
|
| 111 |
+
tags=list(cfg.wandb.tags),
|
| 112 |
+
mode=cfg.wandb.mode,
|
| 113 |
+
config=OmegaConf.to_container(cfg, resolve=True),
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
task = make_task(cfg.task, **OmegaConf.to_container(cfg.task_cfg, resolve=True))
|
| 117 |
+
model_cfg = model_config_from_dict(OmegaConf.to_container(cfg.model, resolve=True))
|
| 118 |
+
model = build_model(model_cfg, cfg.model_type)
|
| 119 |
+
graph_readout = model_cfg.mpnn_graph_readout
|
| 120 |
+
|
| 121 |
+
sample_fwd, _, _ = task.prepare(next(_train_batches(cfg, 0)))
|
| 122 |
+
state = create_train_state(
|
| 123 |
+
model,
|
| 124 |
+
sample_fwd,
|
| 125 |
+
model_type=cfg.model_type,
|
| 126 |
+
lr=t.lr,
|
| 127 |
+
weight_decay=t.weight_decay,
|
| 128 |
+
warmup_steps=t.warmup_steps,
|
| 129 |
+
grad_clip=t.grad_clip,
|
| 130 |
+
ema_decay=t.ema_decay,
|
| 131 |
+
k_init=t.K_train,
|
| 132 |
+
loss_window=t.loss_window,
|
| 133 |
+
seed=cfg.training.seed,
|
| 134 |
+
)
|
| 135 |
+
if cfg.model_type == "sheaf" and bool(cfg.model.get("rm_constant", False)):
|
| 136 |
+
control_model_config = OmegaConf.to_container(cfg.model, resolve=True)
|
| 137 |
+
control_model_config["rm_init"] = "soft_slice"
|
| 138 |
+
control_model_config["rm_constant"] = False
|
| 139 |
+
control_model = build_model(model_config_from_dict(control_model_config), "sheaf")
|
| 140 |
+
control_state = create_train_state(
|
| 141 |
+
control_model,
|
| 142 |
+
sample_fwd,
|
| 143 |
+
model_type="sheaf",
|
| 144 |
+
lr=t.lr,
|
| 145 |
+
weight_decay=t.weight_decay,
|
| 146 |
+
warmup_steps=t.warmup_steps,
|
| 147 |
+
grad_clip=t.grad_clip,
|
| 148 |
+
ema_decay=t.ema_decay,
|
| 149 |
+
k_init=t.K_train,
|
| 150 |
+
loss_window=t.loss_window,
|
| 151 |
+
seed=cfg.training.seed,
|
| 152 |
+
)
|
| 153 |
+
counters = {"step": 0, "training_seed": int(cfg.training.seed)}
|
| 154 |
+
assert_common_bit_identical(
|
| 155 |
+
control_state.params,
|
| 156 |
+
state.params,
|
| 157 |
+
control_state.opt_state,
|
| 158 |
+
state.opt_state,
|
| 159 |
+
counters,
|
| 160 |
+
counters,
|
| 161 |
+
)
|
| 162 |
+
parity_path = Path(HydraConfig.get().runtime.output_dir) / "identity-parity.json"
|
| 163 |
+
atomic_write_json(
|
| 164 |
+
parity_path,
|
| 165 |
+
{
|
| 166 |
+
"format": 1,
|
| 167 |
+
"common_parameter_leaves_bit_identical": True,
|
| 168 |
+
"common_optimizer_leaves_bit_identical": True,
|
| 169 |
+
"counters_bit_identical": True,
|
| 170 |
+
"identity_parameter_tree_has_restriction_map": False,
|
| 171 |
+
"checked_before_first_optimizer_step": True,
|
| 172 |
+
"outcomes": {},
|
| 173 |
+
},
|
| 174 |
+
)
|
| 175 |
+
print("IDENTITY_COMMON_PARITY_VERIFIED", flush=True)
|
| 176 |
+
run.summary["params"] = sum(x.size for x in jax.tree_util.tree_leaves(state.params))
|
| 177 |
+
print(f"[init] model_type={cfg.model_type} params={run.summary['params']:,}")
|
| 178 |
+
|
| 179 |
+
train_step = make_train_step(task, cfg.model_type, graph_readout)
|
| 180 |
+
rng = jax.random.PRNGKey(cfg.training.seed)
|
| 181 |
+
rng_np = np.random.default_rng(cfg.training.seed)
|
| 182 |
+
history: list[dict] = []
|
| 183 |
+
best: dict[str, float] = {}
|
| 184 |
+
step = 0
|
| 185 |
+
|
| 186 |
+
for epoch in range(t.epochs):
|
| 187 |
+
for batch in _train_batches(cfg, epoch):
|
| 188 |
+
fwd, targets, _ = task.prepare(batch)
|
| 189 |
+
rng, sub = jax.random.split(rng)
|
| 190 |
+
k = (
|
| 191 |
+
sample_k(rng_np, t.train_iters_dist, t.train_iters_min, t.K_train)
|
| 192 |
+
if cfg.model_type == "sheaf"
|
| 193 |
+
else t.mpnn_train_rounds
|
| 194 |
+
)
|
| 195 |
+
state, loss = train_step(state, fwd, targets, sub, n_iter=k, loss_window=t.loss_window)
|
| 196 |
+
step += 1
|
| 197 |
+
loss = float(loss)
|
| 198 |
+
run.log({"train/loss": loss, "train/k": k, "epoch": epoch}, step=step)
|
| 199 |
+
if t.exit_on_nan and not np.isfinite(loss):
|
| 200 |
+
print(f"[epoch {epoch}] non-finite loss — stopping (exit_on_nan).")
|
| 201 |
+
run.finish(exit_code=1)
|
| 202 |
+
return
|
| 203 |
+
|
| 204 |
+
if epoch % t.val_interval == 0 or epoch == t.epochs - 1:
|
| 205 |
+
k_eval = t.K_eval if cfg.model_type == "sheaf" else t.mpnn_eval_rounds
|
| 206 |
+
row = {"epoch": epoch, "loss": loss}
|
| 207 |
+
for split in cfg.data.val_splits:
|
| 208 |
+
m = evaluate(
|
| 209 |
+
state,
|
| 210 |
+
task,
|
| 211 |
+
_val_batches(cfg, split),
|
| 212 |
+
model_type=cfg.model_type,
|
| 213 |
+
graph_readout=graph_readout,
|
| 214 |
+
k_eval=k_eval,
|
| 215 |
+
)
|
| 216 |
+
row[split] = m
|
| 217 |
+
run.log({f"val/{split}/{kk}": vv for kk, vv in m.items()}, step=step)
|
| 218 |
+
for kk, vv in m.items(): # track best-so-far in the run summary
|
| 219 |
+
key = f"best/{split}/{kk}"
|
| 220 |
+
best[key] = max(best.get(key, vv), vv)
|
| 221 |
+
print(
|
| 222 |
+
f"[epoch {epoch}] loss={loss:.4f} {split}: "
|
| 223 |
+
+ " ".join(f"{kk}={vv * 100:.2f}%" for kk, vv in m.items())
|
| 224 |
+
)
|
| 225 |
+
history.append(row)
|
| 226 |
+
run.summary.update(best)
|
| 227 |
+
out = Path(HydraConfig.get().runtime.output_dir)
|
| 228 |
+
save_checkpoint_atomic(
|
| 229 |
+
out / "checkpoint.partial.pkl",
|
| 230 |
+
{
|
| 231 |
+
"params": jax.device_get(state.params),
|
| 232 |
+
"ema_params": jax.device_get(state.ema_params),
|
| 233 |
+
"optimizer_state": jax.device_get(state.opt_state),
|
| 234 |
+
"config": OmegaConf.to_container(cfg, resolve=True),
|
| 235 |
+
"next_epoch": epoch + 1,
|
| 236 |
+
"step": step,
|
| 237 |
+
"jax_rng": np.asarray(jax.device_get(rng)),
|
| 238 |
+
"numpy_rng_state": rng_np.bit_generator.state,
|
| 239 |
+
},
|
| 240 |
+
)
|
| 241 |
+
atomic_write_json(out / "history.partial.json", history)
|
| 242 |
+
|
| 243 |
+
out = Path(HydraConfig.get().runtime.output_dir)
|
| 244 |
+
save_checkpoint_atomic(
|
| 245 |
+
out / "checkpoint.pkl",
|
| 246 |
+
{
|
| 247 |
+
"params": jax.device_get(state.params),
|
| 248 |
+
"ema_params": jax.device_get(state.ema_params),
|
| 249 |
+
"optimizer_state": jax.device_get(state.opt_state),
|
| 250 |
+
"config": OmegaConf.to_container(cfg, resolve=True),
|
| 251 |
+
"seed": int(cfg.training.seed),
|
| 252 |
+
"final_epoch": int(t.epochs) - 1,
|
| 253 |
+
},
|
| 254 |
+
)
|
| 255 |
+
atomic_write_json(out / "history.json", history)
|
| 256 |
+
print(f"[done] saved checkpoint + history to {out}")
|
| 257 |
+
run.finish()
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
if __name__ == "__main__":
|
| 261 |
+
main()
|
scripts/train_repro.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Scientific training contract gate for one registered isolated seed."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
import pickle
|
| 10 |
+
import subprocess
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
| 15 |
+
|
| 16 |
+
from repro_control.archives import safe_extract_tar_gz
|
| 17 |
+
from repro_control.artifacts import finalize_attempt, verify_read_back
|
| 18 |
+
from repro_control.configs import (
|
| 19 |
+
config_sha256,
|
| 20 |
+
load_json_config,
|
| 21 |
+
registered_config_for,
|
| 22 |
+
verify_before_optimizer,
|
| 23 |
+
)
|
| 24 |
+
from repro_control.hashing import atomic_write_json, sha256_file
|
| 25 |
+
from repro_control.heartbeat import heartbeat, marker
|
| 26 |
+
from repro_control.interventions import contains_restriction_map
|
| 27 |
+
from repro_control.runtime import (
|
| 28 |
+
configure_scientific_runtime,
|
| 29 |
+
load_job_manifest,
|
| 30 |
+
require_control_freeze,
|
| 31 |
+
require_jax_platform,
|
| 32 |
+
verify_science_spec,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def main() -> int:
|
| 37 |
+
parser = argparse.ArgumentParser()
|
| 38 |
+
parser.add_argument("--science-spec", type=Path, required=True)
|
| 39 |
+
parser.add_argument("--job-manifest", type=Path, required=True)
|
| 40 |
+
parser.add_argument("--control-dir", type=Path, required=True)
|
| 41 |
+
parser.add_argument("--freeze-sha256", required=True)
|
| 42 |
+
parser.add_argument("--data-dir", type=Path)
|
| 43 |
+
parser.add_argument("--train-archive", type=Path)
|
| 44 |
+
parser.add_argument("--output-dir", type=Path)
|
| 45 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 46 |
+
args = parser.parse_args()
|
| 47 |
+
spec = verify_science_spec(args.science_spec)
|
| 48 |
+
manifest = load_job_manifest(args.job_manifest, freeze_sha256=args.freeze_sha256)
|
| 49 |
+
if manifest["job_class"] != "SCIENTIFIC_TRAIN":
|
| 50 |
+
raise SystemExit("training entrypoint requires SCIENTIFIC_TRAIN")
|
| 51 |
+
if manifest["logical_id"] not in spec["run_identities"]:
|
| 52 |
+
raise SystemExit("unregistered training logical identity")
|
| 53 |
+
require_control_freeze(args.control_dir, args.freeze_sha256, manifest["science_spec_sha256"])
|
| 54 |
+
configure_scientific_runtime()
|
| 55 |
+
if args.dry_run:
|
| 56 |
+
print(
|
| 57 |
+
json.dumps(
|
| 58 |
+
{
|
| 59 |
+
"contract_verified": True,
|
| 60 |
+
"logical_id": manifest["logical_id"],
|
| 61 |
+
"outcomes": {},
|
| 62 |
+
}
|
| 63 |
+
)
|
| 64 |
+
)
|
| 65 |
+
return 0
|
| 66 |
+
if args.data_dir is None or args.output_dir is None:
|
| 67 |
+
raise SystemExit("--data-dir and --output-dir are required for execution")
|
| 68 |
+
if args.train_archive:
|
| 69 |
+
safe_extract_tar_gz(args.train_archive, args.data_dir)
|
| 70 |
+
require_jax_platform("gpu")
|
| 71 |
+
marker("GPU_READY", manifest["logical_id"])
|
| 72 |
+
logical_id = manifest["logical_id"]
|
| 73 |
+
seed = manifest["seed"]
|
| 74 |
+
expected_path = args.control_dir / "registered-configs" / f"{logical_id}.json"
|
| 75 |
+
expected = load_json_config(expected_path)
|
| 76 |
+
expected_sha256 = config_sha256(expected)
|
| 77 |
+
if expected_sha256 != manifest["hashes"]["config"]:
|
| 78 |
+
raise SystemExit("registered config hash differs from the Job manifest")
|
| 79 |
+
independently_composed = registered_config_for(
|
| 80 |
+
Path(__file__).resolve().parents[1],
|
| 81 |
+
logical_id,
|
| 82 |
+
data_root=str(args.data_dir),
|
| 83 |
+
)
|
| 84 |
+
verify_before_optimizer(
|
| 85 |
+
independently_composed,
|
| 86 |
+
expected,
|
| 87 |
+
expected_sha256=expected_sha256,
|
| 88 |
+
)
|
| 89 |
+
args.output_dir.mkdir(parents=True, exist_ok=False)
|
| 90 |
+
with heartbeat(f"SCIENTIFIC_TRAIN:{logical_id}"):
|
| 91 |
+
if logical_id.startswith("C3-MNIST-CNN-"):
|
| 92 |
+
from repro_control.cnn_training import train_fixed_cnn
|
| 93 |
+
|
| 94 |
+
train_fixed_cnn(
|
| 95 |
+
Path(expected["data"]["dir"]),
|
| 96 |
+
args.output_dir,
|
| 97 |
+
seed=seed,
|
| 98 |
+
registered_config=expected,
|
| 99 |
+
registered_config_sha256=expected_sha256,
|
| 100 |
+
)
|
| 101 |
+
else:
|
| 102 |
+
root = Path(__file__).resolve().parents[1]
|
| 103 |
+
if logical_id.startswith("C1-SUD-MPNN225-"):
|
| 104 |
+
experiment = "sudoku_mpnn"
|
| 105 |
+
overrides = [
|
| 106 |
+
"model.d_v=225",
|
| 107 |
+
"training.lr=1.7e-3",
|
| 108 |
+
"training.epochs=10",
|
| 109 |
+
]
|
| 110 |
+
elif logical_id.startswith("C2-MAZE-MPNN84-"):
|
| 111 |
+
experiment = "maze_mpnn"
|
| 112 |
+
overrides = []
|
| 113 |
+
elif logical_id.startswith("C4-SUD-IDENTITY-"):
|
| 114 |
+
experiment = "sudoku_sheaf"
|
| 115 |
+
overrides = ["model.rm_init=identity", "+model.rm_constant=true"]
|
| 116 |
+
elif logical_id.startswith("C4-MAZE-QUADRATIC-"):
|
| 117 |
+
experiment = "maze_sheaf"
|
| 118 |
+
overrides = ["model.objective_mode=quadratic"]
|
| 119 |
+
else:
|
| 120 |
+
raise SystemExit("no registered training backend")
|
| 121 |
+
command = [
|
| 122 |
+
sys.executable,
|
| 123 |
+
str(root / "scripts" / "train.py"),
|
| 124 |
+
f"+experiment={experiment}",
|
| 125 |
+
f"training.seed={seed}",
|
| 126 |
+
f"data.dir={expected['data']['dir']}",
|
| 127 |
+
"data.val_splits=[]",
|
| 128 |
+
"wandb.mode=disabled",
|
| 129 |
+
f"hydra.run.dir={args.output_dir}",
|
| 130 |
+
*overrides,
|
| 131 |
+
]
|
| 132 |
+
env = os.environ.copy()
|
| 133 |
+
env.update(
|
| 134 |
+
REPRO_EXPECTED_CONFIG_PATH=str(expected_path),
|
| 135 |
+
REPRO_EXPECTED_CONFIG_SHA256=expected_sha256,
|
| 136 |
+
)
|
| 137 |
+
subprocess.run(command, check=True, env=env)
|
| 138 |
+
|
| 139 |
+
checkpoint_path = args.output_dir / "checkpoint.pkl"
|
| 140 |
+
with checkpoint_path.open("rb") as handle:
|
| 141 |
+
checkpoint = pickle.load(handle)
|
| 142 |
+
if handle.read(1):
|
| 143 |
+
raise SystemExit("checkpoint contains trailing bytes")
|
| 144 |
+
if checkpoint.get("ema_params") is None:
|
| 145 |
+
raise SystemExit("training checkpoint lacks EMA parameters")
|
| 146 |
+
import jax
|
| 147 |
+
|
| 148 |
+
parameter_count = sum(
|
| 149 |
+
int(leaf.size) for leaf in jax.tree_util.tree_leaves(checkpoint["params"])
|
| 150 |
+
)
|
| 151 |
+
identity_parity = None
|
| 152 |
+
if logical_id.startswith("C4-SUD-IDENTITY-"):
|
| 153 |
+
parity_path = args.output_dir / "identity-parity.json"
|
| 154 |
+
if not parity_path.is_file():
|
| 155 |
+
raise SystemExit("identity training lacks the pre-step parity receipt")
|
| 156 |
+
identity_parity = json.loads(parity_path.read_text())
|
| 157 |
+
if not all(
|
| 158 |
+
identity_parity.get(name) is True
|
| 159 |
+
for name in (
|
| 160 |
+
"common_parameter_leaves_bit_identical",
|
| 161 |
+
"common_optimizer_leaves_bit_identical",
|
| 162 |
+
"counters_bit_identical",
|
| 163 |
+
"checked_before_first_optimizer_step",
|
| 164 |
+
)
|
| 165 |
+
):
|
| 166 |
+
raise SystemExit("identity pre-step parity receipt is not affirmative")
|
| 167 |
+
if contains_restriction_map(checkpoint["params"]):
|
| 168 |
+
raise SystemExit("identity checkpoint unexpectedly contains restriction-map parameters")
|
| 169 |
+
receipt = {
|
| 170 |
+
"format": 1,
|
| 171 |
+
"logical_id": logical_id,
|
| 172 |
+
"attempt_id": manifest["attempt_id"],
|
| 173 |
+
"seed": seed,
|
| 174 |
+
"registered_config_sha256": expected_sha256,
|
| 175 |
+
"checkpoint_sha256": sha256_file(checkpoint_path),
|
| 176 |
+
"history_sha256": sha256_file(args.output_dir / "history.json"),
|
| 177 |
+
"parameter_count": parameter_count,
|
| 178 |
+
"ema": True,
|
| 179 |
+
"fixed_final_epoch": int(expected["training"]["epochs"]) - 1,
|
| 180 |
+
"identity_common_parity": identity_parity,
|
| 181 |
+
"outcomes": {},
|
| 182 |
+
}
|
| 183 |
+
atomic_write_json(args.output_dir / "training-receipt.json", receipt)
|
| 184 |
+
finalize_attempt(
|
| 185 |
+
args.output_dir,
|
| 186 |
+
logical_id=logical_id,
|
| 187 |
+
attempt_id=manifest["attempt_id"],
|
| 188 |
+
expected_outputs=manifest["expected_outputs"],
|
| 189 |
+
)
|
| 190 |
+
verify_read_back(
|
| 191 |
+
args.output_dir,
|
| 192 |
+
logical_id=logical_id,
|
| 193 |
+
attempt_id=manifest["attempt_id"],
|
| 194 |
+
)
|
| 195 |
+
marker("DONE", f"{logical_id} {manifest['attempt_id']}")
|
| 196 |
+
return 0
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
if __name__ == "__main__":
|
| 200 |
+
raise SystemExit(main())
|
scripts/visualize.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Emit the paper's analysis artifacts from a trained Sheaf-ADMM checkpoint.
|
| 2 |
+
|
| 3 |
+
Loads a ``checkpoint.pkl`` written by ``scripts/train.py`` (a dict with
|
| 4 |
+
``params`` / ``ema_params`` / ``config``), rebuilds the model + task from the
|
| 5 |
+
embedded config, pulls one validation batch, and writes the three artifacts:
|
| 6 |
+
|
| 7 |
+
* ``prediction_evolution.pdf`` — global prediction vs ADMM count ``k``;
|
| 8 |
+
* ``coordination_dynamics.pdf`` — primal/dual residual heatmaps + curves;
|
| 9 |
+
* ``xz_trajectories.pdf`` — per-agent x-vs-z 2-D trajectories.
|
| 10 |
+
|
| 11 |
+
python -m scripts.visualize --checkpoint outputs/.../checkpoint.pkl --out-dir /tmp/viz
|
| 12 |
+
|
| 13 |
+
The ``--task`` / ``--data-dir`` flags override what the checkpoint config says
|
| 14 |
+
(useful for visualizing on an OOD split). Evaluation uses the EMA parameters by
|
| 15 |
+
default (``--params ema``), matching how the trainer reports metrics.
|
| 16 |
+
Only Sheaf-ADMM checkpoints are supported; MPNN baselines do not expose an ADMM
|
| 17 |
+
trajectory.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import argparse
|
| 23 |
+
import pickle
|
| 24 |
+
import sys
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
|
| 29 |
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
| 30 |
+
if str(PROJECT_ROOT / "src") not in sys.path:
|
| 31 |
+
sys.path.insert(0, str(PROJECT_ROOT / "src"))
|
| 32 |
+
|
| 33 |
+
import sheaf_admm as _sheaf_admm # noqa: F401,E402 (sets matmul precision on import)
|
| 34 |
+
from sheaf_admm.data import ImageDataset, PuzzleDataset # noqa: E402
|
| 35 |
+
from sheaf_admm.models import model_config_from_dict # noqa: E402
|
| 36 |
+
from sheaf_admm.training import build_model, make_task # noqa: E402
|
| 37 |
+
from sheaf_admm.viz import ( # noqa: E402
|
| 38 |
+
plot_coordination_dynamics,
|
| 39 |
+
plot_prediction_evolution,
|
| 40 |
+
plot_xz_trajectories,
|
| 41 |
+
run_trajectory,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# Default per-task ADMM counts for the prediction-evolution panels.
|
| 45 |
+
DEFAULT_KS = {"maze": (1, 3, 5, 10, 30), "sudoku": (1, 3, 5, 10, 20), "mnist": (1, 3, 5, 10, 30)}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _first_batch(cfg: dict, task_name: str, data_dir: str, split: str):
|
| 49 |
+
loader = cfg["data"].get("loader", "puzzle" if task_name in ("maze", "sudoku") else "image")
|
| 50 |
+
if loader == "puzzle":
|
| 51 |
+
ds = PuzzleDataset(data_dir, split)
|
| 52 |
+
for _set, b in ds.iter_test_batches(batch_size=8):
|
| 53 |
+
batch = {"inputs": np.asarray(b["inputs"]), "labels": np.asarray(b["labels"])}
|
| 54 |
+
for key in ("height", "width"):
|
| 55 |
+
if key in b:
|
| 56 |
+
batch[key] = b[key]
|
| 57 |
+
return batch
|
| 58 |
+
ds = ImageDataset(data_dir, split)
|
| 59 |
+
for b in ds.iter_batches(batch_size=8, shuffle=False):
|
| 60 |
+
return {"images": np.asarray(b["images"]), "labels": np.asarray(b["labels"])}
|
| 61 |
+
raise RuntimeError("no batch available from the dataset")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def main() -> None:
|
| 65 |
+
p = argparse.ArgumentParser(description=__doc__)
|
| 66 |
+
p.add_argument("--checkpoint", required=True, help="path to checkpoint.pkl")
|
| 67 |
+
p.add_argument("--out-dir", required=True, help="directory for the artifact files")
|
| 68 |
+
p.add_argument("--task", default=None, help="override task (maze|sudoku|mnist)")
|
| 69 |
+
p.add_argument("--data-dir", default=None, help="override dataset directory")
|
| 70 |
+
p.add_argument("--split", default=None, help="dataset split (default: first val split)")
|
| 71 |
+
p.add_argument("--params", choices=("ema", "raw"), default="ema", help="which weights to use")
|
| 72 |
+
p.add_argument(
|
| 73 |
+
"--num-iters", type=int, default=30, help="ADMM steps for the trajectory artifacts"
|
| 74 |
+
)
|
| 75 |
+
p.add_argument("--batch-index", type=int, default=0, help="example index within the batch")
|
| 76 |
+
p.add_argument(
|
| 77 |
+
"--ks", type=int, nargs="+", default=None, help="ADMM counts for prediction evolution"
|
| 78 |
+
)
|
| 79 |
+
args = p.parse_args()
|
| 80 |
+
|
| 81 |
+
with open(args.checkpoint, "rb") as f:
|
| 82 |
+
ckpt = pickle.load(f)
|
| 83 |
+
cfg = ckpt["config"]
|
| 84 |
+
if cfg.get("model_type") != "sheaf":
|
| 85 |
+
raise ValueError(
|
| 86 |
+
"scripts.visualize supports Sheaf-ADMM checkpoints only "
|
| 87 |
+
f"(got model_type={cfg.get('model_type')!r})."
|
| 88 |
+
)
|
| 89 |
+
params = ckpt.get("ema_params") if args.params == "ema" else ckpt["params"]
|
| 90 |
+
if params is None:
|
| 91 |
+
params = ckpt["params"]
|
| 92 |
+
|
| 93 |
+
task_name = args.task or cfg["task"]
|
| 94 |
+
data_dir = args.data_dir or cfg["data"]["dir"]
|
| 95 |
+
split = args.split or cfg["data"]["val_splits"][0]
|
| 96 |
+
ks = tuple(args.ks) if args.ks else DEFAULT_KS[task_name]
|
| 97 |
+
|
| 98 |
+
task = make_task(task_name, **cfg.get("task_cfg", {}))
|
| 99 |
+
model_cfg = model_config_from_dict(cfg["model"])
|
| 100 |
+
model = build_model(model_cfg, cfg["model_type"])
|
| 101 |
+
|
| 102 |
+
batch = _first_batch(cfg, task_name, data_dir, split)
|
| 103 |
+
fwd, targets, aux = task.prepare(batch)
|
| 104 |
+
|
| 105 |
+
out_dir = Path(args.out_dir)
|
| 106 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 107 |
+
|
| 108 |
+
written = []
|
| 109 |
+
written.append(
|
| 110 |
+
plot_prediction_evolution(
|
| 111 |
+
model,
|
| 112 |
+
params,
|
| 113 |
+
task,
|
| 114 |
+
fwd,
|
| 115 |
+
targets,
|
| 116 |
+
aux,
|
| 117 |
+
ks,
|
| 118 |
+
str(out_dir / "prediction_evolution.pdf"),
|
| 119 |
+
batch_index=args.batch_index,
|
| 120 |
+
title=f"{task_name}: prediction vs k",
|
| 121 |
+
)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
centers = aux.get("centers")
|
| 125 |
+
traj = run_trajectory(
|
| 126 |
+
model,
|
| 127 |
+
params,
|
| 128 |
+
fwd,
|
| 129 |
+
num_iters=args.num_iters,
|
| 130 |
+
batch_index=args.batch_index,
|
| 131 |
+
centers=centers,
|
| 132 |
+
)
|
| 133 |
+
written.append(
|
| 134 |
+
plot_coordination_dynamics(
|
| 135 |
+
traj,
|
| 136 |
+
str(out_dir / "coordination_dynamics.pdf"),
|
| 137 |
+
title=f"{task_name}: coordination dynamics (rho={traj.rho:.3g})",
|
| 138 |
+
)
|
| 139 |
+
)
|
| 140 |
+
written.append(
|
| 141 |
+
plot_xz_trajectories(
|
| 142 |
+
traj,
|
| 143 |
+
str(out_dir / "xz_trajectories.pdf"),
|
| 144 |
+
title=f"{task_name}: x vs z trajectories",
|
| 145 |
+
)
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
for path in written:
|
| 149 |
+
print(f"[viz] wrote {path}")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
if __name__ == "__main__":
|
| 153 |
+
main()
|
src/repro_control/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fail-closed control plane for the registered local reproduction."""
|
| 2 |
+
|
| 3 |
+
from .constants import SCIENCE_SPEC_SHA256
|
| 4 |
+
|
| 5 |
+
__all__ = ["SCIENCE_SPEC_SHA256"]
|
src/repro_control/aggregation.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Accepted-attempt registry and complete-only aggregation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from .artifacts import verify_read_back
|
| 10 |
+
from .hashing import atomic_write_json, sha256_file
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class AggregationError(ValueError):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def accept_attempt(
|
| 18 |
+
registry_path: Path,
|
| 19 |
+
*,
|
| 20 |
+
logical_id: str,
|
| 21 |
+
attempt_id: str,
|
| 22 |
+
prefix: Path,
|
| 23 |
+
) -> dict[str, Any]:
|
| 24 |
+
registry = (
|
| 25 |
+
json.loads(registry_path.read_text())
|
| 26 |
+
if registry_path.exists()
|
| 27 |
+
else {"format": 1, "accepted_attempts": []}
|
| 28 |
+
)
|
| 29 |
+
if set(registry) != {"format", "accepted_attempts"} or registry["format"] != 1:
|
| 30 |
+
raise AggregationError("invalid accepted-attempt registry")
|
| 31 |
+
existing = [
|
| 32 |
+
row for row in registry["accepted_attempts"] if row["logical_id"] == logical_id
|
| 33 |
+
]
|
| 34 |
+
if existing:
|
| 35 |
+
raise AggregationError(f"logical identity already accepted: {logical_id}")
|
| 36 |
+
done = verify_read_back(prefix, logical_id=logical_id, attempt_id=attempt_id)
|
| 37 |
+
row = {
|
| 38 |
+
"logical_id": logical_id,
|
| 39 |
+
"attempt_id": attempt_id,
|
| 40 |
+
"prefix": prefix.as_posix(),
|
| 41 |
+
"done_sha256": sha256_file(prefix / "DONE.json"),
|
| 42 |
+
"object_root_sha256": done["object_root_sha256"],
|
| 43 |
+
}
|
| 44 |
+
registry["accepted_attempts"].append(row)
|
| 45 |
+
registry["accepted_attempts"].sort(key=lambda item: item["logical_id"])
|
| 46 |
+
atomic_write_json(registry_path, registry)
|
| 47 |
+
return row
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def load_complete_units(registry_path: Path, required_logical_ids: set[str]) -> list[dict[str, Any]]:
|
| 51 |
+
registry = json.loads(registry_path.read_text())
|
| 52 |
+
rows = registry["accepted_attempts"]
|
| 53 |
+
identities = [row["logical_id"] for row in rows]
|
| 54 |
+
if len(identities) != len(set(identities)):
|
| 55 |
+
raise AggregationError("duplicate accepted logical identities")
|
| 56 |
+
missing = required_logical_ids - set(identities)
|
| 57 |
+
extra = set(identities) - required_logical_ids
|
| 58 |
+
if missing or extra:
|
| 59 |
+
raise AggregationError(f"accepted identity mismatch: missing={sorted(missing)} extra={sorted(extra)}")
|
| 60 |
+
for row in rows:
|
| 61 |
+
prefix = Path(row["prefix"])
|
| 62 |
+
done = verify_read_back(
|
| 63 |
+
prefix, logical_id=row["logical_id"], attempt_id=row["attempt_id"]
|
| 64 |
+
)
|
| 65 |
+
if sha256_file(prefix / "DONE.json") != row["done_sha256"]:
|
| 66 |
+
raise AggregationError("accepted DONE receipt changed")
|
| 67 |
+
if done["object_root_sha256"] != row["object_root_sha256"]:
|
| 68 |
+
raise AggregationError("accepted object root changed")
|
| 69 |
+
return rows
|
src/repro_control/archives.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic input archives and traversal-safe extraction."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import gzip
|
| 6 |
+
import shutil
|
| 7 |
+
import tarfile
|
| 8 |
+
from pathlib import Path, PurePosixPath
|
| 9 |
+
|
| 10 |
+
from .hashing import sha256_file
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ArchiveError(ValueError):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def create_deterministic_tar_gz(source: Path, destination: Path) -> str:
|
| 18 |
+
if not source.is_dir():
|
| 19 |
+
raise ArchiveError(f"archive source is not a directory: {source}")
|
| 20 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 21 |
+
with (
|
| 22 |
+
destination.open("wb") as raw,
|
| 23 |
+
gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed,
|
| 24 |
+
tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive,
|
| 25 |
+
):
|
| 26 |
+
for path in sorted(source.rglob("*")):
|
| 27 |
+
relative = path.relative_to(source)
|
| 28 |
+
info = archive.gettarinfo(str(path), arcname=relative.as_posix())
|
| 29 |
+
info.uid = 0
|
| 30 |
+
info.gid = 0
|
| 31 |
+
info.uname = ""
|
| 32 |
+
info.gname = ""
|
| 33 |
+
info.mtime = 0
|
| 34 |
+
info.mode = 0o755 if path.is_dir() else 0o644
|
| 35 |
+
if path.is_file():
|
| 36 |
+
with path.open("rb") as handle:
|
| 37 |
+
archive.addfile(info, handle)
|
| 38 |
+
elif path.is_dir():
|
| 39 |
+
archive.addfile(info)
|
| 40 |
+
else:
|
| 41 |
+
raise ArchiveError(f"unsupported archive entry: {path}")
|
| 42 |
+
return sha256_file(destination)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def safe_extract_tar_gz(archive_path: Path, destination: Path) -> None:
|
| 46 |
+
if destination.exists() and any(destination.iterdir()):
|
| 47 |
+
raise ArchiveError(f"extraction destination is not empty: {destination}")
|
| 48 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 49 |
+
with tarfile.open(archive_path, mode="r:gz") as archive:
|
| 50 |
+
members = archive.getmembers()
|
| 51 |
+
names = set()
|
| 52 |
+
for member in members:
|
| 53 |
+
relative = PurePosixPath(member.name)
|
| 54 |
+
if relative.is_absolute() or ".." in relative.parts or not relative.parts:
|
| 55 |
+
raise ArchiveError(f"unsafe archive member: {member.name!r}")
|
| 56 |
+
if member.name in names:
|
| 57 |
+
raise ArchiveError(f"duplicate archive member: {member.name!r}")
|
| 58 |
+
names.add(member.name)
|
| 59 |
+
if member.issym() or member.islnk() or member.isdev():
|
| 60 |
+
raise ArchiveError(f"links/devices are forbidden in input archives: {member.name}")
|
| 61 |
+
archive.extractall(destination, members=members, filter="data")
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def copy_tree_selected(source: Path, destination: Path, names: tuple[str, ...]) -> None:
|
| 65 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 66 |
+
for name in names:
|
| 67 |
+
src = source / name
|
| 68 |
+
dst = destination / name
|
| 69 |
+
if not src.exists():
|
| 70 |
+
raise ArchiveError(f"selected source does not exist: {src}")
|
| 71 |
+
if src.is_dir():
|
| 72 |
+
shutil.copytree(src, dst)
|
| 73 |
+
else:
|
| 74 |
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
| 75 |
+
shutil.copy2(src, dst)
|
src/repro_control/artifacts.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Incremental object manifests, DONE-last completion, and independent read-back."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from .hashing import atomic_write_json, canonical_root, file_entry, sha256_file
|
| 10 |
+
|
| 11 |
+
OBJECT_MANIFEST = "OBJECT-MANIFEST.json"
|
| 12 |
+
DONE_MARKER = "DONE.json"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class ArtifactError(ValueError):
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def build_object_manifest(prefix: Path, expected_outputs: list[str]) -> dict[str, Any]:
|
| 20 |
+
forbidden = {OBJECT_MANIFEST, DONE_MARKER}
|
| 21 |
+
outputs = [name for name in expected_outputs if name not in forbidden]
|
| 22 |
+
entries = []
|
| 23 |
+
for name in outputs:
|
| 24 |
+
path = prefix / name
|
| 25 |
+
if not path.is_file():
|
| 26 |
+
raise ArtifactError(f"missing expected output {name}")
|
| 27 |
+
entries.append(file_entry(path, relative_to=prefix))
|
| 28 |
+
manifest = {"format": 1, "entries": entries, "root_sha256": canonical_root(entries)}
|
| 29 |
+
atomic_write_json(prefix / OBJECT_MANIFEST, manifest)
|
| 30 |
+
return manifest
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def finalize_attempt(
|
| 34 |
+
prefix: Path,
|
| 35 |
+
*,
|
| 36 |
+
logical_id: str,
|
| 37 |
+
attempt_id: str,
|
| 38 |
+
expected_outputs: list[str],
|
| 39 |
+
) -> dict[str, Any]:
|
| 40 |
+
if (prefix / DONE_MARKER).exists():
|
| 41 |
+
raise ArtifactError("DONE.json already exists")
|
| 42 |
+
manifest = build_object_manifest(prefix, expected_outputs)
|
| 43 |
+
done = {
|
| 44 |
+
"format": 1,
|
| 45 |
+
"logical_id": logical_id,
|
| 46 |
+
"attempt_id": attempt_id,
|
| 47 |
+
"object_manifest_sha256": sha256_file(prefix / OBJECT_MANIFEST),
|
| 48 |
+
"object_root_sha256": manifest["root_sha256"],
|
| 49 |
+
"complete": True,
|
| 50 |
+
"outcomes": {},
|
| 51 |
+
}
|
| 52 |
+
atomic_write_json(prefix / DONE_MARKER, done)
|
| 53 |
+
return done
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def verify_read_back(
|
| 57 |
+
prefix: Path, *, logical_id: str | None = None, attempt_id: str | None = None
|
| 58 |
+
) -> dict[str, Any]:
|
| 59 |
+
done_path = prefix / DONE_MARKER
|
| 60 |
+
manifest_path = prefix / OBJECT_MANIFEST
|
| 61 |
+
if not done_path.is_file() or not manifest_path.is_file():
|
| 62 |
+
raise ArtifactError("attempt lacks completion markers")
|
| 63 |
+
done = json.loads(done_path.read_text())
|
| 64 |
+
manifest = json.loads(manifest_path.read_text())
|
| 65 |
+
if done.get("complete") is not True or done.get("outcomes") != {}:
|
| 66 |
+
raise ArtifactError("invalid local pre-outcome DONE marker")
|
| 67 |
+
if logical_id is not None and done.get("logical_id") != logical_id:
|
| 68 |
+
raise ArtifactError("logical_id mismatch")
|
| 69 |
+
if attempt_id is not None and done.get("attempt_id") != attempt_id:
|
| 70 |
+
raise ArtifactError("attempt_id mismatch")
|
| 71 |
+
if done.get("object_manifest_sha256") != sha256_file(manifest_path):
|
| 72 |
+
raise ArtifactError("object manifest hash mismatch")
|
| 73 |
+
actual = []
|
| 74 |
+
for row in manifest.get("entries", []):
|
| 75 |
+
path = prefix / row["path"]
|
| 76 |
+
if not path.is_file():
|
| 77 |
+
raise ArtifactError(f"missing object {row['path']}")
|
| 78 |
+
entry = file_entry(path, relative_to=prefix)
|
| 79 |
+
if entry != row:
|
| 80 |
+
raise ArtifactError(f"read-back mismatch {row['path']}")
|
| 81 |
+
actual.append(entry)
|
| 82 |
+
root = canonical_root(actual)
|
| 83 |
+
if root != manifest.get("root_sha256") or root != done.get("object_root_sha256"):
|
| 84 |
+
raise ArtifactError("object root mismatch")
|
| 85 |
+
return done
|
src/repro_control/c5_runtime.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""C5 GPU admission probing and deterministic padded-tail runtime checks."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
|
| 7 |
+
BATCH_CANDIDATES = (1, 2, 4, 8, 16, 32, 64, 128)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class C5RuntimeError(ValueError):
|
| 11 |
+
pass
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass(frozen=True)
|
| 15 |
+
class Probe:
|
| 16 |
+
batch_size: int
|
| 17 |
+
peak_bytes: int
|
| 18 |
+
post_compile_seconds: tuple[float, float, float]
|
| 19 |
+
weight_swap_compile_counts: tuple[int, int, int]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def select_batch(probes: list[Probe], total_vram_bytes: int) -> Probe:
|
| 23 |
+
by_size = {probe.batch_size: probe for probe in probes}
|
| 24 |
+
if 1 not in by_size:
|
| 25 |
+
raise C5RuntimeError("batch-1 probe failure rejects the route")
|
| 26 |
+
valid = []
|
| 27 |
+
for size in BATCH_CANDIDATES:
|
| 28 |
+
probe = by_size.get(size)
|
| 29 |
+
if probe is None:
|
| 30 |
+
continue
|
| 31 |
+
if probe.peak_bytes <= 0 or probe.peak_bytes > 0.8 * total_vram_bytes:
|
| 32 |
+
continue
|
| 33 |
+
if len(probe.post_compile_seconds) != 3 or any(
|
| 34 |
+
value <= 0 for value in probe.post_compile_seconds
|
| 35 |
+
):
|
| 36 |
+
raise C5RuntimeError("each probe needs three positive post-compile timings")
|
| 37 |
+
if probe.weight_swap_compile_counts != (1, 1, 1):
|
| 38 |
+
raise C5RuntimeError("all three weight swaps must reuse one compiled executable")
|
| 39 |
+
valid.append(probe)
|
| 40 |
+
if not valid:
|
| 41 |
+
raise C5RuntimeError("no batch size meets the 80% VRAM and parity contract")
|
| 42 |
+
return max(valid, key=lambda probe: probe.batch_size)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def padded_tail(total_examples: int, batch_size: int) -> tuple[int, tuple[bool, ...]]:
|
| 46 |
+
if total_examples <= 0 or batch_size <= 0:
|
| 47 |
+
raise ValueError("total_examples and batch_size must be positive")
|
| 48 |
+
remainder = total_examples % batch_size
|
| 49 |
+
real = batch_size if remainder == 0 else remainder
|
| 50 |
+
return (total_examples + batch_size - 1) // batch_size, tuple(
|
| 51 |
+
index < real for index in range(batch_size)
|
| 52 |
+
)
|
src/repro_control/checkpoints.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Neutral checkpoint hashing, structural validation, and atomic round trips."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import pickle
|
| 8 |
+
import tempfile
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
from .hashing import sha256_file
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class CheckpointError(ValueError):
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _tree_leaves(value: Any):
|
| 20 |
+
if isinstance(value, dict):
|
| 21 |
+
for key in sorted(value):
|
| 22 |
+
yield from _tree_leaves(value[key])
|
| 23 |
+
elif isinstance(value, (list, tuple)):
|
| 24 |
+
for item in value:
|
| 25 |
+
yield from _tree_leaves(item)
|
| 26 |
+
else:
|
| 27 |
+
yield value
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def parameter_count(tree: Any) -> int:
|
| 31 |
+
total = 0
|
| 32 |
+
for leaf in _tree_leaves(tree):
|
| 33 |
+
size = getattr(leaf, "size", None)
|
| 34 |
+
if size is None:
|
| 35 |
+
if isinstance(leaf, (int, float, bool, complex)):
|
| 36 |
+
size = 1
|
| 37 |
+
else:
|
| 38 |
+
raise CheckpointError(f"non-array parameter leaf {type(leaf).__name__}")
|
| 39 |
+
total += int(size)
|
| 40 |
+
return total
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_neutral_checkpoint(
|
| 44 |
+
checkpoint_path: Path,
|
| 45 |
+
config_path: Path,
|
| 46 |
+
import_row: dict[str, Any],
|
| 47 |
+
) -> tuple[Any, dict[str, Any]]:
|
| 48 |
+
if checkpoint_path.stat().st_size != import_row["byte_size"]:
|
| 49 |
+
raise CheckpointError("checkpoint byte-size mismatch")
|
| 50 |
+
if sha256_file(checkpoint_path) != import_row["checkpoint_sha256"]:
|
| 51 |
+
raise CheckpointError("checkpoint hash mismatch")
|
| 52 |
+
if sha256_file(config_path) != import_row["sanitized_config_sha256"]:
|
| 53 |
+
raise CheckpointError("sanitized config hash mismatch")
|
| 54 |
+
config = json.loads(config_path.read_text())
|
| 55 |
+
with checkpoint_path.open("rb") as handle:
|
| 56 |
+
checkpoint = pickle.load(handle)
|
| 57 |
+
if handle.read(1):
|
| 58 |
+
raise CheckpointError("trailing checkpoint bytes")
|
| 59 |
+
if not isinstance(checkpoint, dict):
|
| 60 |
+
raise CheckpointError("checkpoint root must be a mapping")
|
| 61 |
+
params = checkpoint.get("params")
|
| 62 |
+
ema = checkpoint.get("ema_params")
|
| 63 |
+
if params is None or ema is None:
|
| 64 |
+
raise CheckpointError("checkpoint must contain params and EMA params")
|
| 65 |
+
if import_row["ema_status"]["present"] is not True:
|
| 66 |
+
raise CheckpointError("import row does not authorize EMA evaluation")
|
| 67 |
+
if parameter_count(params) != parameter_count(ema):
|
| 68 |
+
raise CheckpointError("parameter and EMA tree counts differ")
|
| 69 |
+
return checkpoint, config
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def save_checkpoint_atomic(path: Path, checkpoint: dict[str, Any]) -> str:
|
| 73 |
+
if "params" not in checkpoint or "ema_params" not in checkpoint:
|
| 74 |
+
raise CheckpointError("checkpoint must contain params and ema_params")
|
| 75 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 76 |
+
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
| 77 |
+
try:
|
| 78 |
+
with os.fdopen(descriptor, "wb") as handle:
|
| 79 |
+
pickle.dump(checkpoint, handle, protocol=5)
|
| 80 |
+
handle.flush()
|
| 81 |
+
os.fsync(handle.fileno())
|
| 82 |
+
with open(temporary, "rb") as handle:
|
| 83 |
+
reloaded = pickle.load(handle)
|
| 84 |
+
if parameter_count(reloaded["params"]) != parameter_count(checkpoint["params"]):
|
| 85 |
+
raise CheckpointError("checkpoint round-trip parameter mismatch")
|
| 86 |
+
os.replace(temporary, path)
|
| 87 |
+
except BaseException:
|
| 88 |
+
try:
|
| 89 |
+
os.unlink(temporary)
|
| 90 |
+
except FileNotFoundError:
|
| 91 |
+
pass
|
| 92 |
+
raise
|
| 93 |
+
return sha256_file(path)
|
src/repro_control/cnn.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exact fixed MNIST CNN registered for Claim 3."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
PARAMETER_COUNT = 65_642
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def _jax():
|
| 11 |
+
import jax
|
| 12 |
+
import jax.numpy as jnp
|
| 13 |
+
|
| 14 |
+
return jax, jnp
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def init_params(key) -> dict[str, dict[str, Any]]:
|
| 18 |
+
jax, jnp = _jax()
|
| 19 |
+
|
| 20 |
+
def layer(subkey, shape):
|
| 21 |
+
fan_in = shape[0] * shape[1] * shape[2] if len(shape) == 4 else shape[0]
|
| 22 |
+
fan_out = shape[0] * shape[1] * shape[3] if len(shape) == 4 else shape[1]
|
| 23 |
+
limit = jnp.sqrt(jnp.asarray(6.0 / (fan_in + fan_out), dtype=jnp.float32))
|
| 24 |
+
return jax.random.uniform(subkey, shape, jnp.float32, -limit, limit)
|
| 25 |
+
|
| 26 |
+
keys = jax.random.split(key, 5)
|
| 27 |
+
params = {
|
| 28 |
+
"conv1": {"kernel": layer(keys[0], (3, 3, 1, 32)), "bias": jnp.zeros((32,), jnp.float32)},
|
| 29 |
+
"conv2": {"kernel": layer(keys[1], (3, 3, 32, 32)), "bias": jnp.zeros((32,), jnp.float32)},
|
| 30 |
+
"conv3": {"kernel": layer(keys[2], (3, 3, 32, 64)), "bias": jnp.zeros((64,), jnp.float32)},
|
| 31 |
+
"conv4": {"kernel": layer(keys[3], (3, 3, 64, 64)), "bias": jnp.zeros((64,), jnp.float32)},
|
| 32 |
+
"dense": {"kernel": layer(keys[4], (64, 10)), "bias": jnp.zeros((10,), jnp.float32)},
|
| 33 |
+
}
|
| 34 |
+
if count_parameters(params) != PARAMETER_COUNT:
|
| 35 |
+
raise AssertionError("fixed CNN parameter count mismatch")
|
| 36 |
+
return params
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def forward(params, images, *, return_shapes: bool = False):
|
| 40 |
+
jax, jnp = _jax()
|
| 41 |
+
x = jnp.asarray(images, dtype=jnp.float32)
|
| 42 |
+
if x.ndim != 4 or x.shape[-1] != 1 or x.shape[1] not in (28, 60) or x.shape[2] != x.shape[1]:
|
| 43 |
+
raise ValueError("images must be float32 [B,28,28,1] or [B,60,60,1]")
|
| 44 |
+
shapes = [tuple(x.shape)]
|
| 45 |
+
|
| 46 |
+
def conv(value, layer):
|
| 47 |
+
value = jax.lax.conv_general_dilated(
|
| 48 |
+
value,
|
| 49 |
+
params[layer]["kernel"],
|
| 50 |
+
window_strides=(1, 1),
|
| 51 |
+
padding="SAME",
|
| 52 |
+
dimension_numbers=("NHWC", "HWIO", "NHWC"),
|
| 53 |
+
)
|
| 54 |
+
return jax.nn.relu(value + params[layer]["bias"])
|
| 55 |
+
|
| 56 |
+
x = conv(x, "conv1")
|
| 57 |
+
x = conv(x, "conv2")
|
| 58 |
+
shapes.append(tuple(x.shape))
|
| 59 |
+
x = jax.lax.reduce_window(
|
| 60 |
+
x,
|
| 61 |
+
-jnp.inf,
|
| 62 |
+
jax.lax.max,
|
| 63 |
+
window_dimensions=(1, 2, 2, 1),
|
| 64 |
+
window_strides=(1, 2, 2, 1),
|
| 65 |
+
padding="VALID",
|
| 66 |
+
)
|
| 67 |
+
shapes.append(tuple(x.shape))
|
| 68 |
+
x = conv(x, "conv3")
|
| 69 |
+
x = conv(x, "conv4")
|
| 70 |
+
x = jnp.mean(x, axis=(1, 2))
|
| 71 |
+
logits = x @ params["dense"]["kernel"] + params["dense"]["bias"]
|
| 72 |
+
return (logits, shapes) if return_shapes else logits
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def count_parameters(params) -> int:
|
| 76 |
+
jax, _ = _jax()
|
| 77 |
+
return sum(int(leaf.size) for leaf in jax.tree_util.tree_leaves(params))
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def kernel_weight_decay_mask(params):
|
| 81 |
+
jax, _ = _jax()
|
| 82 |
+
flat, structure = jax.tree_util.tree_flatten_with_path(params)
|
| 83 |
+
leaves = [bool(path and getattr(path[-1], "key", None) == "kernel") for path, _ in flat]
|
| 84 |
+
return jax.tree_util.tree_unflatten(structure, leaves)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def padded_masked_batches(images, labels, *, batch_size: int = 128):
|
| 88 |
+
"""Yield fixed-size batches and real-example masks for exact loss normalization."""
|
| 89 |
+
_, jnp = _jax()
|
| 90 |
+
images, labels = jnp.asarray(images), jnp.asarray(labels)
|
| 91 |
+
if len(images) != len(labels):
|
| 92 |
+
raise ValueError("images and labels length mismatch")
|
| 93 |
+
for start in range(0, len(images), batch_size):
|
| 94 |
+
stop = min(start + batch_size, len(images))
|
| 95 |
+
real = stop - start
|
| 96 |
+
pad = batch_size - real
|
| 97 |
+
yield (
|
| 98 |
+
jnp.pad(images[start:stop], ((0, pad), (0, 0), (0, 0), (0, 0))),
|
| 99 |
+
jnp.pad(labels[start:stop], ((0, pad),)),
|
| 100 |
+
jnp.arange(batch_size) < real,
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def masked_sparse_cross_entropy(logits, labels, mask):
|
| 105 |
+
jax, jnp = _jax()
|
| 106 |
+
log_probs = jax.nn.log_softmax(logits)
|
| 107 |
+
losses = -jnp.take_along_axis(log_probs, labels[:, None], axis=-1)[:, 0]
|
| 108 |
+
weights = jnp.asarray(mask, jnp.float32)
|
| 109 |
+
return jnp.sum(losses * weights) / jnp.maximum(jnp.sum(weights), 1)
|
src/repro_control/cnn_training.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fixed-final-epoch trainer for the registered CNN family."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
from .checkpoints import save_checkpoint_atomic
|
| 8 |
+
from .cnn import forward, init_params, kernel_weight_decay_mask, masked_sparse_cross_entropy
|
| 9 |
+
from .configs import verify_before_optimizer
|
| 10 |
+
from .hashing import atomic_write_json
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def train_fixed_cnn(
|
| 14 |
+
data_dir: Path,
|
| 15 |
+
output_dir: Path,
|
| 16 |
+
*,
|
| 17 |
+
seed: int,
|
| 18 |
+
registered_config: dict,
|
| 19 |
+
registered_config_sha256: str,
|
| 20 |
+
) -> None:
|
| 21 |
+
import jax
|
| 22 |
+
import jax.numpy as jnp
|
| 23 |
+
import numpy as np
|
| 24 |
+
import optax
|
| 25 |
+
|
| 26 |
+
from sheaf_admm.data import ImageDataset
|
| 27 |
+
|
| 28 |
+
verify_before_optimizer(
|
| 29 |
+
registered_config,
|
| 30 |
+
registered_config,
|
| 31 |
+
expected_sha256=registered_config_sha256,
|
| 32 |
+
)
|
| 33 |
+
if registered_config["training"]["seed"] != seed:
|
| 34 |
+
raise ValueError("CNN seed differs from the registered config")
|
| 35 |
+
if Path(registered_config["data"]["dir"]) != data_dir:
|
| 36 |
+
raise ValueError("CNN data path differs from the registered config")
|
| 37 |
+
dataset = ImageDataset(data_dir, "train")
|
| 38 |
+
if dataset.num_examples != 60_000:
|
| 39 |
+
raise ValueError("registered CNN requires the full 60,000-example train set")
|
| 40 |
+
images = np.asarray(dataset.images, dtype=np.float32)
|
| 41 |
+
labels = np.asarray(dataset.labels, dtype=np.int32)
|
| 42 |
+
if images.shape[1:] != (28, 28, 1) or np.min(images) < 0 or np.max(images) > 1:
|
| 43 |
+
raise ValueError("registered CNN input must be [60000,28,28,1] float32 in [0,1]")
|
| 44 |
+
|
| 45 |
+
key = jax.random.PRNGKey(seed)
|
| 46 |
+
params = init_params(key)
|
| 47 |
+
schedule = optax.join_schedules(
|
| 48 |
+
[optax.linear_schedule(0.0, 1e-3, 200), optax.constant_schedule(1e-3)],
|
| 49 |
+
[200],
|
| 50 |
+
)
|
| 51 |
+
optimizer = optax.chain(
|
| 52 |
+
optax.clip_by_global_norm(1.0),
|
| 53 |
+
optax.adamw(
|
| 54 |
+
schedule,
|
| 55 |
+
b1=0.9,
|
| 56 |
+
b2=0.999,
|
| 57 |
+
eps=1e-8,
|
| 58 |
+
weight_decay=1e-7,
|
| 59 |
+
mask=kernel_weight_decay_mask(params),
|
| 60 |
+
),
|
| 61 |
+
)
|
| 62 |
+
optimizer_state = optimizer.init(params)
|
| 63 |
+
ema = jax.tree_util.tree_map(jnp.copy, params)
|
| 64 |
+
|
| 65 |
+
@jax.jit
|
| 66 |
+
def step(params, optimizer_state, ema, batch_images, batch_labels, real_mask):
|
| 67 |
+
loss, grads = jax.value_and_grad(
|
| 68 |
+
lambda p: masked_sparse_cross_entropy(
|
| 69 |
+
forward(p, batch_images), batch_labels, real_mask
|
| 70 |
+
)
|
| 71 |
+
)(params)
|
| 72 |
+
updates, optimizer_state = optimizer.update(grads, optimizer_state, params)
|
| 73 |
+
params = optax.apply_updates(params, updates)
|
| 74 |
+
ema = jax.tree_util.tree_map(lambda e, p: 0.999 * e + 0.001 * p, ema, params)
|
| 75 |
+
return params, optimizer_state, ema, loss
|
| 76 |
+
|
| 77 |
+
history = []
|
| 78 |
+
for epoch in range(20):
|
| 79 |
+
permutation = np.asarray(jax.random.permutation(jax.random.fold_in(key, epoch), 60_000))
|
| 80 |
+
epoch_losses = []
|
| 81 |
+
for start in range(0, 60_000, 128):
|
| 82 |
+
indices = permutation[start : start + 128]
|
| 83 |
+
real = len(indices)
|
| 84 |
+
if real < 128:
|
| 85 |
+
indices = np.pad(indices, (0, 128 - real), constant_values=0)
|
| 86 |
+
mask = np.arange(128) < real
|
| 87 |
+
params, optimizer_state, ema, loss = step(
|
| 88 |
+
params,
|
| 89 |
+
optimizer_state,
|
| 90 |
+
ema,
|
| 91 |
+
images[indices],
|
| 92 |
+
labels[indices],
|
| 93 |
+
mask,
|
| 94 |
+
)
|
| 95 |
+
epoch_losses.append(float(loss))
|
| 96 |
+
history.append({"epoch": epoch, "mean_train_loss": float(np.mean(epoch_losses))})
|
| 97 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 98 |
+
save_checkpoint_atomic(
|
| 99 |
+
output_dir / "checkpoint.partial.pkl",
|
| 100 |
+
{
|
| 101 |
+
"params": jax.device_get(params),
|
| 102 |
+
"ema_params": jax.device_get(ema),
|
| 103 |
+
"optimizer_state": jax.device_get(optimizer_state),
|
| 104 |
+
"seed": seed,
|
| 105 |
+
"next_epoch": epoch + 1,
|
| 106 |
+
"model": "mnist_cnn_repro",
|
| 107 |
+
},
|
| 108 |
+
)
|
| 109 |
+
atomic_write_json(output_dir / "history.partial.json", history)
|
| 110 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 111 |
+
checkpoint = {
|
| 112 |
+
"params": jax.device_get(params),
|
| 113 |
+
"ema_params": jax.device_get(ema),
|
| 114 |
+
"optimizer_state": jax.device_get(optimizer_state),
|
| 115 |
+
"seed": seed,
|
| 116 |
+
"final_epoch": 19,
|
| 117 |
+
"model": "mnist_cnn_repro",
|
| 118 |
+
}
|
| 119 |
+
save_checkpoint_atomic(output_dir / "checkpoint.pkl", checkpoint)
|
| 120 |
+
atomic_write_json(output_dir / "history.json", history)
|
src/repro_control/configs.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exact registered config expansion and pre-optimizer verification."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import copy
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
from .hashing import compact_json_bytes, sha256_bytes
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ConfigError(ValueError):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _set_path(value: dict[str, Any], dotted: str, replacement: Any) -> None:
|
| 18 |
+
parts = dotted.split(".")
|
| 19 |
+
cursor = value
|
| 20 |
+
for part in parts[:-1]:
|
| 21 |
+
child = cursor.get(part)
|
| 22 |
+
if not isinstance(child, dict):
|
| 23 |
+
raise ConfigError(f"override path does not resolve: {dotted}")
|
| 24 |
+
cursor = child
|
| 25 |
+
if parts[-1] not in cursor:
|
| 26 |
+
raise ConfigError(f"override targets unknown key: {dotted}")
|
| 27 |
+
cursor[parts[-1]] = replacement
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def expand_exact(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
|
| 31 |
+
result = copy.deepcopy(base)
|
| 32 |
+
for dotted in sorted(overrides):
|
| 33 |
+
_set_path(result, dotted, overrides[dotted])
|
| 34 |
+
return result
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def config_sha256(value: dict[str, Any]) -> str:
|
| 38 |
+
return sha256_bytes(compact_json_bytes(value))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def verify_before_optimizer(
|
| 42 |
+
resolved: dict[str, Any],
|
| 43 |
+
expected: dict[str, Any],
|
| 44 |
+
*,
|
| 45 |
+
expected_sha256: str,
|
| 46 |
+
) -> None:
|
| 47 |
+
if resolved != expected:
|
| 48 |
+
raise ConfigError("resolved config differs from the frozen expansion")
|
| 49 |
+
if config_sha256(resolved) != expected_sha256:
|
| 50 |
+
raise ConfigError("resolved config hash differs from the frozen expansion")
|
| 51 |
+
training = resolved.get("training", {})
|
| 52 |
+
data = resolved.get("data", {})
|
| 53 |
+
required = {
|
| 54 |
+
"batch_size": 128,
|
| 55 |
+
"warmup_steps": 200,
|
| 56 |
+
"grad_clip": 1.0,
|
| 57 |
+
"ema_decay": 0.999,
|
| 58 |
+
}
|
| 59 |
+
for name, expected_value in required.items():
|
| 60 |
+
if training.get(name) != expected_value:
|
| 61 |
+
raise ConfigError(f"training.{name} must equal {expected_value}")
|
| 62 |
+
if data.get("val_splits") != []:
|
| 63 |
+
raise ConfigError("training configuration must have data.val_splits=[]")
|
| 64 |
+
if resolved.get("dtype") != "float32":
|
| 65 |
+
raise ConfigError("scientific runs require float32")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def load_json_config(path: Path) -> dict[str, Any]:
|
| 69 |
+
value = json.loads(path.read_text())
|
| 70 |
+
if not isinstance(value, dict):
|
| 71 |
+
raise ConfigError("config root must be an object")
|
| 72 |
+
return value
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def compose_hydra_config(
|
| 76 |
+
project_root: Path,
|
| 77 |
+
*,
|
| 78 |
+
experiment: str,
|
| 79 |
+
seed: int,
|
| 80 |
+
data_dir: str,
|
| 81 |
+
overrides: tuple[str, ...] = (),
|
| 82 |
+
) -> dict[str, Any]:
|
| 83 |
+
"""Compose the exact public Hydra application config without running training."""
|
| 84 |
+
from hydra import compose, initialize_config_dir
|
| 85 |
+
from omegaconf import OmegaConf
|
| 86 |
+
|
| 87 |
+
config_dir = (project_root / "configs").resolve()
|
| 88 |
+
hydra_overrides = [
|
| 89 |
+
f"+experiment={experiment}",
|
| 90 |
+
f"training.seed={seed}",
|
| 91 |
+
f"data.dir={data_dir}",
|
| 92 |
+
"data.val_splits=[]",
|
| 93 |
+
"wandb.mode=disabled",
|
| 94 |
+
*overrides,
|
| 95 |
+
]
|
| 96 |
+
with initialize_config_dir(config_dir=str(config_dir), version_base=None):
|
| 97 |
+
cfg = compose(config_name="config", overrides=hydra_overrides)
|
| 98 |
+
value = OmegaConf.to_container(cfg, resolve=True)
|
| 99 |
+
if not isinstance(value, dict):
|
| 100 |
+
raise ConfigError("Hydra expansion did not produce an object")
|
| 101 |
+
return value
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def cnn_registered_config(*, seed: int, data_dir: str) -> dict[str, Any]:
|
| 105 |
+
"""Exact non-Hydra config consumed by the fixed MNIST CNN trainer."""
|
| 106 |
+
return {
|
| 107 |
+
"task": "mnist",
|
| 108 |
+
"model_type": "mnist_cnn_repro",
|
| 109 |
+
"dtype": "float32",
|
| 110 |
+
"wandb": {"mode": "disabled"},
|
| 111 |
+
"training": {
|
| 112 |
+
"seed": seed,
|
| 113 |
+
"lr": 0.001,
|
| 114 |
+
"weight_decay": 0.0000001,
|
| 115 |
+
"epochs": 20,
|
| 116 |
+
"batch_size": 128,
|
| 117 |
+
"warmup_steps": 200,
|
| 118 |
+
"grad_clip": 1.0,
|
| 119 |
+
"ema_decay": 0.999,
|
| 120 |
+
"betas": [0.9, 0.999],
|
| 121 |
+
"epsilon": 0.00000001,
|
| 122 |
+
"schedule": "linear_to_task_lr_then_constant",
|
| 123 |
+
"epoch_permutation": "fold_in(PRNGKey(seed),epoch)",
|
| 124 |
+
"final_batch": "pad_to_128_mask_normalize",
|
| 125 |
+
"selection": "fixed_final_epoch",
|
| 126 |
+
},
|
| 127 |
+
"data": {
|
| 128 |
+
"dir": data_dir,
|
| 129 |
+
"train_split": "train",
|
| 130 |
+
"val_splits": [],
|
| 131 |
+
"loader": "image",
|
| 132 |
+
"examples": 60000,
|
| 133 |
+
"normalization": False,
|
| 134 |
+
"augmentation": False,
|
| 135 |
+
"input_range": [0, 1],
|
| 136 |
+
},
|
| 137 |
+
"model": {
|
| 138 |
+
"layout": "NHWC/HWIO/NHWC",
|
| 139 |
+
"parameter_count": 65642,
|
| 140 |
+
"pre_pool_sizes": [28, 60],
|
| 141 |
+
"post_pool_sizes": [14, 30],
|
| 142 |
+
},
|
| 143 |
+
"task_cfg": {},
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def registered_config_for(
|
| 148 |
+
project_root: Path,
|
| 149 |
+
logical_id: str,
|
| 150 |
+
*,
|
| 151 |
+
data_root: str = "/data/train",
|
| 152 |
+
) -> dict[str, Any]:
|
| 153 |
+
"""Expand one of the exactly fifteen registered scientific identities."""
|
| 154 |
+
try:
|
| 155 |
+
seed = int(logical_id.rsplit("-", 1)[1])
|
| 156 |
+
except (IndexError, ValueError) as exc:
|
| 157 |
+
raise ConfigError(f"logical identity has no registered seed: {logical_id}") from exc
|
| 158 |
+
if seed not in (42, 123, 456):
|
| 159 |
+
raise ConfigError(f"unregistered seed in logical identity: {logical_id}")
|
| 160 |
+
|
| 161 |
+
if logical_id.startswith("C1-SUD-MPNN225-"):
|
| 162 |
+
return compose_hydra_config(
|
| 163 |
+
project_root,
|
| 164 |
+
experiment="sudoku_mpnn",
|
| 165 |
+
seed=seed,
|
| 166 |
+
data_dir=f"{data_root}/sudoku_easy",
|
| 167 |
+
overrides=("model.d_v=225", "training.lr=1.7e-3", "training.epochs=10"),
|
| 168 |
+
)
|
| 169 |
+
if logical_id.startswith("C2-MAZE-MPNN84-"):
|
| 170 |
+
return compose_hydra_config(
|
| 171 |
+
project_root,
|
| 172 |
+
experiment="maze_mpnn",
|
| 173 |
+
seed=seed,
|
| 174 |
+
data_dir=f"{data_root}/maze_std3_19px_10k",
|
| 175 |
+
)
|
| 176 |
+
if logical_id.startswith("C3-MNIST-CNN-"):
|
| 177 |
+
return cnn_registered_config(seed=seed, data_dir=f"{data_root}/mnist")
|
| 178 |
+
if logical_id.startswith("C4-SUD-IDENTITY-"):
|
| 179 |
+
return compose_hydra_config(
|
| 180 |
+
project_root,
|
| 181 |
+
experiment="sudoku_sheaf",
|
| 182 |
+
seed=seed,
|
| 183 |
+
data_dir=f"{data_root}/sudoku_easy",
|
| 184 |
+
overrides=("model.rm_init=identity", "+model.rm_constant=true"),
|
| 185 |
+
)
|
| 186 |
+
if logical_id.startswith("C4-MAZE-QUADRATIC-"):
|
| 187 |
+
return compose_hydra_config(
|
| 188 |
+
project_root,
|
| 189 |
+
experiment="maze_sheaf",
|
| 190 |
+
seed=seed,
|
| 191 |
+
data_dir=f"{data_root}/maze_std3_19px_10k",
|
| 192 |
+
overrides=("model.objective_mode=quadratic",),
|
| 193 |
+
)
|
| 194 |
+
raise ConfigError(f"unregistered training logical identity: {logical_id}")
|
src/repro_control/constants.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen public constants baked into every reproduction entrypoint."""
|
| 2 |
+
|
| 3 |
+
SCIENCE_SPEC_SHA256 = "c0ded12138cb7c15a15966294ed9bdb87d3a66d2373a7d90050578dbb973c6fd"
|
| 4 |
+
DEADLINE_UTC = "2026-08-03T11:59:00Z"
|
| 5 |
+
NOT_APPLICABLE = "NOT_APPLICABLE"
|
| 6 |
+
REGISTERED_SEEDS = (42, 123, 456)
|
| 7 |
+
NORMAL_CAP_MICRO_USD = 75_000_000
|
| 8 |
+
GPU_RETRY_RESERVE_MICRO_USD = 10_000_000
|
| 9 |
+
MINIMUM_BALANCE_MICRO_USD = 10_000_000
|
| 10 |
+
MAX_RETURNED_IDS = 29
|
| 11 |
+
MAX_GPU_RETURNED_IDS = 26
|
| 12 |
+
MAX_CPU_RETURNED_IDS = 3
|
src/repro_control/data.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Registered corruption masks, long-path data, and smoke fixture checks."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import random
|
| 8 |
+
from collections.abc import Iterable
|
| 9 |
+
from dataclasses import asdict, dataclass
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
from .hashing import atomic_write_json, compact_json_bytes, sha256_bytes
|
| 13 |
+
|
| 14 |
+
MNIST_AGENT_COUNT = 81
|
| 15 |
+
MNIST_DROP_COUNT = 24
|
| 16 |
+
MNIST_DROPOUT_MASTER_SEED = 21005300
|
| 17 |
+
C5_SIZES = (19, 23, 27, 31, 35, 39)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def derived_seed(*parts: object) -> int:
|
| 21 |
+
material = "\0".join(str(part) for part in parts).encode()
|
| 22 |
+
return int.from_bytes(hashlib.sha256(material).digest()[:8], "big")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def mnist_keep_mask(
|
| 26 |
+
dataset_revision: str,
|
| 27 |
+
example_id: int,
|
| 28 |
+
condition: str = "drop30",
|
| 29 |
+
master_seed: int = MNIST_DROPOUT_MASTER_SEED,
|
| 30 |
+
) -> tuple[bool, ...]:
|
| 31 |
+
if condition != "drop30":
|
| 32 |
+
raise ValueError("registered dropout mask condition is drop30")
|
| 33 |
+
rng = random.Random(derived_seed(dataset_revision, example_id, condition, master_seed))
|
| 34 |
+
removed = frozenset(rng.sample(range(MNIST_AGENT_COUNT), MNIST_DROP_COUNT))
|
| 35 |
+
mask = tuple(index not in removed for index in range(MNIST_AGENT_COUNT))
|
| 36 |
+
if sum(mask) != MNIST_AGENT_COUNT - MNIST_DROP_COUNT:
|
| 37 |
+
raise AssertionError("dropout mask cardinality failure")
|
| 38 |
+
return mask
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def build_mnist_mask_bank(
|
| 42 |
+
dataset_revision: str,
|
| 43 |
+
example_ids: Iterable[int],
|
| 44 |
+
*,
|
| 45 |
+
master_seed: int = MNIST_DROPOUT_MASTER_SEED,
|
| 46 |
+
) -> dict:
|
| 47 |
+
ids = list(example_ids)
|
| 48 |
+
if len(ids) != len(set(ids)):
|
| 49 |
+
raise ValueError("example IDs must be unique")
|
| 50 |
+
entries = [
|
| 51 |
+
{
|
| 52 |
+
"example_id": example_id,
|
| 53 |
+
"keep_mask": mnist_keep_mask(
|
| 54 |
+
dataset_revision,
|
| 55 |
+
example_id,
|
| 56 |
+
master_seed=master_seed,
|
| 57 |
+
),
|
| 58 |
+
}
|
| 59 |
+
for example_id in ids
|
| 60 |
+
]
|
| 61 |
+
serializable = [
|
| 62 |
+
{"example_id": row["example_id"], "keep_mask": [int(value) for value in row["keep_mask"]]}
|
| 63 |
+
for row in entries
|
| 64 |
+
]
|
| 65 |
+
return {
|
| 66 |
+
"format": 1,
|
| 67 |
+
"dataset_revision": dataset_revision,
|
| 68 |
+
"condition": "drop30",
|
| 69 |
+
"master_seed": master_seed,
|
| 70 |
+
"entries": serializable,
|
| 71 |
+
"bank_sha256": sha256_bytes(compact_json_bytes(serializable)),
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def write_mnist_mask_bank(path: Path, bank: dict) -> None:
|
| 76 |
+
for row in bank["entries"]:
|
| 77 |
+
if len(row["keep_mask"]) != MNIST_AGENT_COUNT or sum(row["keep_mask"]) != 57:
|
| 78 |
+
raise ValueError("invalid registered mask bank")
|
| 79 |
+
atomic_write_json(path, bank)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def apply_patch_dropout(images, keep_masks):
|
| 83 |
+
"""Apply 9x9-agent 3x3 patch dropout; NumPy is imported only when invoked."""
|
| 84 |
+
import numpy as np
|
| 85 |
+
|
| 86 |
+
array = np.asarray(images)
|
| 87 |
+
masks = np.asarray(keep_masks, dtype=bool)
|
| 88 |
+
if array.ndim != 4 or array.shape[1:] != (28, 28, 1):
|
| 89 |
+
raise ValueError("images must have shape [B,28,28,1]")
|
| 90 |
+
if masks.shape != (array.shape[0], 81) or not np.all(masks.sum(axis=1) == 57):
|
| 91 |
+
raise ValueError("keep masks must be [B,81] with exactly 57 kept agents")
|
| 92 |
+
padded = np.pad(array, ((0, 0), (0, 2), (0, 2), (0, 0)))
|
| 93 |
+
result = padded.copy()
|
| 94 |
+
for agent in range(81):
|
| 95 |
+
row, col = divmod(agent, 9)
|
| 96 |
+
result[:, row * 3 : row * 3 + 3, col * 3 : col * 3 + 3, :] *= masks[
|
| 97 |
+
:, agent, None, None, None
|
| 98 |
+
]
|
| 99 |
+
return result[:, :28, :28, :]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def kept_edge_mask(edge_indices, keep_masks):
|
| 103 |
+
"""Remove all incident edges for dropped agents."""
|
| 104 |
+
import numpy as np
|
| 105 |
+
|
| 106 |
+
edges = np.asarray(edge_indices)
|
| 107 |
+
masks = np.asarray(keep_masks, dtype=bool)
|
| 108 |
+
if edges.ndim != 2 or edges.shape[1] != 2:
|
| 109 |
+
raise ValueError("edge_indices must have shape [E,2]")
|
| 110 |
+
return masks[:, edges[:, 0]] & masks[:, edges[:, 1]]
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@dataclass(frozen=True)
|
| 114 |
+
class C5Unit:
|
| 115 |
+
size: int
|
| 116 |
+
generator_seed: int
|
| 117 |
+
min_path_length: int
|
| 118 |
+
examples: int = 1000
|
| 119 |
+
iterations: int = 100
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def c5_units() -> tuple[C5Unit, ...]:
|
| 123 |
+
return tuple(
|
| 124 |
+
C5Unit(
|
| 125 |
+
size=size,
|
| 126 |
+
generator_seed=21005000 + size,
|
| 127 |
+
min_path_length=3 * (size - 1) // 2,
|
| 128 |
+
)
|
| 129 |
+
for size in C5_SIZES
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def canonical_maze_identity(inputs, labels) -> str:
|
| 134 |
+
"""Hash canonical wall/start/goal/path bytes for overlap rejection."""
|
| 135 |
+
import numpy as np
|
| 136 |
+
|
| 137 |
+
inputs_array = np.asarray(inputs, dtype=np.uint8).reshape(-1)
|
| 138 |
+
labels_array = np.asarray(labels, dtype=np.uint8).reshape(-1)
|
| 139 |
+
return sha256_bytes(inputs_array.tobytes() + b"\0" + labels_array.tobytes())
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def generate_c5_size(
|
| 143 |
+
size: int,
|
| 144 |
+
*,
|
| 145 |
+
training_identities: frozenset[str],
|
| 146 |
+
examples: int = 1000,
|
| 147 |
+
):
|
| 148 |
+
"""Generate unique registered long-path examples using the released DFS builder."""
|
| 149 |
+
import numpy as np
|
| 150 |
+
|
| 151 |
+
from sheaf_admm.data.build_maze import (
|
| 152 |
+
_backtrack_maze,
|
| 153 |
+
_paint_grid,
|
| 154 |
+
_pick_positions,
|
| 155 |
+
_shortest_path,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
if size not in C5_SIZES:
|
| 159 |
+
raise ValueError(f"unregistered C5 size {size}")
|
| 160 |
+
rng = np.random.default_rng(21005000 + size)
|
| 161 |
+
minimum = 3 * (size - 1) // 2
|
| 162 |
+
seen = set(training_identities)
|
| 163 |
+
inputs, labels = [], []
|
| 164 |
+
attempts = 0
|
| 165 |
+
while len(inputs) < examples:
|
| 166 |
+
attempts += 1
|
| 167 |
+
if attempts > examples * 10000:
|
| 168 |
+
raise RuntimeError("C5 generator exhausted deterministic replacement guard")
|
| 169 |
+
base = _backtrack_maze(size, size, rng)
|
| 170 |
+
start, goal = _pick_positions(base, rng)
|
| 171 |
+
path = _shortest_path(base, start, goal)
|
| 172 |
+
if len(path) - 1 < minimum:
|
| 173 |
+
continue
|
| 174 |
+
input_grid = _paint_grid(base, start, goal, path, with_path=False)
|
| 175 |
+
label_grid = _paint_grid(base, start, goal, path, with_path=True)
|
| 176 |
+
identity = canonical_maze_identity(input_grid, label_grid)
|
| 177 |
+
if identity in seen:
|
| 178 |
+
continue
|
| 179 |
+
seen.add(identity)
|
| 180 |
+
inputs.append(input_grid.reshape(-1).astype(np.uint8))
|
| 181 |
+
labels.append(label_grid.reshape(-1).astype(np.uint8))
|
| 182 |
+
arrays = {"inputs": np.stack(inputs), "labels": np.stack(labels)}
|
| 183 |
+
manifest = {
|
| 184 |
+
"format": 1,
|
| 185 |
+
"unit": asdict(
|
| 186 |
+
C5Unit(
|
| 187 |
+
size=size,
|
| 188 |
+
generator_seed=21005000 + size,
|
| 189 |
+
min_path_length=minimum,
|
| 190 |
+
examples=examples,
|
| 191 |
+
)
|
| 192 |
+
),
|
| 193 |
+
"attempts": attempts,
|
| 194 |
+
"identity_root_sha256": sha256_bytes(
|
| 195 |
+
compact_json_bytes(
|
| 196 |
+
[canonical_maze_identity(x, y) for x, y in zip(inputs, labels, strict=True)]
|
| 197 |
+
)
|
| 198 |
+
),
|
| 199 |
+
}
|
| 200 |
+
return arrays, manifest
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def deterministic_lpt(costs: dict[int, float]) -> dict[str, list[int]]:
|
| 204 |
+
if set(costs) != set(C5_SIZES) or any(value <= 0 for value in costs.values()):
|
| 205 |
+
raise ValueError("costs must cover all registered sizes with positive values")
|
| 206 |
+
shards: dict[str, list[int]] = {"A": [], "B": []}
|
| 207 |
+
totals = {"A": 0.0, "B": 0.0}
|
| 208 |
+
for size in sorted(C5_SIZES, key=lambda n: (-costs[n], n)):
|
| 209 |
+
shard = "A" if totals["A"] <= totals["B"] else "B"
|
| 210 |
+
shards[shard].append(size)
|
| 211 |
+
totals[shard] += costs[size]
|
| 212 |
+
return shards
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def validate_smoke_fixture_registry(registry_path: Path) -> None:
|
| 216 |
+
registry = json.loads(registry_path.read_text())
|
| 217 |
+
if registry["purpose"].lower().find("verdict metrics are forbidden") < 0:
|
| 218 |
+
raise ValueError("smoke fixture registry must forbid verdict metrics")
|
| 219 |
+
final_seeds = {21005300, *(21005000 + n for n in C5_SIZES)}
|
| 220 |
+
smoke_seeds = {91005101, 91005301, *(91005200 + n for n in C5_SIZES)}
|
| 221 |
+
if final_seeds & smoke_seeds:
|
| 222 |
+
raise ValueError("smoke and final generator seeds overlap")
|
| 223 |
+
for row in registry["entries"]:
|
| 224 |
+
if not row["path"].startswith("smoke-fixtures/"):
|
| 225 |
+
raise ValueError("unregistered smoke fixture path")
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def write_puzzle_split(
|
| 229 |
+
root: Path,
|
| 230 |
+
split: str,
|
| 231 |
+
inputs,
|
| 232 |
+
labels,
|
| 233 |
+
*,
|
| 234 |
+
height: int,
|
| 235 |
+
width: int,
|
| 236 |
+
) -> None:
|
| 237 |
+
"""Write one-example-per-puzzle arrays in the released PuzzleDataset format."""
|
| 238 |
+
import numpy as np
|
| 239 |
+
|
| 240 |
+
from sheaf_admm.data.common import PuzzleDatasetMetadata, save_npy
|
| 241 |
+
|
| 242 |
+
input_array = np.asarray(inputs, dtype=np.uint8)
|
| 243 |
+
label_array = np.asarray(labels, dtype=np.uint8)
|
| 244 |
+
if (
|
| 245 |
+
input_array.ndim != 2
|
| 246 |
+
or input_array.shape != label_array.shape
|
| 247 |
+
or input_array.shape[1] != height * width
|
| 248 |
+
):
|
| 249 |
+
raise ValueError("puzzle arrays do not match the registered board dimensions")
|
| 250 |
+
count = len(input_array)
|
| 251 |
+
split_root = root / split
|
| 252 |
+
offsets = np.arange(count + 1, dtype=np.int32)
|
| 253 |
+
save_npy(split_root / "all__inputs.npy", input_array)
|
| 254 |
+
save_npy(split_root / "all__labels.npy", label_array)
|
| 255 |
+
save_npy(split_root / "all__puzzle_indices.npy", offsets)
|
| 256 |
+
save_npy(split_root / "all__group_indices.npy", offsets)
|
| 257 |
+
save_npy(split_root / "all__puzzle_identifiers.npy", np.zeros(count, dtype=np.int32))
|
| 258 |
+
PuzzleDatasetMetadata(
|
| 259 |
+
pad_id=0,
|
| 260 |
+
ignore_label_id=0,
|
| 261 |
+
blank_identifier_id=0,
|
| 262 |
+
vocab_size=6,
|
| 263 |
+
seq_len=height * width,
|
| 264 |
+
num_puzzle_identifiers=1,
|
| 265 |
+
total_groups=count,
|
| 266 |
+
mean_puzzle_examples=1.0,
|
| 267 |
+
total_puzzles=count,
|
| 268 |
+
sets=["all"],
|
| 269 |
+
height=height,
|
| 270 |
+
width=width,
|
| 271 |
+
).write(split_root / "dataset.json")
|
src/repro_control/evaluation.py
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""EMA-only evaluation backends for fused logical suites and C5 shards."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import pickle
|
| 7 |
+
from collections.abc import Iterator
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
from .cnn import forward as cnn_forward
|
| 12 |
+
from .data import apply_patch_dropout
|
| 13 |
+
from .hashing import atomic_write_json, compact_json_bytes, sha256_bytes, sha256_file
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class EvaluationError(ValueError):
|
| 17 |
+
pass
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _load_pickle(path: Path) -> dict[str, Any]:
|
| 21 |
+
with path.open("rb") as handle:
|
| 22 |
+
value = pickle.load(handle)
|
| 23 |
+
if handle.read(1):
|
| 24 |
+
raise EvaluationError("checkpoint has trailing bytes")
|
| 25 |
+
if not isinstance(value, dict) or value.get("ema_params") is None:
|
| 26 |
+
raise EvaluationError("evaluation requires a checkpoint with EMA parameters")
|
| 27 |
+
return value
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _puzzle_batches(dataset_dir: Path, split: str, batch_size: int):
|
| 31 |
+
import numpy as np
|
| 32 |
+
|
| 33 |
+
from sheaf_admm.data import PuzzleDataset
|
| 34 |
+
|
| 35 |
+
dataset = PuzzleDataset(dataset_dir, split)
|
| 36 |
+
for _set, batch in dataset.iter_test_batches(batch_size):
|
| 37 |
+
out = {"inputs": np.asarray(batch["inputs"]), "labels": np.asarray(batch["labels"])}
|
| 38 |
+
for name in ("height", "width"):
|
| 39 |
+
if name in batch:
|
| 40 |
+
out[name] = batch[name]
|
| 41 |
+
yield out
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _fixed_puzzle_batches(
|
| 45 |
+
dataset_dir: Path,
|
| 46 |
+
split: str,
|
| 47 |
+
batch_size: int,
|
| 48 |
+
) -> Iterator[tuple[dict[str, Any], int]]:
|
| 49 |
+
import numpy as np
|
| 50 |
+
|
| 51 |
+
from sheaf_admm.data import PuzzleDataset
|
| 52 |
+
|
| 53 |
+
dataset = PuzzleDataset(dataset_dir, split)
|
| 54 |
+
if set(dataset.sets) != {"all"}:
|
| 55 |
+
raise EvaluationError("registered C5 split requires exactly the all set")
|
| 56 |
+
puzzle_set = dataset.sets["all"]
|
| 57 |
+
for start in range(0, puzzle_set.num_examples, batch_size):
|
| 58 |
+
stop = min(start + batch_size, puzzle_set.num_examples)
|
| 59 |
+
real = stop - start
|
| 60 |
+
inputs = np.asarray(puzzle_set.inputs[start:stop])
|
| 61 |
+
labels = np.asarray(puzzle_set.labels[start:stop])
|
| 62 |
+
if real < batch_size:
|
| 63 |
+
inputs = np.pad(inputs, ((0, batch_size - real), (0, 0)))
|
| 64 |
+
labels = np.pad(labels, ((0, batch_size - real), (0, 0)))
|
| 65 |
+
yield (
|
| 66 |
+
{
|
| 67 |
+
"inputs": inputs,
|
| 68 |
+
"labels": labels,
|
| 69 |
+
"height": dataset.metadata.height,
|
| 70 |
+
"width": dataset.metadata.width,
|
| 71 |
+
},
|
| 72 |
+
real,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _load_mask_bank(path: Path, expected_sha256: str) -> dict[int, list[int]]:
|
| 77 |
+
value = json.loads(path.read_text())
|
| 78 |
+
if value.get("bank_sha256") != expected_sha256:
|
| 79 |
+
raise EvaluationError("MNIST mask bank hash differs from the unit manifest")
|
| 80 |
+
entries = value.get("entries")
|
| 81 |
+
if not isinstance(entries, list):
|
| 82 |
+
raise EvaluationError("invalid MNIST mask bank")
|
| 83 |
+
if sha256_bytes(compact_json_bytes(entries)) != expected_sha256:
|
| 84 |
+
raise EvaluationError("MNIST mask bank content hash mismatch")
|
| 85 |
+
bank = {int(row["example_id"]): row["keep_mask"] for row in entries}
|
| 86 |
+
if len(bank) != len(entries):
|
| 87 |
+
raise EvaluationError("duplicate MNIST mask example ID")
|
| 88 |
+
return bank
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _image_batches(
|
| 92 |
+
dataset_dir: Path,
|
| 93 |
+
split: str,
|
| 94 |
+
batch_size: int,
|
| 95 |
+
*,
|
| 96 |
+
condition: str,
|
| 97 |
+
mask_bank_path: str = "",
|
| 98 |
+
mask_bank_sha256: str = "",
|
| 99 |
+
) -> Iterator[dict[str, Any]]:
|
| 100 |
+
import numpy as np
|
| 101 |
+
|
| 102 |
+
from sheaf_admm.data import ImageDataset
|
| 103 |
+
|
| 104 |
+
dataset = ImageDataset(dataset_dir, split)
|
| 105 |
+
mask_bank = (
|
| 106 |
+
_load_mask_bank(Path(mask_bank_path), mask_bank_sha256)
|
| 107 |
+
if condition == "drop30"
|
| 108 |
+
else None
|
| 109 |
+
)
|
| 110 |
+
for start in range(0, dataset.num_examples, batch_size):
|
| 111 |
+
stop = min(start + batch_size, dataset.num_examples)
|
| 112 |
+
images = np.asarray(dataset.images[start:stop], dtype=np.float32)
|
| 113 |
+
labels = np.asarray(dataset.labels[start:stop])
|
| 114 |
+
if images.shape[1:] != (28, 28, 1):
|
| 115 |
+
raise EvaluationError("registered MNIST source split must be clean 28x28")
|
| 116 |
+
batch: dict[str, Any] = {"images": images, "labels": labels}
|
| 117 |
+
if condition == "pad16":
|
| 118 |
+
batch["images"] = np.pad(images, ((0, 0), (16, 16), (16, 16), (0, 0)))
|
| 119 |
+
elif condition == "drop30":
|
| 120 |
+
assert mask_bank is not None
|
| 121 |
+
masks = np.asarray([mask_bank[index] for index in range(start, stop)], dtype=bool)
|
| 122 |
+
batch["images"] = apply_patch_dropout(images, masks)
|
| 123 |
+
batch["keep_masks"] = masks
|
| 124 |
+
elif condition != "clean":
|
| 125 |
+
raise EvaluationError(f"unsupported MNIST condition {condition!r}")
|
| 126 |
+
yield batch
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _upstream_backend(unit: dict[str, Any]):
|
| 130 |
+
import jax
|
| 131 |
+
|
| 132 |
+
from sheaf_admm.models import model_config_from_dict
|
| 133 |
+
from sheaf_admm.training import build_model, make_task
|
| 134 |
+
from sheaf_admm.training.loop import _forward
|
| 135 |
+
|
| 136 |
+
checkpoint_path = Path(unit["checkpoint_path"])
|
| 137 |
+
config_path = Path(unit["config_path"])
|
| 138 |
+
checkpoint = _load_pickle(checkpoint_path)
|
| 139 |
+
config = json.loads(config_path.read_text())
|
| 140 |
+
model_config = model_config_from_dict(config["model"])
|
| 141 |
+
model = build_model(model_config, config["model_type"])
|
| 142 |
+
task = make_task(config["task"], **config["task_cfg"])
|
| 143 |
+
params = checkpoint["ema_params"]
|
| 144 |
+
graph_readout = model_config.mpnn_graph_readout
|
| 145 |
+
model_type = config["model_type"]
|
| 146 |
+
k_eval = (
|
| 147 |
+
config["training"]["K_eval"]
|
| 148 |
+
if model_type == "sheaf"
|
| 149 |
+
else config["training"]["mpnn_eval_rounds"]
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
@jax.jit
|
| 153 |
+
def forward(current_params, fwd):
|
| 154 |
+
return _forward(
|
| 155 |
+
model.apply,
|
| 156 |
+
current_params,
|
| 157 |
+
fwd,
|
| 158 |
+
n_iter=k_eval,
|
| 159 |
+
loss_window=1,
|
| 160 |
+
model_type=model_type,
|
| 161 |
+
training=False,
|
| 162 |
+
rng=None,
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
return (
|
| 166 |
+
checkpoint_path,
|
| 167 |
+
config_path,
|
| 168 |
+
params,
|
| 169 |
+
task,
|
| 170 |
+
graph_readout,
|
| 171 |
+
model_type,
|
| 172 |
+
forward,
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def evaluate_upstream_unit(unit: dict[str, Any]) -> dict[str, Any]:
|
| 177 |
+
import jax
|
| 178 |
+
|
| 179 |
+
(
|
| 180 |
+
checkpoint_path,
|
| 181 |
+
config_path,
|
| 182 |
+
params,
|
| 183 |
+
task,
|
| 184 |
+
graph_readout,
|
| 185 |
+
model_type,
|
| 186 |
+
forward,
|
| 187 |
+
) = _upstream_backend(unit)
|
| 188 |
+
|
| 189 |
+
dataset_dir = Path(unit["dataset_dir"])
|
| 190 |
+
config = json.loads(config_path.read_text())
|
| 191 |
+
if config["data"]["loader"] == "image":
|
| 192 |
+
batches = _image_batches(
|
| 193 |
+
dataset_dir,
|
| 194 |
+
unit["split"],
|
| 195 |
+
128,
|
| 196 |
+
condition=unit["condition"],
|
| 197 |
+
mask_bank_path=unit.get("mask_bank_path", ""),
|
| 198 |
+
mask_bank_sha256=unit.get("mask_bank_sha256", ""),
|
| 199 |
+
)
|
| 200 |
+
else:
|
| 201 |
+
batches = _puzzle_batches(dataset_dir, unit["split"], 128)
|
| 202 |
+
totals: dict[str, float] = {}
|
| 203 |
+
count = 0
|
| 204 |
+
for batch in batches:
|
| 205 |
+
fwd, targets, aux = task.prepare(batch)
|
| 206 |
+
logits = forward(params, fwd)
|
| 207 |
+
jax.block_until_ready(logits)
|
| 208 |
+
final = logits if model_type == "mpnn" else logits[-1]
|
| 209 |
+
metrics = (
|
| 210 |
+
task.evaluate_graph(final, targets, aux)
|
| 211 |
+
if model_type == "mpnn" and graph_readout == "graph"
|
| 212 |
+
else task.evaluate(final, targets, aux)
|
| 213 |
+
)
|
| 214 |
+
weight = len(batch["labels"])
|
| 215 |
+
for name, value in metrics.items():
|
| 216 |
+
totals[name] = totals.get(name, 0.0) + float(value) * weight
|
| 217 |
+
count += weight
|
| 218 |
+
if count == 0:
|
| 219 |
+
raise EvaluationError("evaluation unit contained no examples")
|
| 220 |
+
return {
|
| 221 |
+
"neutral_id": unit["neutral_id"],
|
| 222 |
+
"seed": unit["seed"],
|
| 223 |
+
"condition": unit["condition"],
|
| 224 |
+
"examples": count,
|
| 225 |
+
"metrics": {name: value / count for name, value in sorted(totals.items())},
|
| 226 |
+
"checkpoint_sha256": sha256_file(checkpoint_path),
|
| 227 |
+
"config_sha256": sha256_file(config_path),
|
| 228 |
+
"ema": True,
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def evaluate_cnn_unit(unit: dict[str, Any]) -> dict[str, Any]:
|
| 233 |
+
import jax
|
| 234 |
+
import numpy as np
|
| 235 |
+
|
| 236 |
+
checkpoint_path = Path(unit["checkpoint_path"])
|
| 237 |
+
config_path = Path(unit["config_path"])
|
| 238 |
+
checkpoint = _load_pickle(checkpoint_path)
|
| 239 |
+
params = checkpoint["ema_params"]
|
| 240 |
+
run = jax.jit(cnn_forward)
|
| 241 |
+
correct = count = 0
|
| 242 |
+
for batch in _image_batches(
|
| 243 |
+
Path(unit["dataset_dir"]),
|
| 244 |
+
unit["split"],
|
| 245 |
+
128,
|
| 246 |
+
condition=unit["condition"],
|
| 247 |
+
mask_bank_path=unit.get("mask_bank_path", ""),
|
| 248 |
+
mask_bank_sha256=unit.get("mask_bank_sha256", ""),
|
| 249 |
+
):
|
| 250 |
+
logits = run(params, batch["images"])
|
| 251 |
+
predictions = np.asarray(jax.device_get(logits.argmax(axis=-1)))
|
| 252 |
+
labels = np.asarray(batch["labels"])
|
| 253 |
+
correct += int(np.sum(predictions == labels))
|
| 254 |
+
count += len(labels)
|
| 255 |
+
if count == 0:
|
| 256 |
+
raise EvaluationError("CNN evaluation unit contained no examples")
|
| 257 |
+
return {
|
| 258 |
+
"neutral_id": unit["neutral_id"],
|
| 259 |
+
"seed": unit["seed"],
|
| 260 |
+
"condition": unit["condition"],
|
| 261 |
+
"examples": count,
|
| 262 |
+
"metrics": {"acc": correct / count},
|
| 263 |
+
"checkpoint_sha256": sha256_file(checkpoint_path),
|
| 264 |
+
"config_sha256": sha256_file(config_path),
|
| 265 |
+
"ema": True,
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def evaluate_c5_size(units: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
| 270 |
+
import jax
|
| 271 |
+
import jax.numpy as jnp
|
| 272 |
+
|
| 273 |
+
from sheaf_admm.data import views
|
| 274 |
+
from sheaf_admm.models import model_config_from_dict
|
| 275 |
+
from sheaf_admm.training import build_model, make_task
|
| 276 |
+
from sheaf_admm.training.loop import _forward
|
| 277 |
+
|
| 278 |
+
if len(units) != 3 or {unit["seed"] for unit in units} != {42, 123, 456}:
|
| 279 |
+
raise EvaluationError("each C5 size requires exactly the three registered seeds")
|
| 280 |
+
first = units[0]
|
| 281 |
+
config = json.loads(Path(first["config_path"]).read_text())
|
| 282 |
+
model_config = model_config_from_dict(config["model"])
|
| 283 |
+
model = build_model(model_config, "sheaf")
|
| 284 |
+
task = make_task("maze", **config["task_cfg"])
|
| 285 |
+
batch_size = int(first["batch_size"])
|
| 286 |
+
if any(
|
| 287 |
+
int(unit["batch_size"]) != batch_size
|
| 288 |
+
or unit["dataset_dir"] != first["dataset_dir"]
|
| 289 |
+
or unit["split"] != first["split"]
|
| 290 |
+
for unit in units
|
| 291 |
+
):
|
| 292 |
+
raise EvaluationError("C5 size units disagree on data or frozen batch size")
|
| 293 |
+
|
| 294 |
+
@jax.jit
|
| 295 |
+
def forward(params, fwd):
|
| 296 |
+
logits = _forward(
|
| 297 |
+
model.apply,
|
| 298 |
+
params,
|
| 299 |
+
fwd,
|
| 300 |
+
n_iter=100,
|
| 301 |
+
loss_window=1,
|
| 302 |
+
model_type="sheaf",
|
| 303 |
+
training=False,
|
| 304 |
+
rng=None,
|
| 305 |
+
)
|
| 306 |
+
return logits[-1]
|
| 307 |
+
|
| 308 |
+
batches = list(
|
| 309 |
+
_fixed_puzzle_batches(
|
| 310 |
+
Path(first["dataset_dir"]),
|
| 311 |
+
first["split"],
|
| 312 |
+
batch_size,
|
| 313 |
+
)
|
| 314 |
+
)
|
| 315 |
+
if sum(real for _, real in batches) != 1000:
|
| 316 |
+
raise EvaluationError("registered C5 unit must contain exactly 1000 real examples")
|
| 317 |
+
results = []
|
| 318 |
+
for unit in sorted(units, key=lambda row: row["seed"]):
|
| 319 |
+
checkpoint_path = Path(unit["checkpoint_path"])
|
| 320 |
+
config_path = Path(unit["config_path"])
|
| 321 |
+
params = _load_pickle(checkpoint_path)["ema_params"]
|
| 322 |
+
solved_count = cell_correct = cell_count = example_count = 0
|
| 323 |
+
for batch, real in batches:
|
| 324 |
+
fwd, targets, aux = task.prepare(batch)
|
| 325 |
+
logits = forward(params, fwd)
|
| 326 |
+
jax.block_until_ready(logits)
|
| 327 |
+
reconstructed = jnp.asarray(
|
| 328 |
+
views.reassemble_logits(
|
| 329 |
+
logits,
|
| 330 |
+
aux["centers"],
|
| 331 |
+
aux["image_hw"],
|
| 332 |
+
model_config.num_classes,
|
| 333 |
+
mode="mean",
|
| 334 |
+
)
|
| 335 |
+
)
|
| 336 |
+
predictions = jnp.argmax(reconstructed, axis=-1)[:real]
|
| 337 |
+
labels = targets["labels_img"][:real]
|
| 338 |
+
solved = jnp.all(
|
| 339 |
+
(predictions == 5) == (labels == 5),
|
| 340 |
+
axis=(1, 2),
|
| 341 |
+
)
|
| 342 |
+
solved_count += int(jnp.sum(solved))
|
| 343 |
+
cell_correct += int(jnp.sum(predictions == labels))
|
| 344 |
+
cell_count += int(labels.size)
|
| 345 |
+
example_count += real
|
| 346 |
+
results.append(
|
| 347 |
+
{
|
| 348 |
+
"neutral_id": unit["neutral_id"],
|
| 349 |
+
"seed": unit["seed"],
|
| 350 |
+
"condition": "c5",
|
| 351 |
+
"size": unit["size"],
|
| 352 |
+
"examples": example_count,
|
| 353 |
+
"metrics": {
|
| 354 |
+
"solved": solved_count / example_count,
|
| 355 |
+
"cell_acc": cell_correct / cell_count,
|
| 356 |
+
},
|
| 357 |
+
"checkpoint_sha256": sha256_file(checkpoint_path),
|
| 358 |
+
"config_sha256": sha256_file(config_path),
|
| 359 |
+
"ema": True,
|
| 360 |
+
"batch_size": batch_size,
|
| 361 |
+
"padded_tail_excluded": True,
|
| 362 |
+
}
|
| 363 |
+
)
|
| 364 |
+
return results
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def evaluate_units(unit_manifest_path: Path, output_path: Path) -> None:
|
| 368 |
+
manifest = json.loads(unit_manifest_path.read_text())
|
| 369 |
+
if set(manifest) != {"format", "units"} or manifest["format"] != 1:
|
| 370 |
+
raise EvaluationError("invalid evaluator unit manifest")
|
| 371 |
+
identities = [unit["neutral_id"] for unit in manifest["units"]]
|
| 372 |
+
if len(identities) != len(set(identities)):
|
| 373 |
+
raise EvaluationError("duplicate evaluator unit identity")
|
| 374 |
+
results = []
|
| 375 |
+
c5_by_size: dict[int, list[dict[str, Any]]] = {}
|
| 376 |
+
for unit in manifest["units"]:
|
| 377 |
+
if unit["condition"] not in {"clean", "pad16", "drop30", "c5"}:
|
| 378 |
+
raise EvaluationError("unregistered evaluation condition")
|
| 379 |
+
if unit["condition"] == "c5":
|
| 380 |
+
c5_by_size.setdefault(int(unit["size"]), []).append(unit)
|
| 381 |
+
else:
|
| 382 |
+
result = (
|
| 383 |
+
evaluate_cnn_unit(unit)
|
| 384 |
+
if unit["model_kind"] == "mnist_cnn_repro"
|
| 385 |
+
else evaluate_upstream_unit(unit)
|
| 386 |
+
)
|
| 387 |
+
results.append(result)
|
| 388 |
+
for size in sorted(c5_by_size):
|
| 389 |
+
results.extend(evaluate_c5_size(c5_by_size[size]))
|
| 390 |
+
atomic_write_json(
|
| 391 |
+
output_path,
|
| 392 |
+
{
|
| 393 |
+
"format": 1,
|
| 394 |
+
"unit_manifest_sha256": sha256_file(unit_manifest_path),
|
| 395 |
+
"units": results,
|
| 396 |
+
"verdicts": {},
|
| 397 |
+
},
|
| 398 |
+
)
|
src/repro_control/hashing.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Canonical hashing and atomic local writes."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
import tempfile
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any, Iterable
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def sha256_bytes(data: bytes) -> str:
|
| 14 |
+
return hashlib.sha256(data).hexdigest()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def sha256_file(path: Path) -> str:
|
| 18 |
+
digest = hashlib.sha256()
|
| 19 |
+
with path.open("rb") as handle:
|
| 20 |
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
| 21 |
+
digest.update(chunk)
|
| 22 |
+
return digest.hexdigest()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def compact_json_bytes(value: Any) -> bytes:
|
| 26 |
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def canonical_root(entries: Iterable[dict[str, Any]]) -> str:
|
| 30 |
+
rows = sorted(entries, key=lambda row: row["path"])
|
| 31 |
+
return sha256_bytes(compact_json_bytes(rows))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def file_entry(path: Path, *, relative_to: Path) -> dict[str, Any]:
|
| 35 |
+
return {
|
| 36 |
+
"path": path.relative_to(relative_to).as_posix(),
|
| 37 |
+
"bytes": path.stat().st_size,
|
| 38 |
+
"sha256": sha256_file(path),
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def tree_entries(root: Path, *, excluded_names: frozenset[str] = frozenset()) -> list[dict]:
|
| 43 |
+
return [
|
| 44 |
+
file_entry(path, relative_to=root)
|
| 45 |
+
for path in sorted(root.rglob("*"))
|
| 46 |
+
if path.is_file() and not any(part in excluded_names for part in path.relative_to(root).parts)
|
| 47 |
+
]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def atomic_write_bytes(path: Path, data: bytes) -> None:
|
| 51 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 52 |
+
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
| 53 |
+
try:
|
| 54 |
+
with os.fdopen(descriptor, "wb") as handle:
|
| 55 |
+
handle.write(data)
|
| 56 |
+
handle.flush()
|
| 57 |
+
os.fsync(handle.fileno())
|
| 58 |
+
os.replace(temporary, path)
|
| 59 |
+
except BaseException:
|
| 60 |
+
try:
|
| 61 |
+
os.unlink(temporary)
|
| 62 |
+
except FileNotFoundError:
|
| 63 |
+
pass
|
| 64 |
+
raise
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def atomic_write_json(path: Path, value: Any) -> None:
|
| 68 |
+
atomic_write_bytes(path, json.dumps(value, indent=2, sort_keys=True).encode() + b"\n")
|
src/repro_control/heartbeat.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Line-buffered readiness and heartbeat markers for supervised HF Jobs."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import threading
|
| 6 |
+
from collections.abc import Iterator
|
| 7 |
+
from contextlib import contextmanager
|
| 8 |
+
from datetime import UTC, datetime
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def marker(name: str, detail: str = "") -> None:
|
| 12 |
+
suffix = f" {detail}" if detail else ""
|
| 13 |
+
print(f"{name} {datetime.now(UTC).isoformat()}{suffix}", flush=True)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@contextmanager
|
| 17 |
+
def heartbeat(stage: str, *, interval_seconds: int = 60) -> Iterator[None]:
|
| 18 |
+
stop = threading.Event()
|
| 19 |
+
|
| 20 |
+
def emit() -> None:
|
| 21 |
+
while not stop.wait(interval_seconds):
|
| 22 |
+
marker("HEARTBEAT", stage)
|
| 23 |
+
|
| 24 |
+
worker = threading.Thread(target=emit, name=f"heartbeat-{stage}", daemon=True)
|
| 25 |
+
worker.start()
|
| 26 |
+
try:
|
| 27 |
+
yield
|
| 28 |
+
finally:
|
| 29 |
+
stop.set()
|
| 30 |
+
worker.join(timeout=1)
|
src/repro_control/hf_provider.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hugging Face Jobs adapter for the durable direct-launch state machine."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
import re
|
| 7 |
+
from datetime import UTC, datetime
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
from huggingface_hub import HfApi, Volume
|
| 11 |
+
from huggingface_hub.errors import HfHubHTTPError
|
| 12 |
+
from requests import RequestException
|
| 13 |
+
|
| 14 |
+
from .launcher import AmbiguousSubmission, LaunchError
|
| 15 |
+
|
| 16 |
+
MARKER_TIME = re.compile(
|
| 17 |
+
r"\b(?P<marker>CPU_READY|GPU_READY|HEARTBEAT|DONE)\s+"
|
| 18 |
+
r"(?P<time>\d{4}-\d{2}-\d{2}T[^\s]+)"
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _iso(value: Any) -> str:
|
| 23 |
+
if value is None:
|
| 24 |
+
return ""
|
| 25 |
+
if isinstance(value, datetime):
|
| 26 |
+
return value.astimezone(UTC).isoformat()
|
| 27 |
+
return str(value)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class HFJobsProvider:
|
| 31 |
+
def __init__(self, *, namespace: str = "Mindcraft", api: HfApi | None = None):
|
| 32 |
+
self.namespace = namespace
|
| 33 |
+
self.api = api or HfApi()
|
| 34 |
+
|
| 35 |
+
@staticmethod
|
| 36 |
+
def _volumes(rows: list[dict[str, Any]]) -> list[Volume]:
|
| 37 |
+
volumes = []
|
| 38 |
+
for row in rows:
|
| 39 |
+
volumes.append(
|
| 40 |
+
Volume(
|
| 41 |
+
type="bucket",
|
| 42 |
+
source=row["volume_source"],
|
| 43 |
+
path=row["volume_path"] or None,
|
| 44 |
+
mount_path=row["mount_path"],
|
| 45 |
+
read_only=row["read_only"],
|
| 46 |
+
)
|
| 47 |
+
)
|
| 48 |
+
return volumes
|
| 49 |
+
|
| 50 |
+
@staticmethod
|
| 51 |
+
def _assert_no_credentials(request: dict[str, Any]) -> None:
|
| 52 |
+
if request.get("secrets"):
|
| 53 |
+
raise LaunchError("Jobs may not receive provider secrets under TRACE_MODE=none")
|
| 54 |
+
env = request.get("env") or {}
|
| 55 |
+
forbidden = ("TOKEN", "SECRET", "PASSWORD", "AUTHORIZATION", "API_KEY")
|
| 56 |
+
matches = [name for name in env if any(term in name.upper() for term in forbidden)]
|
| 57 |
+
if matches:
|
| 58 |
+
raise LaunchError(f"credential-like Job environment names are forbidden: {matches}")
|
| 59 |
+
|
| 60 |
+
def submit(self, request: dict[str, Any]) -> dict[str, Any]:
|
| 61 |
+
self._assert_no_credentials(request)
|
| 62 |
+
labels = {
|
| 63 |
+
"repro-attempt": request["label"],
|
| 64 |
+
"logical-id": request["logical_id"],
|
| 65 |
+
"job-class": request["job_class"],
|
| 66 |
+
}
|
| 67 |
+
try:
|
| 68 |
+
info = self.api.run_job(
|
| 69 |
+
image=request["image"],
|
| 70 |
+
command=request["command"],
|
| 71 |
+
env=request.get("env") or {},
|
| 72 |
+
secrets=None,
|
| 73 |
+
flavor=request["flavor"],
|
| 74 |
+
timeout=request["timeout_seconds"],
|
| 75 |
+
labels=labels,
|
| 76 |
+
volumes=self._volumes(request.get("mounts", [])),
|
| 77 |
+
namespace=self.namespace,
|
| 78 |
+
)
|
| 79 |
+
except HfHubHTTPError as exc:
|
| 80 |
+
status = getattr(exc.response, "status_code", None)
|
| 81 |
+
if status is not None and 400 <= status < 500 and status not in {408, 429}:
|
| 82 |
+
raise LaunchError(
|
| 83 |
+
f"provider rejected Job before acceptance: HTTP {status}"
|
| 84 |
+
) from exc
|
| 85 |
+
raise AmbiguousSubmission("provider response is ambiguous; reconcile by label") from exc
|
| 86 |
+
except (RequestException, TimeoutError, ConnectionError) as exc:
|
| 87 |
+
raise AmbiguousSubmission("provider response is ambiguous; reconcile by label") from exc
|
| 88 |
+
return self._normalize_identity(info)
|
| 89 |
+
|
| 90 |
+
def find_by_label(self, label: str) -> list[dict[str, Any]]:
|
| 91 |
+
return [
|
| 92 |
+
self._normalize_identity(info)
|
| 93 |
+
for info in self.api.list_jobs(namespace=self.namespace)
|
| 94 |
+
if (info.labels or {}).get("repro-attempt") == label
|
| 95 |
+
]
|
| 96 |
+
|
| 97 |
+
def get(self, job_id: str) -> dict[str, Any]:
|
| 98 |
+
info = self.api.inspect_job(job_id=job_id, namespace=self.namespace)
|
| 99 |
+
logs = self._logs(job_id)
|
| 100 |
+
stage = getattr(info.status.stage, "value", str(info.status.stage))
|
| 101 |
+
states = {
|
| 102 |
+
"SCHEDULING": "STARTING",
|
| 103 |
+
"RUNNING": "RUNNING",
|
| 104 |
+
"COMPLETED": "COMPLETED",
|
| 105 |
+
"ERROR": "FAILED",
|
| 106 |
+
"CANCELED": "CANCELED",
|
| 107 |
+
"DELETED": "FAILED",
|
| 108 |
+
}
|
| 109 |
+
if stage not in states:
|
| 110 |
+
raise LaunchError(f"unrecognized Hugging Face Job stage {stage!r}")
|
| 111 |
+
ready_times = []
|
| 112 |
+
heartbeat_times = []
|
| 113 |
+
done_times = []
|
| 114 |
+
for line in logs:
|
| 115 |
+
for match in MARKER_TIME.finditer(line):
|
| 116 |
+
try:
|
| 117 |
+
parsed = datetime.fromisoformat(match.group("time").replace("Z", "+00:00"))
|
| 118 |
+
except ValueError:
|
| 119 |
+
continue
|
| 120 |
+
marker = match.group("marker")
|
| 121 |
+
if marker in {"CPU_READY", "GPU_READY"}:
|
| 122 |
+
ready_times.append(parsed)
|
| 123 |
+
elif marker == "HEARTBEAT":
|
| 124 |
+
heartbeat_times.append(parsed)
|
| 125 |
+
elif marker == "DONE":
|
| 126 |
+
done_times.append(parsed)
|
| 127 |
+
duration = getattr(info, "durations", None)
|
| 128 |
+
running_seconds = getattr(duration, "running_secs", None)
|
| 129 |
+
normalized = {
|
| 130 |
+
"state": states[stage],
|
| 131 |
+
"provider_stage": stage,
|
| 132 |
+
"provider_message": info.status.message,
|
| 133 |
+
"starting_at": _iso(info.started_at),
|
| 134 |
+
"started_at": _iso(info.started_at),
|
| 135 |
+
"finished_at": _iso(info.finished_at),
|
| 136 |
+
"ready_marker": bool(ready_times),
|
| 137 |
+
"ready_at": _iso(min(ready_times)) if ready_times else "",
|
| 138 |
+
"heartbeat": bool(heartbeat_times),
|
| 139 |
+
"heartbeat_at": _iso(max(heartbeat_times)) if heartbeat_times else "",
|
| 140 |
+
"done_marker": bool(done_times),
|
| 141 |
+
"done_at": _iso(max(done_times)) if done_times else "",
|
| 142 |
+
"billable_duration_seconds": (
|
| 143 |
+
max(0, math.ceil(float(running_seconds)))
|
| 144 |
+
if running_seconds is not None
|
| 145 |
+
and states[stage] in {"COMPLETED", "FAILED", "CANCELED"}
|
| 146 |
+
else None
|
| 147 |
+
),
|
| 148 |
+
"logs_tail": logs[-200:],
|
| 149 |
+
}
|
| 150 |
+
return normalized
|
| 151 |
+
|
| 152 |
+
def cancel(self, job_id: str) -> None:
|
| 153 |
+
self.api.cancel_job(job_id=job_id, namespace=self.namespace)
|
| 154 |
+
|
| 155 |
+
def _logs(self, job_id: str) -> list[str]:
|
| 156 |
+
try:
|
| 157 |
+
return list(
|
| 158 |
+
self.api.fetch_job_logs(
|
| 159 |
+
job_id=job_id,
|
| 160 |
+
namespace=self.namespace,
|
| 161 |
+
follow=False,
|
| 162 |
+
tail=1000,
|
| 163 |
+
)
|
| 164 |
+
)
|
| 165 |
+
except HfHubHTTPError as exc:
|
| 166 |
+
status = getattr(exc.response, "status_code", None)
|
| 167 |
+
if status in {404, 409}:
|
| 168 |
+
return []
|
| 169 |
+
raise
|
| 170 |
+
|
| 171 |
+
@staticmethod
|
| 172 |
+
def _normalize_identity(info: Any) -> dict[str, Any]:
|
| 173 |
+
job_id = str(info.id)
|
| 174 |
+
url = str(info.url or f"https://huggingface.co/jobs/{job_id}")
|
| 175 |
+
return {
|
| 176 |
+
"job_id": job_id,
|
| 177 |
+
"job_url": url,
|
| 178 |
+
"state": "SUBMITTED",
|
| 179 |
+
}
|
src/repro_control/interventions.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bit-identity checks for the registered literal Sudoku intervention."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
from collections.abc import Mapping
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
import jax
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class InterventionError(ValueError):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def without_restriction_map(tree: Any) -> Any:
|
| 18 |
+
"""Return a structurally equivalent tree with every named ``rm`` subtree removed."""
|
| 19 |
+
if isinstance(tree, Mapping):
|
| 20 |
+
return {
|
| 21 |
+
key: without_restriction_map(value)
|
| 22 |
+
for key, value in tree.items()
|
| 23 |
+
if key not in {"rm", "restriction_maps", "R_indices"}
|
| 24 |
+
}
|
| 25 |
+
if isinstance(tree, list):
|
| 26 |
+
return [without_restriction_map(value) for value in tree]
|
| 27 |
+
if isinstance(tree, tuple):
|
| 28 |
+
return tuple(without_restriction_map(value) for value in tree)
|
| 29 |
+
return tree
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def stable_tree_digest(tree: Any) -> str:
|
| 33 |
+
"""Digest semantic containers and array bytes without device/container metadata."""
|
| 34 |
+
digest = hashlib.sha256()
|
| 35 |
+
|
| 36 |
+
def visit(value: Any) -> None:
|
| 37 |
+
if isinstance(value, Mapping):
|
| 38 |
+
digest.update(b"M{")
|
| 39 |
+
for key in sorted(value, key=lambda item: str(item)):
|
| 40 |
+
encoded = str(key).encode()
|
| 41 |
+
digest.update(len(encoded).to_bytes(8, "big"))
|
| 42 |
+
digest.update(encoded)
|
| 43 |
+
visit(value[key])
|
| 44 |
+
digest.update(b"}")
|
| 45 |
+
return
|
| 46 |
+
if isinstance(value, (list, tuple)):
|
| 47 |
+
digest.update(b"S[")
|
| 48 |
+
digest.update(len(value).to_bytes(8, "big"))
|
| 49 |
+
for item in value:
|
| 50 |
+
visit(item)
|
| 51 |
+
digest.update(b"]")
|
| 52 |
+
return
|
| 53 |
+
array = np.asarray(jax.device_get(value))
|
| 54 |
+
digest.update(b"A")
|
| 55 |
+
digest.update(array.dtype.str.encode())
|
| 56 |
+
digest.update(repr(array.shape).encode())
|
| 57 |
+
digest.update(array.tobytes(order="C"))
|
| 58 |
+
|
| 59 |
+
visit(tree)
|
| 60 |
+
return digest.hexdigest()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def contains_restriction_map(tree: Any) -> bool:
|
| 64 |
+
if isinstance(tree, Mapping):
|
| 65 |
+
return any(
|
| 66 |
+
key in {"rm", "restriction_maps", "R_indices"} or contains_restriction_map(value)
|
| 67 |
+
for key, value in tree.items()
|
| 68 |
+
)
|
| 69 |
+
if isinstance(tree, (list, tuple)):
|
| 70 |
+
return any(contains_restriction_map(value) for value in tree)
|
| 71 |
+
return False
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def assert_common_bit_identical(
|
| 75 |
+
learned_params: Any,
|
| 76 |
+
identity_params: Any,
|
| 77 |
+
learned_optimizer: Any,
|
| 78 |
+
identity_optimizer: Any,
|
| 79 |
+
learned_counters: Any,
|
| 80 |
+
identity_counters: Any,
|
| 81 |
+
) -> None:
|
| 82 |
+
if contains_restriction_map(identity_params):
|
| 83 |
+
raise InterventionError("identity parameter tree contains a restriction-map leaf")
|
| 84 |
+
if contains_restriction_map(identity_optimizer):
|
| 85 |
+
raise InterventionError("identity optimizer tree contains a restriction-map leaf")
|
| 86 |
+
pairs = (
|
| 87 |
+
(without_restriction_map(learned_params), identity_params, "parameter"),
|
| 88 |
+
(without_restriction_map(learned_optimizer), identity_optimizer, "optimizer"),
|
| 89 |
+
(learned_counters, identity_counters, "counter"),
|
| 90 |
+
)
|
| 91 |
+
for learned, identity, name in pairs:
|
| 92 |
+
if stable_tree_digest(learned) != stable_tree_digest(identity):
|
| 93 |
+
raise InterventionError(f"common {name} leaves are not bit-identical")
|
src/repro_control/launcher.py
ADDED
|
@@ -0,0 +1,432 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Durable direct-launch state machine with submission disabled by default."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from datetime import datetime, timedelta, timezone
|
| 9 |
+
from decimal import Decimal, ROUND_CEILING
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any, Protocol
|
| 12 |
+
|
| 13 |
+
from .constants import (
|
| 14 |
+
DEADLINE_UTC,
|
| 15 |
+
GPU_RETRY_RESERVE_MICRO_USD,
|
| 16 |
+
MAX_CPU_RETURNED_IDS,
|
| 17 |
+
MAX_GPU_RETURNED_IDS,
|
| 18 |
+
MAX_RETURNED_IDS,
|
| 19 |
+
MINIMUM_BALANCE_MICRO_USD,
|
| 20 |
+
NORMAL_CAP_MICRO_USD,
|
| 21 |
+
)
|
| 22 |
+
from .hashing import atomic_write_json
|
| 23 |
+
from .manifests import REQUIRED_FIELDS, validate_manifest
|
| 24 |
+
|
| 25 |
+
ACTIVE = frozenset({"PENDING_SUBMISSION", "SUBMITTED", "STARTING", "RUNNING", "CANCELING"})
|
| 26 |
+
TERMINAL = frozenset({"COMPLETED", "FAILED", "CANCELED"})
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class LaunchError(RuntimeError):
|
| 30 |
+
pass
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class AmbiguousSubmission(LaunchError):
|
| 34 |
+
pass
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class Provider(Protocol):
|
| 38 |
+
def submit(self, request: dict[str, Any]) -> dict[str, Any]: ...
|
| 39 |
+
|
| 40 |
+
def find_by_label(self, label: str) -> list[dict[str, Any]]: ...
|
| 41 |
+
|
| 42 |
+
def get(self, job_id: str) -> dict[str, Any]: ...
|
| 43 |
+
|
| 44 |
+
def cancel(self, job_id: str) -> None: ...
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def parse_utc(value: str) -> datetime:
|
| 48 |
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
| 49 |
+
if parsed.tzinfo is None:
|
| 50 |
+
raise ValueError("timestamp must include a timezone")
|
| 51 |
+
return parsed.astimezone(timezone.utc)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def quantum_cost(rate_micro_usd_per_hour: int, seconds: int) -> int:
|
| 55 |
+
if rate_micro_usd_per_hour < 0 or seconds < 0:
|
| 56 |
+
raise ValueError("cost inputs must be nonnegative")
|
| 57 |
+
minutes = (seconds + 59) // 60
|
| 58 |
+
return (rate_micro_usd_per_hour * minutes + 59) // 60
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def rate_to_micro_usd(rate: str) -> int:
|
| 62 |
+
value = (Decimal(rate) * Decimal(1_000_000)).to_integral_value(rounding=ROUND_CEILING)
|
| 63 |
+
return int(value)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def deterministic_attempt_id(
|
| 67 |
+
logical_id: str, ordinal: int, science_spec_sha256: str, science_freeze_sha256: str
|
| 68 |
+
) -> str:
|
| 69 |
+
material = f"{logical_id}\0{ordinal}\0{science_spec_sha256}\0{science_freeze_sha256}"
|
| 70 |
+
return f"a{ordinal:02d}-{hashlib.sha256(material.encode()).hexdigest()[:20]}"
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def deterministic_label(logical_id: str, attempt_id: str) -> str:
|
| 74 |
+
return f"repro-{logical_id.lower()}-{attempt_id}"
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@dataclass(frozen=True)
|
| 78 |
+
class BudgetSnapshot:
|
| 79 |
+
verified_balance_micro_usd: int
|
| 80 |
+
unsettled_estimate_micro_usd: int
|
| 81 |
+
incurred_micro_usd: int
|
| 82 |
+
active_remaining_micro_usd: int
|
| 83 |
+
mandatory_unsubmitted_micro_usd: int
|
| 84 |
+
non_job_cost_micro_usd: int = 0
|
| 85 |
+
gpu_retry_reserve_micro_usd: int = GPU_RETRY_RESERVE_MICRO_USD
|
| 86 |
+
cpu_retry_reserve_micro_usd: int = 0
|
| 87 |
+
|
| 88 |
+
def prospective_commitment(self, proposed_micro_usd: int) -> int:
|
| 89 |
+
return (
|
| 90 |
+
self.incurred_micro_usd
|
| 91 |
+
+ self.active_remaining_micro_usd
|
| 92 |
+
+ proposed_micro_usd
|
| 93 |
+
+ self.mandatory_unsubmitted_micro_usd
|
| 94 |
+
+ self.non_job_cost_micro_usd
|
| 95 |
+
+ self.gpu_retry_reserve_micro_usd
|
| 96 |
+
+ self.cpu_retry_reserve_micro_usd
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
def assert_feasible(self, proposed_micro_usd: int) -> None:
|
| 100 |
+
commitment = self.prospective_commitment(proposed_micro_usd)
|
| 101 |
+
if commitment > NORMAL_CAP_MICRO_USD:
|
| 102 |
+
raise LaunchError(f"normal spend cap exceeded: {commitment}>{NORMAL_CAP_MICRO_USD}")
|
| 103 |
+
available_after = (
|
| 104 |
+
self.verified_balance_micro_usd
|
| 105 |
+
- self.unsettled_estimate_micro_usd
|
| 106 |
+
- (commitment - self.incurred_micro_usd)
|
| 107 |
+
)
|
| 108 |
+
if available_after < MINIMUM_BALANCE_MICRO_USD:
|
| 109 |
+
raise LaunchError(
|
| 110 |
+
f"minimum balance gate failed: {available_after}<{MINIMUM_BALANCE_MICRO_USD}"
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@dataclass
|
| 115 |
+
class LauncherState:
|
| 116 |
+
format: int = 1
|
| 117 |
+
attempts: list[dict[str, Any]] = field(default_factory=list)
|
| 118 |
+
reservations: list[dict[str, Any]] = field(default_factory=list)
|
| 119 |
+
|
| 120 |
+
@classmethod
|
| 121 |
+
def load(cls, path: Path) -> "LauncherState":
|
| 122 |
+
if not path.exists():
|
| 123 |
+
return cls()
|
| 124 |
+
value = json.loads(path.read_text())
|
| 125 |
+
if set(value) != {"format", "attempts", "reservations"} or value["format"] != 1:
|
| 126 |
+
raise LaunchError("invalid launcher state")
|
| 127 |
+
return cls(**value)
|
| 128 |
+
|
| 129 |
+
def save(self, path: Path) -> None:
|
| 130 |
+
atomic_write_json(
|
| 131 |
+
path,
|
| 132 |
+
{"format": self.format, "attempts": self.attempts, "reservations": self.reservations},
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
def returned_counts(self) -> tuple[int, int, int]:
|
| 136 |
+
returned = [row for row in self.attempts if row.get("job_id")]
|
| 137 |
+
gpu = sum(row.get("resource_class") == "GPU" for row in returned)
|
| 138 |
+
cpu = sum(row.get("resource_class") == "CPU" for row in returned)
|
| 139 |
+
return len(returned), gpu, cpu
|
| 140 |
+
|
| 141 |
+
def assert_identity_available(self, logical_id: str) -> None:
|
| 142 |
+
duplicate = [
|
| 143 |
+
row
|
| 144 |
+
for row in self.attempts
|
| 145 |
+
if row["logical_id"] == logical_id
|
| 146 |
+
and (row["state"] in ACTIVE or row["state"] == "COMPLETED")
|
| 147 |
+
]
|
| 148 |
+
if duplicate:
|
| 149 |
+
raise LaunchError(f"duplicate active/complete logical identity {logical_id}")
|
| 150 |
+
|
| 151 |
+
def assert_id_capacity(self, resource_class: str) -> None:
|
| 152 |
+
total, gpu, cpu = self.returned_counts()
|
| 153 |
+
provisional = sum(row.get("provisional_id_slot", False) for row in self.attempts)
|
| 154 |
+
if total + provisional + 1 > MAX_RETURNED_IDS:
|
| 155 |
+
raise LaunchError("absolute returned-ID gate failed")
|
| 156 |
+
if resource_class == "GPU" and gpu + provisional + 1 > MAX_GPU_RETURNED_IDS:
|
| 157 |
+
raise LaunchError("GPU returned-ID gate failed")
|
| 158 |
+
if resource_class == "CPU" and cpu + provisional + 1 > MAX_CPU_RETURNED_IDS:
|
| 159 |
+
raise LaunchError("CPU returned-ID gate failed")
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class DirectLauncher:
|
| 163 |
+
"""Provider-neutral launcher.
|
| 164 |
+
|
| 165 |
+
A real provider must be injected and ``enable_submit`` must be explicit.
|
| 166 |
+
The CLI never enables this in the local implementation turn.
|
| 167 |
+
"""
|
| 168 |
+
|
| 169 |
+
def __init__(
|
| 170 |
+
self,
|
| 171 |
+
state_path: Path,
|
| 172 |
+
*,
|
| 173 |
+
provider: Provider | None = None,
|
| 174 |
+
enable_submit: bool = False,
|
| 175 |
+
deadline_utc: str = DEADLINE_UTC,
|
| 176 |
+
enforce_admission_sequence: bool = True,
|
| 177 |
+
):
|
| 178 |
+
self.state_path = state_path
|
| 179 |
+
self.provider = provider
|
| 180 |
+
self.enable_submit = enable_submit
|
| 181 |
+
self.deadline = parse_utc(deadline_utc)
|
| 182 |
+
self.enforce_admission_sequence = enforce_admission_sequence
|
| 183 |
+
|
| 184 |
+
def assert_deadline(
|
| 185 |
+
self, now: datetime, guarded_critical_path_seconds: int, *, safety_hours: int = 12
|
| 186 |
+
) -> None:
|
| 187 |
+
if now.tzinfo is None:
|
| 188 |
+
raise LaunchError("now must be timezone-aware")
|
| 189 |
+
finish = now.astimezone(timezone.utc) + timedelta(seconds=guarded_critical_path_seconds)
|
| 190 |
+
if finish > self.deadline - timedelta(hours=safety_hours):
|
| 191 |
+
raise LaunchError("guarded critical path misses the deadline safety margin")
|
| 192 |
+
|
| 193 |
+
def prepare(
|
| 194 |
+
self,
|
| 195 |
+
request: dict[str, Any],
|
| 196 |
+
*,
|
| 197 |
+
budget: BudgetSnapshot,
|
| 198 |
+
now: datetime,
|
| 199 |
+
guarded_critical_path_seconds: int,
|
| 200 |
+
) -> dict[str, Any]:
|
| 201 |
+
state = LauncherState.load(self.state_path)
|
| 202 |
+
logical_id = request["logical_id"]
|
| 203 |
+
resource_class = request["resource_class"]
|
| 204 |
+
if self.enforce_admission_sequence:
|
| 205 |
+
canaries = [row for row in state.attempts if row["logical_id"] == "HF-CPU-CANARY"]
|
| 206 |
+
if not canaries and logical_id != "HF-CPU-CANARY":
|
| 207 |
+
raise LaunchError("the first attempted external mutation must be HF-CPU-CANARY")
|
| 208 |
+
if logical_id != "HF-CPU-CANARY" and not any(
|
| 209 |
+
row["state"] == "COMPLETED" for row in canaries
|
| 210 |
+
):
|
| 211 |
+
raise LaunchError("CPU canary has not completed")
|
| 212 |
+
if request.get("is_retry"):
|
| 213 |
+
reason = request.get("retry_reason_class")
|
| 214 |
+
if reason != "INFRASTRUCTURE":
|
| 215 |
+
raise LaunchError("only classified infrastructure failures authorize retry")
|
| 216 |
+
prior_retries = sum(
|
| 217 |
+
row["resource_class"] == resource_class and row["request"].get("is_retry")
|
| 218 |
+
for row in state.attempts
|
| 219 |
+
)
|
| 220 |
+
ceiling = 3 if resource_class == "GPU" else 1
|
| 221 |
+
if prior_retries >= ceiling:
|
| 222 |
+
raise LaunchError(f"{resource_class} retry ceiling exhausted")
|
| 223 |
+
state.assert_identity_available(logical_id)
|
| 224 |
+
state.assert_id_capacity(resource_class)
|
| 225 |
+
self.assert_deadline(now, guarded_critical_path_seconds)
|
| 226 |
+
proposed = quantum_cost(
|
| 227 |
+
request["frozen_hourly_rate_micro_usd"], request["timeout_seconds"]
|
| 228 |
+
)
|
| 229 |
+
budget.assert_feasible(proposed)
|
| 230 |
+
ordinal = 1 + sum(row["logical_id"] == logical_id for row in state.attempts)
|
| 231 |
+
attempt_id = deterministic_attempt_id(
|
| 232 |
+
logical_id,
|
| 233 |
+
ordinal,
|
| 234 |
+
request["science_spec_sha256"],
|
| 235 |
+
request["science_freeze_sha256"],
|
| 236 |
+
)
|
| 237 |
+
if request.get("attempt_id") != attempt_id:
|
| 238 |
+
raise LaunchError(
|
| 239 |
+
f"request attempt_id mismatch: {request.get('attempt_id')!r}!={attempt_id!r}"
|
| 240 |
+
)
|
| 241 |
+
try:
|
| 242 |
+
manifest = {name: request[name] for name in REQUIRED_FIELDS}
|
| 243 |
+
except KeyError as exc:
|
| 244 |
+
raise LaunchError(f"request is missing manifest field {exc.args[0]!r}") from exc
|
| 245 |
+
try:
|
| 246 |
+
validate_manifest(
|
| 247 |
+
manifest,
|
| 248 |
+
freeze_sha256=(
|
| 249 |
+
request["science_freeze_sha256"]
|
| 250 |
+
if request["job_class"].startswith("SCIENTIFIC")
|
| 251 |
+
else ""
|
| 252 |
+
),
|
| 253 |
+
)
|
| 254 |
+
except ValueError as exc:
|
| 255 |
+
raise LaunchError(f"invalid Job manifest: {exc}") from exc
|
| 256 |
+
if request.get("hardware") != request.get("flavor"):
|
| 257 |
+
raise LaunchError("manifest hardware must equal provider flavor")
|
| 258 |
+
image = request.get("image")
|
| 259 |
+
if not isinstance(image, str) or not image.endswith(f"@{request['image_digest']}"):
|
| 260 |
+
raise LaunchError("provider image must end in the manifest image digest")
|
| 261 |
+
expected_resource_class = "CPU" if request["job_class"].startswith("CPU_") else "GPU"
|
| 262 |
+
if resource_class != expected_resource_class:
|
| 263 |
+
raise LaunchError(
|
| 264 |
+
f"{request['job_class']} requires resource_class={expected_resource_class}"
|
| 265 |
+
)
|
| 266 |
+
row = {
|
| 267 |
+
"logical_id": logical_id,
|
| 268 |
+
"attempt_id": attempt_id,
|
| 269 |
+
"label": deterministic_label(logical_id, attempt_id),
|
| 270 |
+
"resource_class": resource_class,
|
| 271 |
+
"state": "PENDING_SUBMISSION",
|
| 272 |
+
"job_id": "",
|
| 273 |
+
"job_url": "",
|
| 274 |
+
"provisional_id_slot": True,
|
| 275 |
+
"request": request,
|
| 276 |
+
"terminal_duration_seconds": None,
|
| 277 |
+
"estimated_cost_micro_usd": None,
|
| 278 |
+
"ready_at": "",
|
| 279 |
+
"last_heartbeat_at": "",
|
| 280 |
+
}
|
| 281 |
+
state.attempts.append(row)
|
| 282 |
+
state.reservations.append(
|
| 283 |
+
{
|
| 284 |
+
"attempt_id": attempt_id,
|
| 285 |
+
"reserved_timeout_cost_micro_usd": proposed,
|
| 286 |
+
"released": False,
|
| 287 |
+
"release_reason": "",
|
| 288 |
+
}
|
| 289 |
+
)
|
| 290 |
+
state.save(self.state_path)
|
| 291 |
+
return row
|
| 292 |
+
|
| 293 |
+
def submit_prepared(self, attempt_id: str) -> dict[str, Any]:
|
| 294 |
+
if not self.enable_submit:
|
| 295 |
+
raise LaunchError("submission disabled; inject a provider and explicitly enable it")
|
| 296 |
+
if self.provider is None:
|
| 297 |
+
raise LaunchError("no provider injected")
|
| 298 |
+
state = LauncherState.load(self.state_path)
|
| 299 |
+
row = _one(state, attempt_id)
|
| 300 |
+
if row["state"] != "PENDING_SUBMISSION":
|
| 301 |
+
raise LaunchError("attempt is not pending submission")
|
| 302 |
+
try:
|
| 303 |
+
result = self.provider.submit({**row["request"], "label": row["label"]})
|
| 304 |
+
except AmbiguousSubmission:
|
| 305 |
+
state.save(self.state_path)
|
| 306 |
+
raise
|
| 307 |
+
self._record_submit_result(state, row, result)
|
| 308 |
+
state.save(self.state_path)
|
| 309 |
+
return row
|
| 310 |
+
|
| 311 |
+
def reconcile_ambiguous(self, attempt_id: str) -> dict[str, Any]:
|
| 312 |
+
if self.provider is None:
|
| 313 |
+
raise LaunchError("no provider injected")
|
| 314 |
+
state = LauncherState.load(self.state_path)
|
| 315 |
+
row = _one(state, attempt_id)
|
| 316 |
+
if row["state"] != "PENDING_SUBMISSION":
|
| 317 |
+
raise LaunchError("only pending submissions can be reconciled")
|
| 318 |
+
matches = self.provider.find_by_label(row["label"])
|
| 319 |
+
if len(matches) > 1:
|
| 320 |
+
raise LaunchError("duplicate provider labels; owner review required")
|
| 321 |
+
if len(matches) == 1:
|
| 322 |
+
self._record_submit_result(state, row, matches[0])
|
| 323 |
+
else:
|
| 324 |
+
row["provisional_id_slot"] = False
|
| 325 |
+
row["state"] = "ABSENT_AFTER_RECONCILIATION"
|
| 326 |
+
_release_reservation(state, attempt_id, "provider_absence_proved")
|
| 327 |
+
state.save(self.state_path)
|
| 328 |
+
return row
|
| 329 |
+
|
| 330 |
+
@staticmethod
|
| 331 |
+
def _record_submit_result(
|
| 332 |
+
state: LauncherState, row: dict[str, Any], result: dict[str, Any]
|
| 333 |
+
) -> None:
|
| 334 |
+
job_id, job_url = result.get("job_id"), result.get("job_url")
|
| 335 |
+
if not job_id or not job_url:
|
| 336 |
+
raise AmbiguousSubmission("provider response did not contain a complete identity")
|
| 337 |
+
if any(other.get("job_id") == job_id for other in state.attempts if other is not row):
|
| 338 |
+
raise LaunchError("provider job ID is already recorded")
|
| 339 |
+
row.update(
|
| 340 |
+
state="SUBMITTED",
|
| 341 |
+
job_id=job_id,
|
| 342 |
+
job_url=job_url,
|
| 343 |
+
provisional_id_slot=False,
|
| 344 |
+
)
|
| 345 |
+
|
| 346 |
+
def poll(self, attempt_id: str, *, now: datetime) -> dict[str, Any]:
|
| 347 |
+
if self.provider is None:
|
| 348 |
+
raise LaunchError("no provider injected")
|
| 349 |
+
state = LauncherState.load(self.state_path)
|
| 350 |
+
row = _one(state, attempt_id)
|
| 351 |
+
if not row["job_id"]:
|
| 352 |
+
raise LaunchError("attempt has no provider job ID")
|
| 353 |
+
remote = self.provider.get(row["job_id"])
|
| 354 |
+
remote_state = remote["state"]
|
| 355 |
+
if remote_state not in ACTIVE | TERMINAL:
|
| 356 |
+
raise LaunchError(f"unknown provider lifecycle state {remote_state!r}")
|
| 357 |
+
row["state"] = remote_state
|
| 358 |
+
if remote.get("ready_marker"):
|
| 359 |
+
row["ready_at"] = remote.get("ready_at") or now.astimezone(timezone.utc).isoformat()
|
| 360 |
+
if remote.get("heartbeat"):
|
| 361 |
+
row["last_heartbeat_at"] = remote.get("heartbeat_at") or now.astimezone(
|
| 362 |
+
timezone.utc
|
| 363 |
+
).isoformat()
|
| 364 |
+
self._enforce_runtime_gates(row, remote, now)
|
| 365 |
+
if remote_state in TERMINAL:
|
| 366 |
+
duration = remote.get("billable_duration_seconds")
|
| 367 |
+
if not isinstance(duration, int) or duration < 0:
|
| 368 |
+
raise LaunchError("terminal attempt lacks final billable duration")
|
| 369 |
+
row["terminal_duration_seconds"] = duration
|
| 370 |
+
row["estimated_cost_micro_usd"] = quantum_cost(
|
| 371 |
+
row["request"]["frozen_hourly_rate_micro_usd"], duration
|
| 372 |
+
)
|
| 373 |
+
_release_reservation(state, attempt_id, "terminal_duration_recorded")
|
| 374 |
+
state.save(self.state_path)
|
| 375 |
+
return row
|
| 376 |
+
|
| 377 |
+
@staticmethod
|
| 378 |
+
def _enforce_runtime_gates(
|
| 379 |
+
row: dict[str, Any], remote: dict[str, Any], now: datetime
|
| 380 |
+
) -> None:
|
| 381 |
+
billable_start = next(
|
| 382 |
+
(
|
| 383 |
+
remote.get(name)
|
| 384 |
+
for name in ("starting_at", "started_at", "first_running_at")
|
| 385 |
+
if remote.get(name)
|
| 386 |
+
),
|
| 387 |
+
None,
|
| 388 |
+
)
|
| 389 |
+
if not billable_start or row["state"] in TERMINAL:
|
| 390 |
+
return
|
| 391 |
+
elapsed = (now.astimezone(timezone.utc) - parse_utc(billable_start)).total_seconds()
|
| 392 |
+
if not row["ready_at"] and elapsed > 180:
|
| 393 |
+
raise LaunchError("readiness deadline exceeded")
|
| 394 |
+
heartbeat_deadline = (
|
| 395 |
+
(
|
| 396 |
+
max(
|
| 397 |
+
180,
|
| 398 |
+
3 * row["request"]["largest_guarded_no_output_block_seconds"],
|
| 399 |
+
)
|
| 400 |
+
+ 59
|
| 401 |
+
)
|
| 402 |
+
// 60
|
| 403 |
+
) * 60
|
| 404 |
+
heartbeat_from = parse_utc(row["last_heartbeat_at"] or billable_start)
|
| 405 |
+
if (now.astimezone(timezone.utc) - heartbeat_from).total_seconds() > heartbeat_deadline:
|
| 406 |
+
raise LaunchError("heartbeat deadline exceeded")
|
| 407 |
+
|
| 408 |
+
def cancel(self, attempt_id: str) -> None:
|
| 409 |
+
if not self.enable_submit or self.provider is None:
|
| 410 |
+
raise LaunchError("cancellation disabled without an explicitly enabled provider")
|
| 411 |
+
state = LauncherState.load(self.state_path)
|
| 412 |
+
row = _one(state, attempt_id)
|
| 413 |
+
if row["state"] not in ACTIVE or not row["job_id"]:
|
| 414 |
+
raise LaunchError("attempt is not cancelable")
|
| 415 |
+
self.provider.cancel(row["job_id"])
|
| 416 |
+
row["state"] = "CANCELING"
|
| 417 |
+
state.save(self.state_path)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def _one(state: LauncherState, attempt_id: str) -> dict[str, Any]:
|
| 421 |
+
matches = [row for row in state.attempts if row["attempt_id"] == attempt_id]
|
| 422 |
+
if len(matches) != 1:
|
| 423 |
+
raise LaunchError(f"attempt lookup returned {len(matches)} rows")
|
| 424 |
+
return matches[0]
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
def _release_reservation(state: LauncherState, attempt_id: str, reason: str) -> None:
|
| 428 |
+
matches = [row for row in state.reservations if row["attempt_id"] == attempt_id]
|
| 429 |
+
if len(matches) != 1:
|
| 430 |
+
raise LaunchError("attempt reservation is missing or duplicated")
|
| 431 |
+
matches[0]["released"] = True
|
| 432 |
+
matches[0]["release_reason"] = reason
|
src/repro_control/manifests.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict static and lifecycle-aware Job manifest validation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import re
|
| 6 |
+
from dataclasses import dataclass
|
| 7 |
+
from pathlib import PurePosixPath
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
from .constants import NOT_APPLICABLE, SCIENCE_SPEC_SHA256
|
| 11 |
+
|
| 12 |
+
JOB_CLASSES = frozenset(
|
| 13 |
+
{"CPU_CANARY", "CPU_IMPORT", "GPU_SMOKE", "SCIENTIFIC_TRAIN", "SCIENTIFIC_EVAL"}
|
| 14 |
+
)
|
| 15 |
+
REQUIRED_FIELDS = frozenset(
|
| 16 |
+
{
|
| 17 |
+
"job_class",
|
| 18 |
+
"logical_id",
|
| 19 |
+
"attempt_id",
|
| 20 |
+
"image_digest",
|
| 21 |
+
"science_spec_sha256",
|
| 22 |
+
"science_freeze_sha256",
|
| 23 |
+
"hashes",
|
| 24 |
+
"task",
|
| 25 |
+
"seed",
|
| 26 |
+
"intervention",
|
| 27 |
+
"hardware",
|
| 28 |
+
"frozen_hourly_rate_micro_usd",
|
| 29 |
+
"timeout_seconds",
|
| 30 |
+
"largest_guarded_no_output_block_seconds",
|
| 31 |
+
"command",
|
| 32 |
+
"expected_outputs",
|
| 33 |
+
"mounts",
|
| 34 |
+
}
|
| 35 |
+
)
|
| 36 |
+
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
| 37 |
+
ID_RE = re.compile(r"^[A-Z0-9][A-Z0-9-]{1,95}$")
|
| 38 |
+
ATTEMPT_RE = re.compile(r"^[a-z0-9][a-z0-9-]{7,95}$")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class ManifestError(ValueError):
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _exact_keys(value: dict[str, Any], expected: frozenset[str], where: str) -> None:
|
| 46 |
+
missing = expected - value.keys()
|
| 47 |
+
extra = value.keys() - expected
|
| 48 |
+
if missing or extra:
|
| 49 |
+
raise ManifestError(f"{where} keys: missing={sorted(missing)}, extra={sorted(extra)}")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _sha_or_empty(value: Any, name: str) -> None:
|
| 53 |
+
if value != "" and (not isinstance(value, str) or not SHA256_RE.fullmatch(value)):
|
| 54 |
+
raise ManifestError(f"{name} must be empty or lowercase SHA-256")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def validate_mount(mount: dict[str, Any]) -> None:
|
| 58 |
+
_exact_keys(
|
| 59 |
+
mount,
|
| 60 |
+
frozenset({"volume_source", "volume_path", "mount_path", "read_only"}),
|
| 61 |
+
"mount",
|
| 62 |
+
)
|
| 63 |
+
for name in ("volume_source", "volume_path", "mount_path"):
|
| 64 |
+
if not isinstance(mount[name], str) or not mount[name]:
|
| 65 |
+
raise ManifestError(f"mount {name} must be a non-empty string")
|
| 66 |
+
path = PurePosixPath(mount["mount_path"])
|
| 67 |
+
if not path.is_absolute() or ".." in path.parts:
|
| 68 |
+
raise ManifestError("mount_path must be normalized and absolute")
|
| 69 |
+
if not isinstance(mount["read_only"], bool):
|
| 70 |
+
raise ManifestError("mount read_only must be boolean")
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def validate_manifest(value: dict[str, Any], *, freeze_sha256: str = "") -> None:
|
| 74 |
+
_exact_keys(value, REQUIRED_FIELDS, "manifest")
|
| 75 |
+
job_class = value["job_class"]
|
| 76 |
+
if job_class not in JOB_CLASSES:
|
| 77 |
+
raise ManifestError(f"unknown job_class {job_class!r}")
|
| 78 |
+
if not isinstance(value["logical_id"], str) or not ID_RE.fullmatch(value["logical_id"]):
|
| 79 |
+
raise ManifestError("logical_id must be a neutral uppercase identity")
|
| 80 |
+
if not isinstance(value["attempt_id"], str) or not ATTEMPT_RE.fullmatch(value["attempt_id"]):
|
| 81 |
+
raise ManifestError("attempt_id must be a deterministic lowercase identity")
|
| 82 |
+
if not isinstance(value["image_digest"], str) or not (
|
| 83 |
+
value["image_digest"].startswith("sha256:") and SHA256_RE.fullmatch(value["image_digest"][7:])
|
| 84 |
+
):
|
| 85 |
+
raise ManifestError("image_digest must be sha256:<digest>")
|
| 86 |
+
expected_spec = NOT_APPLICABLE if job_class == "CPU_CANARY" else SCIENCE_SPEC_SHA256
|
| 87 |
+
if value["science_spec_sha256"] != expected_spec:
|
| 88 |
+
raise ManifestError(f"{job_class} science_spec_sha256 must be {expected_spec}")
|
| 89 |
+
if job_class in {"SCIENTIFIC_TRAIN", "SCIENTIFIC_EVAL"}:
|
| 90 |
+
if not freeze_sha256 or not SHA256_RE.fullmatch(freeze_sha256):
|
| 91 |
+
raise ManifestError("scientific manifest requires the expected freeze SHA-256")
|
| 92 |
+
if value["science_freeze_sha256"] != freeze_sha256:
|
| 93 |
+
raise ManifestError("scientific manifest freeze mismatch")
|
| 94 |
+
elif value["science_freeze_sha256"] != NOT_APPLICABLE:
|
| 95 |
+
raise ManifestError(f"{job_class} science_freeze_sha256 must be NOT_APPLICABLE")
|
| 96 |
+
if not isinstance(value["hashes"], dict):
|
| 97 |
+
raise ManifestError("hashes must be an object")
|
| 98 |
+
for name in ("source", "config", "data", "checkpoint"):
|
| 99 |
+
if name not in value["hashes"]:
|
| 100 |
+
raise ManifestError(f"hashes missing {name}")
|
| 101 |
+
_sha_or_empty(value["hashes"][name], f"hashes.{name}")
|
| 102 |
+
if value["seed"] not in (None, 42, 123, 456):
|
| 103 |
+
raise ManifestError("seed must be null or registered")
|
| 104 |
+
if not isinstance(value["frozen_hourly_rate_micro_usd"], int) or value[
|
| 105 |
+
"frozen_hourly_rate_micro_usd"
|
| 106 |
+
] < 0:
|
| 107 |
+
raise ManifestError("rate must be a nonnegative integer")
|
| 108 |
+
for name in ("timeout_seconds", "largest_guarded_no_output_block_seconds"):
|
| 109 |
+
if not isinstance(value[name], int) or value[name] <= 0:
|
| 110 |
+
raise ManifestError(f"{name} must be a positive integer")
|
| 111 |
+
if not isinstance(value["command"], list) or not value["command"] or not all(
|
| 112 |
+
isinstance(item, str) and item for item in value["command"]
|
| 113 |
+
):
|
| 114 |
+
raise ManifestError("command must be a non-empty argv list")
|
| 115 |
+
if not isinstance(value["expected_outputs"], list) or not value["expected_outputs"]:
|
| 116 |
+
raise ManifestError("expected_outputs must be a non-empty list")
|
| 117 |
+
if "DONE.json" not in value["expected_outputs"]:
|
| 118 |
+
raise ManifestError("expected_outputs must include DONE.json")
|
| 119 |
+
if not isinstance(value["mounts"], list):
|
| 120 |
+
raise ManifestError("mounts must be a list")
|
| 121 |
+
for mount in value["mounts"]:
|
| 122 |
+
validate_mount(mount)
|
| 123 |
+
if job_class in {"SCIENTIFIC_TRAIN", "SCIENTIFIC_EVAL"}:
|
| 124 |
+
controls = [m for m in value["mounts"] if m["mount_path"] == "/repro-control"]
|
| 125 |
+
if len(controls) != 1 or not controls[0]["read_only"]:
|
| 126 |
+
raise ManifestError("scientific jobs require one read-only /repro-control mount")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def validate_prefix_capability(
|
| 130 |
+
requested_path: str, allowed_prefix: str, *, write: bool, read_only: bool
|
| 131 |
+
) -> None:
|
| 132 |
+
requested = PurePosixPath("/" + requested_path.lstrip("/"))
|
| 133 |
+
allowed = PurePosixPath("/" + allowed_prefix.lstrip("/"))
|
| 134 |
+
if ".." in PurePosixPath(requested_path).parts:
|
| 135 |
+
raise ManifestError("parent traversal forbidden")
|
| 136 |
+
try:
|
| 137 |
+
requested.relative_to(allowed)
|
| 138 |
+
except ValueError as exc:
|
| 139 |
+
raise ManifestError("requested object lies outside the mounted prefix") from exc
|
| 140 |
+
if write and read_only:
|
| 141 |
+
raise ManifestError("write forbidden by read-only capability")
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
@dataclass(frozen=True)
|
| 145 |
+
class ReadinessContract:
|
| 146 |
+
ready_deadline_seconds: int = 180
|
| 147 |
+
heartbeat_interval_seconds: int = 60
|
| 148 |
+
|
| 149 |
+
@staticmethod
|
| 150 |
+
def heartbeat_deadline(largest_guarded_no_output_block_seconds: int) -> int:
|
| 151 |
+
raw = max(180, 3 * largest_guarded_no_output_block_seconds)
|
| 152 |
+
return ((raw + 59) // 60) * 60
|
src/repro_control/privacy.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic public-byte inventory and conservative privacy scanner."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import re
|
| 6 |
+
from collections.abc import Iterable
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
from .hashing import atomic_write_json, canonical_root, file_entry
|
| 11 |
+
|
| 12 |
+
SCANNER_VERSION = "repro-privacy-1"
|
| 13 |
+
EXCLUDED_LOCAL_PARTS = frozenset({".git", ".venv", ".uv-cache", "__pycache__"})
|
| 14 |
+
RULES = {
|
| 15 |
+
"credential_assignment": re.compile(
|
| 16 |
+
rb"(?i)(?:hf_token|huggingface_token|api[_-]?key|authorization)\\s*[:=]\\s*[\"']?[A-Za-z0-9_-]{12,}"
|
| 17 |
+
),
|
| 18 |
+
"hf_token_shape": re.compile(rb"hf_[A-Za-z0-9]{20,}"),
|
| 19 |
+
"private_key": re.compile(rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
| 20 |
+
"home_path": re.compile(rb"/(?:Users|home)/[A-Za-z0-9._-]+/"),
|
| 21 |
+
"private_phase_program": re.compile(
|
| 22 |
+
rb"(?i)(?:ph" rb"ase[\s_-]*1|sheaf-p" rb"11|p1\.[0-9])"
|
| 23 |
+
),
|
| 24 |
+
"private_phase0_run_id": re.compile(
|
| 25 |
+
rb"\bE-(?:mnist|maze|sudoku)-s(?:42|123|456)(?:-[A-Za-z0-9_-]+)?\b"
|
| 26 |
+
),
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class PrivacyError(ValueError):
|
| 31 |
+
pass
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def scan_paths(
|
| 35 |
+
roots: Iterable[Path],
|
| 36 |
+
*,
|
| 37 |
+
relative_to: Path,
|
| 38 |
+
intended_remote_map: dict[str, str],
|
| 39 |
+
allow_checkpoint_references: frozenset[str] = frozenset(),
|
| 40 |
+
) -> dict[str, Any]:
|
| 41 |
+
relative_to = relative_to.resolve()
|
| 42 |
+
paths: list[Path] = []
|
| 43 |
+
for root in roots:
|
| 44 |
+
root = root.resolve()
|
| 45 |
+
if root.is_file():
|
| 46 |
+
paths.append(root)
|
| 47 |
+
elif root.is_dir():
|
| 48 |
+
paths.extend(
|
| 49 |
+
path
|
| 50 |
+
for path in root.rglob("*")
|
| 51 |
+
if path.is_file()
|
| 52 |
+
and not (set(path.relative_to(relative_to).parts) & EXCLUDED_LOCAL_PARTS)
|
| 53 |
+
)
|
| 54 |
+
unique = sorted(set(paths))
|
| 55 |
+
entries, findings = [], []
|
| 56 |
+
for path in unique:
|
| 57 |
+
entry = file_entry(path, relative_to=relative_to)
|
| 58 |
+
entries.append(entry)
|
| 59 |
+
data = path.read_bytes()
|
| 60 |
+
display = entry["path"]
|
| 61 |
+
if display.endswith((".pkl", ".pickle")) and display not in allow_checkpoint_references:
|
| 62 |
+
findings.append(
|
| 63 |
+
{
|
| 64 |
+
"path": display,
|
| 65 |
+
"rule": "checkpoint_bytes",
|
| 66 |
+
"offset": 0,
|
| 67 |
+
"match_sha256": entry["sha256"],
|
| 68 |
+
}
|
| 69 |
+
)
|
| 70 |
+
for rule, pattern in RULES.items():
|
| 71 |
+
for match in pattern.finditer(data):
|
| 72 |
+
findings.append(
|
| 73 |
+
{
|
| 74 |
+
"path": display,
|
| 75 |
+
"rule": rule,
|
| 76 |
+
"offset": match.start(),
|
| 77 |
+
"match_sha256": __import__("hashlib").sha256(match.group()).hexdigest(),
|
| 78 |
+
}
|
| 79 |
+
)
|
| 80 |
+
return {
|
| 81 |
+
"format": 1,
|
| 82 |
+
"scanner_version": SCANNER_VERSION,
|
| 83 |
+
"rules": sorted(RULES),
|
| 84 |
+
"files": entries,
|
| 85 |
+
"root_sha256": canonical_root(entries),
|
| 86 |
+
"findings": findings,
|
| 87 |
+
"intended_remote_map": intended_remote_map,
|
| 88 |
+
"exit_code": 1 if findings else 0,
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def write_receipt(path: Path, receipt: dict[str, Any], *, require_clean: bool = True) -> None:
|
| 93 |
+
atomic_write_json(path, receipt)
|
| 94 |
+
if require_clean and receipt["exit_code"] != 0:
|
| 95 |
+
raise PrivacyError(f"privacy scanner found {len(receipt['findings'])} findings")
|
src/repro_control/reducers.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exhaustive registered reducers; callers must supply accepted complete seeds."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
import statistics
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from typing import Iterable
|
| 9 |
+
|
| 10 |
+
T95_DF2 = 4.302652729911275
|
| 11 |
+
T90_DF2 = 2.919985580355516
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ReducerInputError(ValueError):
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass(frozen=True)
|
| 19 |
+
class Summary:
|
| 20 |
+
seeds: tuple[float, float, float]
|
| 21 |
+
mean: float
|
| 22 |
+
sample_std: float
|
| 23 |
+
lower: float
|
| 24 |
+
upper: float
|
| 25 |
+
confidence: int
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _three(values: Iterable[float], name: str) -> tuple[float, float, float]:
|
| 29 |
+
row = tuple(float(value) for value in values)
|
| 30 |
+
if len(row) != 3 or not all(math.isfinite(value) for value in row):
|
| 31 |
+
raise ReducerInputError(f"{name} must contain three finite accepted-seed values")
|
| 32 |
+
return row # type: ignore[return-value]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def summarize(values: Iterable[float], confidence: int = 95) -> Summary:
|
| 36 |
+
row = _three(values, "values")
|
| 37 |
+
critical = {95: T95_DF2, 90: T90_DF2}.get(confidence)
|
| 38 |
+
if critical is None:
|
| 39 |
+
raise ReducerInputError("confidence must be 90 or 95")
|
| 40 |
+
mean = statistics.fmean(row)
|
| 41 |
+
sample_std = statistics.stdev(row)
|
| 42 |
+
half = critical * sample_std / math.sqrt(3)
|
| 43 |
+
return Summary(row, mean, sample_std, mean - half, mean + half, confidence)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def paired(left: Iterable[float], right: Iterable[float], confidence: int = 95) -> Summary:
|
| 47 |
+
left_row, right_row = _three(left, "left"), _three(right, "right")
|
| 48 |
+
return summarize((a - b for a, b in zip(left_row, right_row, strict=True)), confidence)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def reduce_c1(
|
| 52 |
+
sheaf: Iterable[float],
|
| 53 |
+
mpnn: Iterable[float],
|
| 54 |
+
*,
|
| 55 |
+
sheaf_parameters: int,
|
| 56 |
+
mpnn_parameters: int,
|
| 57 |
+
) -> dict:
|
| 58 |
+
s, m, gap = summarize(sheaf), summarize(mpnn), paired(sheaf, mpnn)
|
| 59 |
+
mismatch = abs(mpnn_parameters - sheaf_parameters) / sheaf_parameters
|
| 60 |
+
parameter = (
|
| 61 |
+
"PARAMETER_MATCH_REPRODUCED"
|
| 62 |
+
if mismatch <= 0.05
|
| 63 |
+
else "RELEASED_IMPLEMENTATION_NOT_PARAMETER_MATCHED"
|
| 64 |
+
)
|
| 65 |
+
if s.mean >= 85 and m.mean <= 20 and gap.lower > 60:
|
| 66 |
+
functional = "FUNCTIONAL_GAP_REPRODUCED"
|
| 67 |
+
elif s.mean < 85 or m.mean > 20 or gap.upper <= 60:
|
| 68 |
+
functional = "NOT_REPRODUCED_UNDER_PAPER_ROW_RECONSTRUCTION"
|
| 69 |
+
else:
|
| 70 |
+
functional = "INCONCLUSIVE"
|
| 71 |
+
overall = (
|
| 72 |
+
"FUNCTIONAL_GAP_REPRODUCED_BUT_RELEASE_NOT_PARAMETER_MATCHED"
|
| 73 |
+
if functional == "FUNCTIONAL_GAP_REPRODUCED"
|
| 74 |
+
and parameter == "RELEASED_IMPLEMENTATION_NOT_PARAMETER_MATCHED"
|
| 75 |
+
else functional
|
| 76 |
+
)
|
| 77 |
+
return {
|
| 78 |
+
"functional": functional,
|
| 79 |
+
"parameter": parameter,
|
| 80 |
+
"overall": overall,
|
| 81 |
+
"relative_count_mismatch": mismatch,
|
| 82 |
+
"summaries": {"sheaf": s, "mpnn": m, "paired_gap": gap},
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def reduce_c2(sheaf: Iterable[float], mpnn: Iterable[float]) -> dict:
|
| 87 |
+
s, m, gap = summarize(sheaf), summarize(mpnn), paired(sheaf, mpnn, 90)
|
| 88 |
+
if s.mean >= 95 and m.mean >= 95 and gap.lower >= -2 and gap.upper <= 2:
|
| 89 |
+
verdict = "STATE_EFFICIENCY_REPRODUCED"
|
| 90 |
+
elif s.mean < 90 or m.mean < 90 or gap.lower > 2 or gap.upper < -2:
|
| 91 |
+
verdict = "NOT_REPRODUCED"
|
| 92 |
+
else:
|
| 93 |
+
verdict = "INCONCLUSIVE"
|
| 94 |
+
return {
|
| 95 |
+
"verdict": verdict,
|
| 96 |
+
"per_vertex_latent_dimension_ratio": 8.4,
|
| 97 |
+
"memory_reduction_claimed": False,
|
| 98 |
+
"summaries": {"sheaf": s, "mpnn": m, "paired_gap_90": gap},
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _adequate(values: tuple[float, float, float]) -> bool:
|
| 103 |
+
return statistics.fmean(values) >= 98.5 and min(values) >= 98.0
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def reduce_c3(
|
| 107 |
+
cnn_clean: Iterable[float],
|
| 108 |
+
sheaf_clean: Iterable[float],
|
| 109 |
+
cnn_pad16: Iterable[float],
|
| 110 |
+
sheaf_pad16: Iterable[float],
|
| 111 |
+
cnn_drop30: Iterable[float],
|
| 112 |
+
sheaf_drop30: Iterable[float],
|
| 113 |
+
) -> dict:
|
| 114 |
+
cnn_clean_row = _three(cnn_clean, "cnn_clean")
|
| 115 |
+
sheaf_clean_row = _three(sheaf_clean, "sheaf_clean")
|
| 116 |
+
cnn_ok, sheaf_ok = _adequate(cnn_clean_row), _adequate(sheaf_clean_row)
|
| 117 |
+
if not cnn_ok or not sheaf_ok:
|
| 118 |
+
who = "BOTH" if not cnn_ok and not sheaf_ok else ("CNN" if not cnn_ok else "SHEAF")
|
| 119 |
+
verdict = f"INCONCLUSIVE_{who}_CLEAN_REFERENCE_INADEQUATE"
|
| 120 |
+
pad_gap = drop_gap = None
|
| 121 |
+
else:
|
| 122 |
+
pad_gap = paired(sheaf_pad16, cnn_pad16)
|
| 123 |
+
drop_gap = paired(sheaf_drop30, cnn_drop30)
|
| 124 |
+
if pad_gap.lower > 20 and drop_gap.lower > 10:
|
| 125 |
+
verdict = "ROBUSTNESS_GAPS_REPRODUCED"
|
| 126 |
+
elif pad_gap.upper <= 20 or drop_gap.upper <= 10:
|
| 127 |
+
verdict = "NOT_REPRODUCED_UNDER_TIER_B_RECONSTRUCTION"
|
| 128 |
+
else:
|
| 129 |
+
verdict = "INCONCLUSIVE"
|
| 130 |
+
return {
|
| 131 |
+
"verdict": f"{verdict}__DROPOUT_SEMANTICS_TARGET_SELECTED",
|
| 132 |
+
"clean_adequacy": {"cnn": cnn_ok, "sheaf": sheaf_ok},
|
| 133 |
+
"paired_gaps": {"pad16": pad_gap, "drop30": drop_gap},
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def reduce_c4_sudoku(learned: Iterable[float], identity: Iterable[float]) -> dict:
|
| 138 |
+
learned_summary, identity_summary = summarize(learned), summarize(identity)
|
| 139 |
+
gap = paired(learned, identity)
|
| 140 |
+
if learned_summary.mean >= 85 and identity_summary.mean <= 15 and gap.lower > 60:
|
| 141 |
+
verdict = "IDENTITY_ABLATION_REPRODUCED"
|
| 142 |
+
elif learned_summary.mean < 85 or identity_summary.mean > 15 or gap.upper <= 60:
|
| 143 |
+
verdict = "NOT_REPRODUCED"
|
| 144 |
+
else:
|
| 145 |
+
verdict = "INCONCLUSIVE"
|
| 146 |
+
return {"verdict": verdict, "learned": learned_summary, "identity": identity_summary, "gap": gap}
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def reduce_c4_maze(default: Iterable[float], quadratic: Iterable[float]) -> dict:
|
| 150 |
+
default_summary, quadratic_summary = summarize(default), summarize(quadratic)
|
| 151 |
+
gap = paired(default, quadratic)
|
| 152 |
+
if default_summary.mean >= 90 and quadratic_summary.mean >= 90 and gap.upper < 20:
|
| 153 |
+
verdict = "SUPPLIED_COLLAPSE_WORDING_FALSIFIED"
|
| 154 |
+
elif default_summary.mean >= 90 and quadratic_summary.mean <= 60 and gap.lower > 20:
|
| 155 |
+
verdict = "DIRECTION_SUPPORTED_UNDER_RECONSTRUCTION"
|
| 156 |
+
else:
|
| 157 |
+
verdict = "INCONCLUSIVE"
|
| 158 |
+
return {
|
| 159 |
+
"verdict": f"{verdict}__PROMPT_MISSTATES_PAPER_TABLE",
|
| 160 |
+
"default": default_summary,
|
| 161 |
+
"quadratic": quadratic_summary,
|
| 162 |
+
"gap": gap,
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def reduce_c5(by_size: dict[int, Iterable[float]]) -> dict:
|
| 167 |
+
expected = {19, 23, 27, 31, 35, 39}
|
| 168 |
+
if set(by_size) != expected:
|
| 169 |
+
raise ReducerInputError("C5 requires all six registered sizes")
|
| 170 |
+
summaries = {size: summarize(by_size[size]) for size in sorted(by_size)}
|
| 171 |
+
if all(summary.mean >= 95 for summary in summaries.values()):
|
| 172 |
+
verdict = "QUALITATIVE_2X_GENERALIZATION_SUPPORTED_UNDER_REGISTERED_RECONSTRUCTION"
|
| 173 |
+
else:
|
| 174 |
+
verdict = "NOT_REPRODUCED_UNDER_REGISTERED_LONG_PATH_RECONSTRUCTION"
|
| 175 |
+
return {
|
| 176 |
+
"verdict": verdict,
|
| 177 |
+
"exact_protocol": "INCONCLUSIVE_EXACT_DENSE_FIGURE6_CONFIG_UNRELEASED",
|
| 178 |
+
"summaries": summaries,
|
| 179 |
+
}
|
src/repro_control/runtime.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Entrypoint guards shared by import, smoke, train, and evaluation."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
import os
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from .constants import SCIENCE_SPEC_SHA256
|
| 12 |
+
from .manifests import validate_manifest
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class RuntimeContractError(RuntimeError):
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def sha256_file(path: Path) -> str:
|
| 20 |
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def verify_science_spec(path: Path) -> dict:
|
| 24 |
+
if sha256_file(path) != SCIENCE_SPEC_SHA256:
|
| 25 |
+
raise RuntimeContractError("SCIENCE-SPEC.yaml hash mismatch")
|
| 26 |
+
value = json.loads(path.read_text())
|
| 27 |
+
if value.get("outcomes") != {}:
|
| 28 |
+
raise RuntimeContractError("science spec outcomes must remain empty")
|
| 29 |
+
return value
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def load_job_manifest(path: Path, *, freeze_sha256: str = "") -> dict:
|
| 33 |
+
value = json.loads(path.read_text())
|
| 34 |
+
validate_manifest(value, freeze_sha256=freeze_sha256)
|
| 35 |
+
return value
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def assert_smoke_isolated(manifest: dict) -> None:
|
| 39 |
+
if manifest["job_class"] != "GPU_SMOKE":
|
| 40 |
+
raise RuntimeContractError("smoke entrypoint requires GPU_SMOKE")
|
| 41 |
+
forbidden = ("final", "verdict", "test_hard", "test_ood", "evaluation")
|
| 42 |
+
for mount in manifest["mounts"]:
|
| 43 |
+
text = f"{mount['volume_path']} {mount['mount_path']}".lower()
|
| 44 |
+
if any(term in text for term in forbidden):
|
| 45 |
+
raise RuntimeContractError("smoke manifest exposes final/evaluation data")
|
| 46 |
+
for output in manifest["expected_outputs"]:
|
| 47 |
+
if "verdict" in output.lower():
|
| 48 |
+
raise RuntimeContractError("smoke entrypoint cannot emit verdict metrics")
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def assert_evaluation_isolated(manifest: dict) -> None:
|
| 52 |
+
if manifest["job_class"] != "SCIENTIFIC_EVAL":
|
| 53 |
+
raise RuntimeContractError("evaluation entrypoint requires SCIENTIFIC_EVAL")
|
| 54 |
+
for mount in manifest["mounts"]:
|
| 55 |
+
text = f"{mount['volume_path']} {mount['mount_path']}".lower()
|
| 56 |
+
if "train-data" in text or "/train" in text or text.endswith(" train"):
|
| 57 |
+
raise RuntimeContractError("evaluation manifest exposes registered training data")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def require_control_freeze(
|
| 61 |
+
control_dir: Path, expected_freeze_sha256: str, expected_spec_sha256: str
|
| 62 |
+
) -> dict:
|
| 63 |
+
freeze_path = control_dir / "SCIENCE-FREEZE.yaml"
|
| 64 |
+
if not freeze_path.is_file() or sha256_file(freeze_path) != expected_freeze_sha256:
|
| 65 |
+
raise RuntimeContractError("control freeze hash mismatch")
|
| 66 |
+
value = json.loads(freeze_path.read_text())
|
| 67 |
+
if value.get("science_spec_sha256") != expected_spec_sha256:
|
| 68 |
+
raise RuntimeContractError("freeze binds a different science spec")
|
| 69 |
+
return value
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def configure_scientific_runtime() -> None:
|
| 73 |
+
os.environ["JAX_DEFAULT_MATMUL_PRECISION"] = "highest"
|
| 74 |
+
os.environ.setdefault("TRACE_MODE", "none")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def require_jax_platform(expected: str) -> str:
|
| 78 |
+
import jax
|
| 79 |
+
|
| 80 |
+
observed = jax.devices()[0].platform
|
| 81 |
+
if observed != expected:
|
| 82 |
+
raise RuntimeContractError(
|
| 83 |
+
f"scientific runtime requires JAX platform {expected!r}, observed {observed!r}"
|
| 84 |
+
)
|
| 85 |
+
return observed
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def common_parser(description: str) -> argparse.ArgumentParser:
|
| 89 |
+
parser = argparse.ArgumentParser(description=description)
|
| 90 |
+
parser.add_argument("--science-spec", type=Path, required=True)
|
| 91 |
+
parser.add_argument("--job-manifest", type=Path, required=True)
|
| 92 |
+
parser.add_argument("--dry-run", action="store_true", default=False)
|
| 93 |
+
return parser
|