diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index d4245d9..949a97b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - uses: prefix-dev/setup-pixi@v0.9.3 with: @@ -27,7 +27,7 @@ jobs: manifest-path: pyproject.toml run-install: false cache: false - - name: Resolve released dependencies once for this workflow + - name: Resolve example dependencies once for this workflow run: pixi update sequince sequince-density - uses: actions/upload-artifact@v4 with: @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: @@ -81,7 +81,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false - uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/pr-documentation.yml b/.github/workflows/pr-documentation.yml new file mode 100644 index 0000000..a40de1d --- /dev/null +++ b/.github/workflows/pr-documentation.yml @@ -0,0 +1,35 @@ +name: Publish PR documentation +on: + workflow_run: + workflows: [Example documentation] + types: [completed] + +permissions: + actions: read + contents: write + pull-requests: read + +jobs: + publish: + if: >- + github.repository == 'sequince-dev/sequince-examples' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 55 + concurrency: + group: examples-rtd-pr-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + steps: + # workflow_run uses trusted default-branch code. Never check out the PR + # or execute files from its artifacts in this credential-bearing job. + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Publish bundle and rebuild PR preview + run: python3 scripts/publish_pr_docs.py + env: + GH_TOKEN: ${{ github.token }} + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_PROJECT: ${{ vars.RTD_PROJECT }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7cd14f..3626cbe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,8 +43,9 @@ Keep local `main` for upstream changes and develop on separate branches. Install [Pixi](https://pixi.prefix.dev/latest/installation/), then run: ```bash -make -C docs setup -pixi run --locked -e check pre-commit install +pixi update sequince sequince-density +pixi install -e default -e docs -e check +pixi run -e check pre-commit install ``` Dependencies, environments, and tasks are defined in `pyproject.toml`. @@ -82,15 +83,18 @@ Avoid retaining unnecessary histories or adding expensive diagnostics by default Before requesting review, run: ```bash -make -C docs check -make -C docs hooks +pixi run -e check test +pixi run -e check lint +pixi run -e check format-check +pixi run -e check pre-commit run --all-files ``` -The first command runs pytest, Ruff linting, and formatting checks. Pre-commit -also checks spelling, whitespace, YAML, and large files, and may modify files. -Review any edits and rerun the checks. Do not bypass hooks to make commits pass. -The configurations in `pyproject.toml` and `.pre-commit-config.yaml` are -authoritative; this repository does not currently configure mypy. +These commands run pytest, Ruff linting and formatting checks, and pre-commit. +Pre-commit also checks spelling, whitespace, YAML, and large files, and may +modify files. Review any edits and rerun the checks. Do not bypass hooks to make +commits pass. The configurations in `pyproject.toml` and +`.pre-commit-config.yaml` are authoritative; this repository does not currently +configure mypy. Tests live in `tests/`. Test observable behavior and numerical invariants, using small fixed inputs or mocked execution for pipeline tests. Some tests require @@ -106,30 +110,52 @@ directly in the tutorial, with explanations and the relevant source code. The reading order in `docs/index.rst` starts with basic examples, then introduces other methods, with flow preconditioning before persistent sampling. -Documentation commands run from the repository root: +Run documentation commands from the repository root. To generate outputs for +one example, select its declared environment: ```bash -make -C docs help -make -C docs example EXAMPLE=gaussian -make -C docs example EXAMPLE=flow-preconditioning ENV=flow -make -C docs examples -make -C docs docs -make -C docs docs-only -make -C docs validate +pixi run -e default example gaussian +pixi run -e flow example flow-preconditioning ``` -`example` runs one manifest entry; `examples` runs all entries sequentially in -their declared environments. Each execution batch resolves released dependencies -once and uses the same lock throughout. `docs` refreshes dependencies, reuses -valid outputs when possible, and executes examples if outputs are missing or -stale. `docs-only` validates existing outputs and builds Sphinx HTML without -sampling. `validate` runs checks, reruns all examples, then builds the docs. -HTML is written to `docs/_build/html/`. +To run all registered examples sequentially in their declared environments: -Full example execution can consume substantial memory and time. Choose a single -example when developing, and use `docs-only` when valid results are available. -Execution has timeouts, but does not currently enforce a memory limit. Do not -assume that parallel local execution is safe. +```bash +pixi run examples +``` + +Each execution batch resolves released dependencies once and uses the same lock +throughout. Full example execution can consume substantial memory and time; +choose a single example when developing. Execution has timeouts, but does not +currently enforce a memory limit. Do not assume parallel local execution is safe. + +To validate existing outputs, stage the tutorial pages, and build Sphinx HTML: + +```bash +pixi run -e docs docs +``` + +Open `docs/_build/html/index.html` in a browser to review the result. This task +builds with warnings treated as errors and does not execute sampling. Missing or +stale outputs cause it to stop; regenerate those outputs before building again. +Pixi may resolve dependencies and update `pixi.lock` before running a task when +needed. + +The Makefile is a standard Sphinx interface, matching Sequince's documentation. +It supports builders such as `html`, `linkcheck`, and `clean`; `docs` is an alias +for `html`. It does not install dependencies, run checks, or execute examples. +To use Make, stage validated outputs first and run it in the docs environment: + +```bash +pixi run -e docs prepare-docs +pixi run -e docs make -C docs html SPHINXOPTS="-W --keep-going" +``` + +Use `pixi run -e docs make -C docs help` to list Sphinx targets. The Pixi `docs` +task above performs preparation and the HTML build together. + +For full validation, run the checks listed above, then run `pixi run examples` +and `pixi run -e docs docs`, in that order. Stop if any command fails. ### Adding an example @@ -161,6 +187,14 @@ installs `docs/requirements.txt`, fetches the exact matching bundle, and verifie it before building; it does not run sampling. Freshness checks guard `latest` builds, and there is no fallback to results from another commit. +Pull requests run examples and build Sphinx at the PR head commit, matching +Read the Docs previews. After a successful run, the separate +`Publish PR documentation` workflow runs trusted code from the default branch. +It reads the completed run's artifact as data, checks its archive and commit, +publishes it under `docs-pr--`, and requests the PR preview. +PR code and artifacts are never executed in that publishing job. Read the Docs +then performs the same output provenance validation used for `latest`. + Maintainers must configure the Read the Docs project, the repository variable `RTD_PROJECT`, and the secret `RTD_TOKEN` before enabling this hosting workflow. Keep credentials out of source files and pull request jobs. The configuration diff --git a/docs/Makefile b/docs/Makefile index 49f98a1..bcd747f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,66 +1,22 @@ -.DEFAULT_GOAL := help +# Minimal makefile for Sphinx documentation +# -DOCS_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) -REPO_ROOT := $(abspath $(DOCS_DIR)/..) - -PIXI ?= pixi -EXAMPLE ?= gaussian -ENV ?= default - -.PHONY: help setup test lint format-check check hooks example examples refresh docs docs-only validate +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +# Put it first so that "make" without argument is like "make help". help: - @printf '%s\n' \ - 'make setup Refresh released dependencies and install environments' \ - 'make refresh Resolve released dependencies without sampling' \ - 'make check Run tests, lint and formatting checks' \ - 'make test Run pipeline tests' \ - 'make lint Run Ruff lint checks' \ - 'make format-check Check Python formatting' \ - 'make hooks Run all pre-commit hooks (may edit files)' \ - 'make example EXAMPLE=gaussian [ENV=default]' \ - ' Run one example in the selected environment' \ - 'make examples Run all documentation examples in their environments' \ - 'make docs Reuse valid outputs or run examples, then build docs' \ - 'make docs-only Build docs using existing validated outputs' \ - 'make validate Run checks, all examples, then build docs' - -refresh: - cd "$(REPO_ROOT)" && $(PIXI) update sequince sequince-density - -setup: refresh - cd "$(REPO_ROOT)" && $(PIXI) install --locked -e default -e docs -e check - -test: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e check test - -lint: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e check lint - -format-check: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e check format-check - -check: test lint format-check - -hooks: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e check pre-commit run --all-files - -example: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e $(ENV) example $(EXAMPLE) - -examples: - cd "$(REPO_ROOT)" && $(PIXI) run --locked examples + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -docs: refresh - @if ! (cd "$(REPO_ROOT)" && $(PIXI) run --locked -e docs validate-outputs); then \ - $(MAKE) -C "$(DOCS_DIR)" examples || exit $$?; \ - fi - $(MAKE) -C "$(DOCS_DIR)" docs-only +.PHONY: help docs Makefile -docs-only: - cd "$(REPO_ROOT)" && $(PIXI) run --locked -e docs docs +docs: html -# Keep execution before Sphinx even when invoked with make -j. -validate: check - $(MAKE) -C "$(DOCS_DIR)" examples - $(MAKE) -C "$(DOCS_DIR)" docs-only +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/index.rst b/docs/index.rst index f80dd6a..9c1d09b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,5 +45,5 @@ dependencies. For background and the full interface reference, see Sequince's `user guide `_ and `API reference `_. -The examples use :external:py:class:`sequince.ParticleState` to represent the +The examples use :external:py:class:`~sequince.state.ParticleState` to represent the weighted particle population. diff --git a/pixi.lock b/pixi.lock index 98c9df7..03f89d8 100644 --- a/pixi.lock +++ b/pixi.lock @@ -138,7 +138,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-9_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl @@ -154,6 +153,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a9/df/e7c10292f348cf55178a5b3c0092a38ea68891a3945f199ae8b7b8ac9dc1/matplotlib-3.11.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ab/43/86aacc027959108b4c66eeae8b73cedb057dfa6eb3a335d05ad65197081c/kiwisolver-1.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d6/cb/17a130e548ee1777f2a66e09240c531a9bf79454add7ef1198fe1e41c412/corner-2.3.0-py3-none-any.whl @@ -179,7 +179,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.16-hb59dee6_101_cp313.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl @@ -197,6 +196,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a9/29/2a47b700545aca050e0ed73c89ba21e08407e11f5f42a567421b220de6c1/matplotlib-3.11.2-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/f4/e410b8c913da5b3fdbb4d16db0f2d2a0952f59c4db8d52dcf2d421d82044/fonttools-4.66.1-cp313-cp313-macosx_10_13_universal2.whl - pypi: https://files.pythonhosted.org/packages/d6/cb/17a130e548ee1777f2a66e09240c531a9bf79454add7ef1198fe1e41c412/corner-2.3.0-py3-none-any.whl @@ -399,8 +399,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.16.0-h69aa097_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/sequince-dev/sequince-flows.git?rev=cbe32939713f17be2c5deb8f813ba1ec4550ea7e#cbe32939713f17be2c5deb8f813ba1ec4550ea7e - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 + - pypi: git+https://github.com/sequince-dev/sequince-density.git?branch=main#9c57f37db95d702eb75fc8f53d5f656db0ac7de0 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl @@ -414,6 +413,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a9/df/e7c10292f348cf55178a5b3c0092a38ea68891a3945f199ae8b7b8ac9dc1/matplotlib-3.11.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ab/43/86aacc027959108b4c66eeae8b73cedb057dfa6eb3a335d05ad65197081c/kiwisolver-1.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d6/cb/17a130e548ee1777f2a66e09240c531a9bf79454add7ef1198fe1e41c412/corner-2.3.0-py3-none-any.whl @@ -482,8 +482,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/sleef-3.9.0-hb028509_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - - pypi: git+https://github.com/sequince-dev/sequince-flows.git?rev=cbe32939713f17be2c5deb8f813ba1ec4550ea7e#cbe32939713f17be2c5deb8f813ba1ec4550ea7e - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 + - pypi: git+https://github.com/sequince-dev/sequince-density.git?branch=main#9c57f37db95d702eb75fc8f53d5f656db0ac7de0 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl @@ -498,6 +497,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a9/29/2a47b700545aca050e0ed73c89ba21e08407e11f5f42a567421b220de6c1/matplotlib-3.11.2-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/f4/e410b8c913da5b3fdbb4d16db0f2d2a0952f59c4db8d52dcf2d421d82044/fonttools-4.66.1-cp313-cp313-macosx_10_13_universal2.whl - pypi: https://files.pythonhosted.org/packages/d6/cb/17a130e548ee1777f2a66e09240c531a9bf79454add7ef1198fe1e41c412/corner-2.3.0-py3-none-any.whl @@ -533,7 +533,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-9_cp313.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl @@ -547,11 +546,15 @@ environments: - pypi: https://files.pythonhosted.org/packages/76/3e/0fa5690a3a8303d821a46fd35691b1b6f8a9c06b97b41847ca41aed72e90/orng-0.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/89/a5/da8ae6c6f1babe4b68e3e55d43d39b529e29774f10e0910671a6b8c86eb8/ml_dtypes-0.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/8a/69/6a93d8600c339d7687a05857c7907bd4dd8cf88691a5ea106d7a50af90a1/optax-0.2.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/93/b2/696029cd979f5429ec25c12767734bf0e13175d3341a2aeac59127d00ca0/optree-0.20.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/99/a0/24f50e0cfd432b47103677c6fc6b220acc716c4b20dc9d6021a01d7924fa/blackjax-1.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/df/e7c10292f348cf55178a5b3c0092a38ea68891a3945f199ae8b7b8ac9dc1/matplotlib-3.11.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ab/43/86aacc027959108b4c66eeae8b73cedb057dfa6eb3a335d05ad65197081c/kiwisolver-1.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b3/7d/01e62f59e4166af1be6238d9f2b7f3453630c51b8044b412d05b5606532a/absl_py-2.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ca/87/0c0ca8433e1135f6d1acd355cff21f31c3d433d937d6547aa545c8b060b8/jax-0.11.2-py3-none-any.whl @@ -578,7 +581,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.16-hb59dee6_101_cp313.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - - pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/cf/83c3377e7018ee3c455153d9560c6a972b0a407075868f5e58979fc640a2/minipcn-0.3.0a1-py3-none-any.whl @@ -594,11 +596,15 @@ environments: - pypi: https://files.pythonhosted.org/packages/76/3e/0fa5690a3a8303d821a46fd35691b1b6f8a9c06b97b41847ca41aed72e90/orng-0.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/86/16/1a8fd2b19544b84575cf84ef7aa3ad4c173b756d5f087c91f85d1b295777/array_api_compat-1.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/8a/69/6a93d8600c339d7687a05857c7907bd4dd8cf88691a5ea106d7a50af90a1/optax-0.2.8-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/c0/85fa4951ed8fcfc54ea10367e20617b60292780999835f636a6bdbf01c34/optree-0.20.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/99/a0/24f50e0cfd432b47103677c6fc6b220acc716c4b20dc9d6021a01d7924fa/blackjax-1.7.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/29/2a47b700545aca050e0ed73c89ba21e08407e11f5f42a567421b220de6c1/matplotlib-3.11.2-cp313-cp313-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b0/65/5e088a45d0f43cd814bc5bec521c051d42005a472e804b1a36c48dada09b/h5py-3.16.0-cp313-cp313-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/b3/7d/01e62f59e4166af1be6238d9f2b7f3453630c51b8044b412d05b5606532a/absl_py-2.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b3/fcf70b2ed935f26fc15dfd81f72f781d395b09e302af5c86ec285d63524e/array_api_extra-0.11.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ca/87/0c0ca8433e1135f6d1acd355cff21f31c3d433d937d6547aa545c8b060b8/jax-0.11.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/cb/f4/e410b8c913da5b3fdbb4d16db0f2d2a0952f59c4db8d52dcf2d421d82044/fonttools-4.66.1-cp313-cp313-macosx_10_13_universal2.whl @@ -3192,9 +3198,9 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 433687 timestamp: 1786599629846 -- pypi: git+https://github.com/sequince-dev/sequince-flows.git?rev=cbe32939713f17be2c5deb8f813ba1ec4550ea7e#cbe32939713f17be2c5deb8f813ba1ec4550ea7e +- pypi: git+https://github.com/sequince-dev/sequince-density.git?branch=main#9c57f37db95d702eb75fc8f53d5f656db0ac7de0 name: sequince-density - version: 0.1.dev30+gcbe329397 + version: 0.1.dev32+g9c57f37db requires_dist: - array-api-compat>=1.5 - numpy>=1.25.2 @@ -3208,28 +3214,6 @@ packages: - equinox ; extra == 'flowjax' - h5py>=3 ; extra == 'flowjax' requires_python: '>=3.11' -- pypi: git+https://github.com/sequince-dev/sequince.git?branch=main#9968074e158e13760a9af5d5a1be0c28e554d036 - name: sequince - version: 0.1.dev170+g9968074e1 - requires_dist: - - acai-control>=0.1.0 - - scipy>=1.16 - - matplotlib>=3.5 - - numpy>=1.25.2 - - minipcn>=0.3.0a1 - - array-api-compat>=1.13 - - array-api-extra>=0.10 - - optree>=0.19 - - orng>=0.2.1 - - jax>0.5.3 ; extra == 'jax' - - jaxlib ; extra == 'jax' - - blackjax>=1.2 ; extra == 'blackjax' - - emcee>=3.0 ; extra == 'emcee' - - eryn>=1.2.6 ; extra == 'eryn' - - numpy<2.4 ; extra == 'eryn' - - corner ; extra == 'corner' - - h5py>=3 ; extra == 'hdf5' - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/07/ef/9b77fc80fc9bf28978d5649a3c71c0e77efe9e1966e5f608e7c3d5ba86b4/acai_control-0.1.0-py3-none-any.whl name: acai-control version: 0.1.0 @@ -3560,6 +3544,31 @@ packages: - pylint>=2.6.0 ; extra == 'dev' - pyink ; extra == 'dev' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/8a/69/6a93d8600c339d7687a05857c7907bd4dd8cf88691a5ea106d7a50af90a1/optax-0.2.8-py3-none-any.whl + name: optax + version: 0.2.8 + sha256: e3ca2d36c99daab1800ae9dbc0545034382d6bc780b24d969e1b0df65fa31cb4 + requires_dist: + - absl-py>=0.7.1 + - jax>=0.5.3 + - jaxlib>=0.5.3 + - numpy>=1.18.0 + - sphinx>=6.0.0 ; extra == 'docs' + - sphinx-book-theme>=1.0.1 ; extra == 'docs' + - sphinxcontrib-katex ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - ipython>=8.8.0 ; extra == 'docs' + - myst-nb>=1.0.0 ; extra == 'docs' + - matplotlib>=3.5.0 ; extra == 'docs' + - sphinx-gallery>=0.14.0 ; extra == 'docs' + - sphinx-collections>=0.0.1 ; extra == 'docs' + - flax ; extra == 'docs' + - sphinx-contributors ; extra == 'docs' + - setuptools ; extra == 'docs' + - flax>=0.5.3 ; extra == 'test' + - scipy>=1.7.1 ; extra == 'test' + - scikit-learn ; extra == 'test' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/8d/c0/85fa4951ed8fcfc54ea10367e20617b60292780999835f636a6bdbf01c34/optree-0.20.0-cp313-cp313-macosx_11_0_arm64.whl name: optree version: 0.20.0 @@ -3669,6 +3678,20 @@ packages: - sphinx==7.3.7 ; extra == 'docs' - pytest>=8.0.0 ; extra == 'test' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/99/a0/24f50e0cfd432b47103677c6fc6b220acc716c4b20dc9d6021a01d7924fa/blackjax-1.7.1-py3-none-any.whl + name: blackjax + version: 1.7.1 + sha256: be2c7f982f4ab3afef2dd5f075dae271023fdec3d9c5d8b599ba7f2f281524fa + requires_dist: + - jax>=0.9.0 + - jaxlib>=0.9.0 + - numpy>=1.25 + - optax>=0.2.3 + - scipy>=1.13 + - typing-extensions>=4.4.0 + - tqdm>=4.66 ; extra == 'progress' + - jax-tap>=0.3.1 ; extra == 'progress' + requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl name: tqdm version: 4.70.1 @@ -3725,11 +3748,39 @@ packages: requires_dist: - numpy>=1.21.2 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/b3/7d/01e62f59e4166af1be6238d9f2b7f3453630c51b8044b412d05b5606532a/absl_py-2.5.1-py3-none-any.whl + name: absl-py + version: 2.5.1 + sha256: 721200f2f0e9960f2ca9dc3a2a706b201f5f75d812c158f059cbbe29eeafbdb8 + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl name: six version: 1.17.0 sha256: 4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/bc/bc/8d12a3ea38b7569e773efa574964110435b20ab3d9719b0185dc8d716cc5/sequince-0.1.0b1-py3-none-any.whl + name: sequince + version: 0.1.0b1 + sha256: a4c85cfe9c6c0ebd3ebeed5953fec79ef58d6b365df3232d8fff57e8d4954b40 + requires_dist: + - acai-control>=0.1.0 + - scipy>=1.16 + - matplotlib>=3.5 + - numpy>=1.25.2 + - minipcn>=0.3.0a1 + - array-api-compat>=1.13 + - array-api-extra>=0.10 + - optree>=0.19 + - orng>=0.2.1 + - jax>0.5.3 ; extra == 'jax' + - jaxlib ; extra == 'jax' + - blackjax>=1.2 ; extra == 'blackjax' + - emcee>=3.0 ; extra == 'emcee' + - eryn>=1.2.6 ; extra == 'eryn' + - numpy<2.4 ; extra == 'eryn' + - corner ; extra == 'corner' + - h5py>=3 ; extra == 'hdf5' + requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/bd/98/ef2b6fe2903e377cbe870c3b2800d62552f1e3dbe81ce49e1923c53d1c5c/h5py-3.16.0-cp313-cp313-manylinux_2_28_x86_64.whl name: h5py version: 3.16.0 diff --git a/pyproject.toml b/pyproject.toml index 17729a7..4d33e4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ sequince = { version = "*", extras = ["corner", "hdf5"] } [tool.pixi.feature.jax.pypi-dependencies] jax = ">0.5.3" +blackjax = ">=1.2" [tool.pixi.feature.jax.activation.env] JAX_PLATFORMS = "cpu" @@ -30,7 +31,8 @@ JAX_PLATFORMS = "cpu" pytorch-cpu = "*" [tool.pixi.feature.flow.pypi-dependencies] -sequince-density = { version = "*", extras = ["zuko"] } +# Use the public repository until sequince-density is released on PyPI. +sequince-density = { git = "https://github.com/sequince-dev/sequince-density.git", branch = "main", extras = ["zuko"] } [tool.pixi.feature.docs.dependencies] sphinx = "8.2.*" diff --git a/scripts/docs_pipeline.py b/scripts/docs_pipeline.py index 18ad5bf..4a11275 100644 --- a/scripts/docs_pipeline.py +++ b/scripts/docs_pipeline.py @@ -16,6 +16,7 @@ import sys import tarfile import tempfile +import urllib.error import urllib.request from pathlib import Path @@ -128,14 +129,14 @@ def resolved_versions(example: dict) -> dict[str, str]: ): if any("git+" in source for source, _ in matches): raise ValueError( - f"Stale Git lock for {package}; run pixi update sequince sequince-density after the releases are available" + f"Stale Git lock for {package}; run pixi update sequince sequince-density" ) versions[package] = found.pop() return versions def refresh() -> None: - """Resolve the latest released dependencies once per execution batch.""" + """Resolve example dependencies once per execution batch.""" subprocess.run( ["pixi", "update", "sequince", "sequince-density"], cwd=ROOT, @@ -486,13 +487,22 @@ def unpack( def check_latest() -> None: - """Stop stale latest builds both before Sphinx and before RTD upload.""" + """Stop stale latest and PR builds before Sphinx and before RTD upload.""" if os.environ.get("READTHEDOCS_VERSION") == "latest": remote = git("ls-remote", "origin", "refs/heads/main").split()[0] if git("rev-parse", "HEAD") != remote: raise ValueError( "Stale latest checkout: main advanced; rebuild latest" ) + elif os.environ.get("READTHEDOCS_VERSION_TYPE") == "external": + number = os.environ["READTHEDOCS_VERSION"] + if not number.isascii() or not number.isdigit(): + raise ValueError("Invalid Read the Docs PR number") + remote = git("ls-remote", "origin", f"refs/pull/{number}/head") + if not remote or git("rev-parse", "HEAD") != remote.split()[0]: + raise ValueError( + "Stale PR checkout: head advanced; rebuild preview" + ) def fetch() -> None: @@ -501,16 +511,30 @@ def fetch() -> None: sha = git("rev-parse", "HEAD") # No credentials: RTD Community and these assets must be publicly readable. name = f"docs-outputs-{sha}.tar.gz" - base = ( - f"https://github.com/{REPOSITORY}/releases/download/docs-output-{sha}" - ) + tag = f"docs-output-{sha}" + external = os.environ.get("READTHEDOCS_VERSION_TYPE") == "external" + if external: + number = os.environ["READTHEDOCS_VERSION"] + if not number.isascii() or not number.isdigit(): + raise ValueError("Invalid Read the Docs PR number") + tag = f"docs-pr-{number}-{sha}" + base = f"https://github.com/{REPOSITORY}/releases/download/{tag}" with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / name for suffix in ("", ".sha256"): - with urllib.request.urlopen( - base + "/" + name + suffix, timeout=60 - ) as response: - Path(str(path) + suffix).write_bytes(response.read()) + try: + with urllib.request.urlopen( + base + "/" + name + suffix, timeout=60 + ) as response: + Path(str(path) + suffix).write_bytes(response.read()) + except urllib.error.HTTPError as error: + if external and not suffix and error.code == 404: + print( + "PR outputs are not published yet; GitHub Actions " + "will request a preview after publication." + ) + raise SystemExit(183) from error + raise expected = Path(str(path) + ".sha256").read_text().split() if expected != [digest(path), name]: raise ValueError("Downloaded bundle checksum mismatch") diff --git a/scripts/publish_docs.py b/scripts/publish_docs.py index 17bdf21..ce32516 100644 --- a/scripts/publish_docs.py +++ b/scripts/publish_docs.py @@ -39,16 +39,8 @@ def wait_idle(project: str, deadline: float) -> None: time.sleep(20) -def main() -> None: - """Publish only main, verify assets, and monitor the triggered build.""" - sha = git("rev-parse", "HEAD") - if git("ls-remote", "origin", "refs/heads/main").split()[0] != sha: - print("Main has advanced; skipping stale publication") - return - if set(git("diff", "--name-only", "HEAD").splitlines()) - {"pixi.lock"}: - raise ValueError("Refusing publication from a modified checkout") - path = bundle() - tag = f"docs-output-{sha}" +def publish_assets(path: Path, tag: str, target: str) -> None: + """Publish a bundle and checksum without replacing existing assets.""" # Dedicated prereleases in the examples repository cannot become a library # release or GitHub's latest software release. view = subprocess.run( @@ -63,11 +55,11 @@ def main() -> None: "create", tag, "--target", - sha, + target, "--title", tag, "--notes", - f"Validated example documentation outputs for {sha}.", + f"Example documentation outputs: {tag}.", "--prerelease", "--latest=false", "--draft", @@ -98,14 +90,13 @@ def main() -> None: gh("release", "upload", tag, str(asset)) if release["isDraft"]: gh("release", "edit", tag, "--draft=false", "--latest=false") - # The workflow's non-cancelling concurrency lock covers this whole wait. - project = urllib.parse.quote(os.environ["RTD_PROJECT"], safe="") - deadline = time.monotonic() + 2400 - wait_idle(project, deadline) - if git("ls-remote", "origin", "refs/heads/main").split()[0] != sha: - print("Bundle uploaded; main advanced, skipping latest trigger") - return - result = rtd(f"projects/{project}/versions/latest/builds/", "POST") + + +def build_version( + project: str, version: str, sha: str, deadline: float +) -> None: + """Trigger and monitor an RTD version, requiring the expected commit.""" + result = rtd(f"projects/{project}/versions/{version}/builds/", "POST") build_id = result["build"]["id"] while True: build = rtd(f"projects/{project}/builds/{build_id}/") @@ -119,10 +110,29 @@ def main() -> None: if not build["success"] or build["commit"] != sha: raise ValueError( f"RTD build {build_id} failed or checked out another commit; " - "inspect RTD and rerun the workflow for current main" + "inspect RTD and rerun the workflow for the current commit" ) print(f"RTD build {build_id} succeeded for {sha}") +def main() -> None: + """Publish only main, verify assets, and monitor the triggered build.""" + sha = git("rev-parse", "HEAD") + if git("ls-remote", "origin", "refs/heads/main").split()[0] != sha: + print("Main has advanced; skipping stale publication") + return + if set(git("diff", "--name-only", "HEAD").splitlines()) - {"pixi.lock"}: + raise ValueError("Refusing publication from a modified checkout") + publish_assets(bundle(), f"docs-output-{sha}", sha) + # The workflow's non-cancelling concurrency lock covers this whole wait. + project = urllib.parse.quote(os.environ["RTD_PROJECT"], safe="") + deadline = time.monotonic() + 2400 + wait_idle(project, deadline) + if git("ls-remote", "origin", "refs/heads/main").split()[0] != sha: + print("Bundle uploaded; main advanced, skipping latest trigger") + return + build_version(project, "latest", sha, deadline) + + if __name__ == "__main__": main() diff --git a/scripts/publish_pr_docs.py b/scripts/publish_pr_docs.py new file mode 100644 index 0000000..5f57edb --- /dev/null +++ b/scripts/publish_pr_docs.py @@ -0,0 +1,175 @@ +"""Publish PR bundles as data using only trusted default-branch code.""" + +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +import tempfile +import time +import urllib.parse +import zipfile +from pathlib import Path + +from docs_pipeline import REPOSITORY, ROOT, digest, git +from publish_docs import build_version, gh, publish_assets, wait_idle + +MAX_BYTES = 128 * 1024 * 1024 + + +def api(path: str) -> dict | list: + """Read GitHub metadata from the fixed examples repository.""" + return json.loads(gh("api", f"repos/{REPOSITORY}/{path}")) + + +def current_pr(sha: str) -> dict | None: + """Find the open PR whose current head is the completed run's commit.""" + matches = [ + pr + for pr in api(f"commits/{sha}/pulls?per_page=100") + if pr["state"] == "open" + and pr["head"]["sha"] == sha + and pr["base"]["repo"]["full_name"] == REPOSITORY + ] + if len(matches) > 1: + raise ValueError("Multiple open PRs for the same head commit") + return matches[0] if matches else None + + +def verify_bundle(path: Path, sha: str) -> None: + """Check untrusted archive data without extracting or executing files.""" + hashes = {} + manifest = None + size = 0 + # Bound decompression before parsing tar/PAX headers as well as file data. + with gzip.open(path, "rb") as compressed: + contents = compressed.read(MAX_BYTES + 1) + if len(contents) > MAX_BYTES: + raise ValueError("Oversized decompressed PR bundle") + with tarfile.open(fileobj=io.BytesIO(contents), mode="r:") as archive: + for member in archive: + name = Path(member.name) + size += member.size + if ( + not member.isfile() + or name.is_absolute() + or ".." in name.parts + or str(name) != member.name + or member.name in hashes + or member.size < 0 + or size > MAX_BYTES + or len(hashes) >= 2000 + ): + raise ValueError("Unsafe or oversized PR bundle") + stream = archive.extractfile(member) + if stream is None: + raise ValueError("Unreadable PR bundle member") + data = stream.read() + hashes[member.name] = hashlib.sha256(data).hexdigest() + if member.name == "bundle-manifest.json": + manifest = json.loads(data) + elif member.name.endswith("/metadata.json"): + if json.loads(data)["source_commit"] != sha: + raise ValueError("PR metadata commit mismatch") + hashes.pop("bundle-manifest.json", None) + if ( + not manifest + or manifest["schema"] != 1 + or manifest["source_commit"] != sha + or manifest["files"] != hashes + or "execution-pixi.lock" not in hashes + or not any(name.endswith("/metadata.json") for name in hashes) + ): + raise ValueError("PR bundle manifest/commit/checksum mismatch") + + +def download_bundle(run_id: int, sha: str, destination: Path) -> Path: + """Read only the named artifact from this run into an isolated directory.""" + artifacts = api(f"actions/runs/{run_id}/artifacts?per_page=100") + matches = [ + item + for item in artifacts["artifacts"] + if item["name"] == "validated-bundle" and not item["expired"] + ] + if len(matches) != 1 or matches[0]["size_in_bytes"] > MAX_BYTES: + raise ValueError("Missing, ambiguous or oversized PR artifact") + artifact_id = int(matches[0]["id"]) + zipped = destination / "artifact.zip" + with zipped.open("wb") as stream: + subprocess.run( + [ + "gh", + "api", + f"repos/{REPOSITORY}/actions/artifacts/{artifact_id}/zip", + ], + cwd=ROOT, + stdout=stream, + check=True, + ) + if zipped.stat().st_size > MAX_BYTES: + raise ValueError("Oversized PR artifact download") + path = destination / f"docs-outputs-{sha}.tar.gz" + expected = {path.name, path.name + ".sha256"} + with zipfile.ZipFile(zipped) as archive: + members = archive.infolist() + if ( + len(members) != 2 + or {member.filename for member in members} != expected + or sum(member.file_size for member in members) > MAX_BYTES + ): + raise ValueError("Unexpected files in PR artifact") + for name in expected: + (destination / name).write_bytes(archive.read(name)) + checksum = Path(str(path) + ".sha256").read_text().split() + if checksum != [digest(path), path.name]: + raise ValueError("PR artifact checksum mismatch") + verify_bundle(path, sha) + return path + + +def main() -> None: + """Publish successful current PR runs, then request their RTD preview.""" + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + run_id = int(event["workflow_run"]["id"]) + run = api(f"actions/runs/{run_id}") + if ( + run["event"] != "pull_request" + or run["conclusion"] != "success" + or run["path"] != ".github/workflows/documentation.yml" + or run["repository"]["full_name"] != REPOSITORY + ): + raise ValueError("Unexpected workflow run for PR publication") + sha = run["head_sha"] + if not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ValueError("Invalid workflow head commit") + pr = current_pr(sha) + if pr is None: + print("PR closed or advanced; skipping stale publication") + return + number = int(pr["number"]) + with tempfile.TemporaryDirectory(prefix="pr-docs-") as tmp: + path = download_bundle(run_id, sha, Path(tmp)) + current = current_pr(sha) + if current is None or current["number"] != number: + print("PR changed; skipping stale publication") + return + # Tag only the trusted checkout; PR data has a separate release namespace. + publish_assets( + path, f"docs-pr-{number}-{sha}", git("rev-parse", "HEAD") + ) + project = urllib.parse.quote(os.environ["RTD_PROJECT"], safe="") + deadline = time.monotonic() + 2400 + wait_idle(project, deadline) + if current_pr(sha) is None: + print("Bundle uploaded; PR closed or advanced, skipping preview") + return + build_version(project, str(number), sha, deadline) + + +if __name__ == "__main__": + main() diff --git a/tests/test_docs_pipeline.py b/tests/test_docs_pipeline.py index 4c71d44..5548f56 100644 --- a/tests/test_docs_pipeline.py +++ b/tests/test_docs_pipeline.py @@ -16,6 +16,70 @@ read_resolved_versions = pipeline.resolved_versions +@pytest.mark.parametrize( + ("external", "status", "suffix", "cancel"), + [ + (True, 404, "", True), + (False, 404, "", False), + (True, 403, "", False), + (True, 404, ".sha256", False), + ], +) +def test_pending_pr_bundle( + monkeypatch: pytest.MonkeyPatch, + external: bool, + status: int, + suffix: str, + cancel: bool, +) -> None: + """Only a missing PR bundle cancels; broken published assets still fail.""" + from urllib.error import HTTPError + + monkeypatch.setenv( + "READTHEDOCS_VERSION_TYPE", "external" if external else "branch" + ) + monkeypatch.setenv("READTHEDOCS_VERSION", "17" if external else "latest") + monkeypatch.setattr(pipeline, "git", lambda *args: "a" * 40) + monkeypatch.setattr(pipeline, "check_latest", lambda: None) + + def download(url: str, **kwargs: object) -> io.BytesIO: + tag = "docs-pr-17-" if external else "docs-output-" + assert f"/releases/download/{tag}{'a' * 40}/" in url + if not suffix or url.endswith(suffix): + raise HTTPError(url, status, "unavailable", {}, None) + return io.BytesIO(b"bundle") + + monkeypatch.setattr(pipeline.urllib.request, "urlopen", download) + if cancel: + with pytest.raises(SystemExit) as error: + pipeline.fetch() + assert error.value.code == 183 + else: + with pytest.raises(HTTPError): + pipeline.fetch() + + +def test_fetch_pr_restores_exact_bundle( + outputs: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Fetch a PR's actual archive through the full provenance checks.""" + monkeypatch.setattr(pipeline, "BUNDLES", tmp_path) + monkeypatch.setattr(pipeline, "GENERATED", tmp_path / "generated") + path = pipeline.bundle() + data = path.read_bytes() + checksum = Path(str(path) + ".sha256").read_bytes() + monkeypatch.setenv("READTHEDOCS_VERSION_TYPE", "external") + monkeypatch.setenv("READTHEDOCS_VERSION", "17") + + def download(url: str, **kwargs: object) -> io.BytesIO: + assert f"/docs-pr-17-{'a' * 40}/" in url + return io.BytesIO(checksum if url.endswith(".sha256") else data) + + monkeypatch.setattr(pipeline.urllib.request, "urlopen", download) + pipeline.fetch() + assert (pipeline.GENERATED / "results/gaussian/metadata.json").is_file() + + @pytest.fixture(autouse=True) def release_versions(monkeypatch: pytest.MonkeyPatch) -> None: """Use synthetic release versions; no unavailable packages are resolved.""" @@ -141,6 +205,21 @@ def test_latest_checkout_guard(monkeypatch: pytest.MonkeyPatch) -> None: pipeline.check_latest() +def test_pr_checkout_guard(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("READTHEDOCS_VERSION", "17") + monkeypatch.setenv("READTHEDOCS_VERSION_TYPE", "external") + + def git(*args: str) -> str: + if args[0] == "ls-remote": + assert args[-1] == "refs/pull/17/head" + return "b" * 40 + return "a" * 40 + + monkeypatch.setattr(pipeline, "git", git) + with pytest.raises(ValueError, match="Stale PR checkout"): + pipeline.check_latest() + + def test_missing_output_root_is_actionable(tmp_path: Path) -> None: with pytest.raises(ValueError, match="missing example outputs:.*gaussian"): pipeline.validate(tmp_path / "absent") diff --git a/tests/test_publish_pr_docs.py b/tests/test_publish_pr_docs.py new file mode 100644 index 0000000..3db4ff8 --- /dev/null +++ b/tests/test_publish_pr_docs.py @@ -0,0 +1,213 @@ +"""Exercise PR publication without credentials, sampling or external writes.""" + +from __future__ import annotations + +import hashlib +import io +import json +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import publish_pr_docs as publisher # noqa: E402 + +SHA = "a" * 40 + + +def make_bundle(path: Path, sha: str = SHA, unsafe: bool = False) -> None: + """Write a small archive with the real transport manifest format.""" + files = { + "execution-pixi.lock": b"lock", + "gaussian/metadata.json": json.dumps({"source_commit": sha}).encode(), + } + if unsafe: + files["../escape"] = b"bad" + files["bundle-manifest.json"] = json.dumps( + { + "schema": 1, + "source_commit": sha, + "files": { + name: hashlib.sha256(data).hexdigest() + for name, data in files.items() + }, + } + ).encode() + with tarfile.open(path, "w:gz") as archive: + for name, data in files.items(): + member = tarfile.TarInfo(name) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + + +@pytest.mark.parametrize("failure", [None, "commit", "path", "size"]) +def test_archive_validation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None +) -> None: + path = tmp_path / "outputs.tar.gz" + make_bundle( + path, "b" * 40 if failure == "commit" else SHA, failure == "path" + ) + if failure == "size": + monkeypatch.setattr(publisher, "MAX_BYTES", 1) + if failure: + with pytest.raises(ValueError): + publisher.verify_bundle(path, SHA) + else: + publisher.verify_bundle(path, SHA) + assert not (tmp_path.parent / "escape").exists() + + +@pytest.mark.parametrize("failure", [None, "zip-path", "checksum"]) +def test_download_from_exact_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None +) -> None: + source = tmp_path / f"docs-outputs-{SHA}.tar.gz" + make_bundle(source) + zipped = io.BytesIO() + with zipfile.ZipFile(zipped, "w") as archive: + name = "../escape" if failure == "zip-path" else source.name + archive.writestr(name, source.read_bytes()) + archive.writestr( + source.name + ".sha256", + f"{'wrong' if failure == 'checksum' else publisher.digest(source)}" + f" {source.name}\n", + ) + + def api(path: str) -> dict: + assert path == "actions/runs/123/artifacts?per_page=100" + return { + "artifacts": [ + { + "id": 456, + "name": "validated-bundle", + "expired": False, + "size_in_bytes": len(zipped.getvalue()), + } + ] + } + + def download(args: list, **kwargs: object) -> None: + assert args[-1].endswith("actions/artifacts/456/zip") + kwargs["stdout"].write(zipped.getvalue()) + + monkeypatch.setattr(publisher, "api", api) + monkeypatch.setattr(publisher.subprocess, "run", download) + destination = tmp_path / "download" + destination.mkdir() + if failure: + with pytest.raises(ValueError): + publisher.download_bundle(123, SHA, destination) + else: + result = publisher.download_bundle(123, SHA, destination) + assert result.read_bytes() == source.read_bytes() + assert not (tmp_path / "escape").exists() + + +@pytest.fixture +def publication(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list: + """Record the separate trusted publication path for a successful PR run.""" + calls = [] + event = tmp_path / "event.json" + event.write_text(json.dumps({"workflow_run": {"id": 123}})) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("RTD_PROJECT", "examples") + monkeypatch.setattr( + publisher, + "api", + lambda path: { + "event": "pull_request", + "conclusion": "success", + "path": ".github/workflows/documentation.yml", + "repository": {"full_name": publisher.REPOSITORY}, + "head_sha": SHA, + }, + ) + monkeypatch.setattr(publisher, "current_pr", lambda sha: {"number": 17}) + monkeypatch.setattr(publisher, "git", lambda *args: "b" * 40) + monkeypatch.setattr( + publisher, + "download_bundle", + lambda run_id, sha, directory: calls.append(("download", run_id, sha)) + or directory / "bundle.tar.gz", + ) + monkeypatch.setattr( + publisher, + "publish_assets", + lambda path, tag, target: calls.append(("publish", tag, target)), + ) + monkeypatch.setattr( + publisher, + "wait_idle", + lambda *args: calls.append(("wait",)), + ) + monkeypatch.setattr( + publisher, + "build_version", + lambda project, version, sha, deadline: calls.append( + ("build", project, version, sha) + ), + ) + return calls + + +def test_publish_pr_before_preview(publication: list) -> None: + publisher.main() + assert publication == [ + ("download", 123, SHA), + ("publish", f"docs-pr-17-{SHA}", "b" * 40), + ("wait",), + ("build", "examples", "17", SHA), + ] + + +@pytest.mark.parametrize("advance_at", [0, 1, 2]) +def test_stale_or_closed_pr( + publication: list, monkeypatch: pytest.MonkeyPatch, advance_at: int +) -> None: + responses = iter([{"number": 17}] * advance_at + [None]) + monkeypatch.setattr(publisher, "current_pr", lambda sha: next(responses)) + publisher.main() + assert not any(call[0] == "build" for call in publication) + if advance_at < 2: + assert not any(call[0] == "publish" for call in publication) + + +@pytest.mark.parametrize( + ("field", "value"), + [("event", "push"), ("conclusion", "failure"), ("path", "other.yml")], +) +def test_unexpected_run_rejected( + publication: list, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: str, +) -> None: + original = publisher.api + monkeypatch.setattr( + publisher, "api", lambda path: original(path) | {field: value} + ) + with pytest.raises(ValueError, match="Unexpected workflow"): + publisher.main() + assert not publication + + +def test_current_pr_uses_github_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + records = [ + { + "number": 17, + "state": "open", + "head": {"sha": SHA}, + "base": {"repo": {"full_name": publisher.REPOSITORY}}, + } + ] + monkeypatch.setattr(publisher, "api", lambda path: records) + assert publisher.current_pr(SHA)["number"] == 17 + assert publisher.current_pr("b" * 40) is None + records[0]["state"] = "closed" + assert publisher.current_pr(SHA) is None