diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index b93004c2..21d23604 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -57,7 +57,7 @@ jobs: run: tox - name: Convert coverage information - if: ${{ always() && github.event.repository.owner.login == 'bits' }} + if: ${{ always() && github.event.repository.owner.login == 'bitsorg' }} run: | coverage combine .tox/coverage.* # Codecov only understands XML, JSON and LCOV files. @@ -65,12 +65,15 @@ jobs: # finding some source files, but our source shouldn't be affected, so # ignore these errors. coverage json --ignore-errors -o coverage.json + continue-on-error: true # coverage never fails the check - name: Upload coverage information - if: ${{ always() && github.event.repository.owner.login == 'bits' }} + if: ${{ always() && github.event.repository.owner.login == 'bitsorg' }} uses: codecov/codecov-action@v5 with: files: coverage.json + token: ${{ secrets.CODECOV_TOKEN }} + continue-on-error: true mac-unittest: name: mac-python${{ matrix.python-version }} @@ -111,13 +114,14 @@ jobs: # conflicts with the one in o2-full-deps [O2-5580] brew uninstall --ignore-dependencies --force pkg-config@0.29.2 brew install modules alisw/system-deps/o2-full-deps sapling - python3 -m pip install --upgrade tox tox-gh-actions coverage + # tox -e darwin names its env: tox-gh-actions would only warn. + python3 -m pip install --upgrade tox coverage - name: Run tests run: tox -e darwin - name: Convert coverage information - if: ${{ always() && github.event.repository.owner.login == 'bits' }} + if: ${{ always() && github.event.repository.owner.login == 'bitsorg' }} run: | coverage combine .tox/coverage.* # Codecov only understands XML, JSON and LCOV files. @@ -125,12 +129,15 @@ jobs: # finding some source files, but our source shouldn't be affected, so # ignore these errors. coverage json --ignore-errors -o coverage.json + continue-on-error: true # coverage never fails the check - name: Upload coverage information - if: ${{ always() && github.event.repository.owner.login == 'bits' }} + if: ${{ always() && github.event.repository.owner.login == 'bitsorg' }} uses: codecov/codecov-action@v5 with: files: coverage.json + token: ${{ secrets.CODECOV_TOKEN }} + continue-on-error: true # Lint with ruff (issue #59). CHECK-ONLY: the ruleset is configured in # pyproject.toml [tool.ruff.lint] to the real-bug subset (E9/F63/F7/F82), so diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..072a95b3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,85 @@ +# Working on bits + +## Repository map + +- `bits`, `bitsBuild`, and the compatibility wrappers expose the CLI. Some environment commands have shell fast paths in `bits`; check those as well as Python argument parsing when changing CLI behavior. +- `bits_helpers/build.py` coordinates resolution, reuse, source checkout, scheduling, and final synchronization. The generated package build script comes from `bits_helpers/build_template.sh`. +- `bits_helpers/workarea.py` handles source checkout, extraction, and patches; `bits_helpers/download.py` handles downloads and source caches; `bits_helpers/sync.py` implements store backends. +- `bits_helpers/build_progress.py` owns package labels and human-readable stage reporting. `bits_helpers/progress.py` handles CI progress separately. +- Package recipes normally live in separate `*.bits` repositories. Use `tests/testdist` for local recipe fixtures. +- `console-backend/` is a separate Python project with its own README, dependencies, and tests. Root CLI tests do not validate backend changes. +- User documentation lives in `docs/USERGUIDE.md`, `docs/COOKBOOK.md`, and `docs/REFERENCE.md`; keep examples consistent with actual CLI behavior. + +## Development and validation + +Support Python 3.8 and newer. Check `pyproject.toml`, `tox.ini`, and `.github/workflows/pr-check.yml` before using newer APIs or changing dependencies. Linux and macOS are both supported. + +Use an isolated virtual environment with the project's dependencies installed: + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e . pytest ruff +``` + +The runtime dependencies include `boto3` and its `botocore` dependency. Missing them can break test collection as well as store tests. An unrelated installed top-level `tests` package can also shadow this repository's test modules in subprocesses; prefer a clean virtual environment when diagnosing `tests.test_provenance` import errors. + +Run the full suite before every commit, not only the tests of the code you changed: a shared mock or fixture can break elsewhere. From the repository root: + +```bash +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -q tests +ruff check . +git diff --check +``` + +The console backend has its own suite, which the root one does not run: + +```bash +cd console-backend && PYTHONPATH=..:.:tests python3 -m unittest discover -s tests +``` + +Disabling pytest plugin autoload keeps unrelated globally installed plugins out of local runs. Tests are primarily `unittest` based; a focused run can also use: + +```bash +python -m unittest discover -s tests -p 'test_build_progress.py' +``` + +CI uses `tox`, which includes unit tests and integration commands that clone recipes and build packages. See `tox.ini` for the appropriate interpreter environment and external requirements. Use a small fixture package for local build smoke checks; check installed content and failure exit codes, not just successful command startup. + +Ruff is a check-only gate with a deliberately limited ruleset. Preserve surrounding style, including the existing two-space indentation in many modules. Do not run a repository-wide formatter or blanket automatic fixes as part of an unrelated change. Never run `ruff check --fix` on the tree: it removes imports kept only to probe availability (such as `psutil` in `args.py`); mark those `# noqa: F401`. Inspect generated package metadata after editable installs and keep unintended generated changes out of commits. + +## Build correctness + +- Changes to build environments, generated initialization scripts, relocation, or installed layouts can affect every artifact. Review `bits_helpers/build_epoch.py`: bump `BUILD_EPOCH` when artifact contents or compatibility change. `tests/test_build_epoch.py` checks the fingerprint of covered files. +- A reporting-only change to a covered file still needs a refreshed `COVERED_SHA256`, but does not require an epoch bump. Compute the fingerprint with `python -c 'from bits_helpers.build_epoch import covered_sha256; print(covered_sha256())'` and document the decision. +- `bits_helpers/build_template.sh` is formatted with Python's `%` operator. Escape literal percent signs as `%%`, preserve the named placeholders, and validate a generated script when changing shell behavior. +- Preserve real recipe exit codes across `tee` pipelines. Read `${PIPESTATUS[0]}` immediately after the pipeline, before another command overwrites it. +- Exercise source builds, cached archive reuse, and incremental builds when changing shared build-script paths. A successful source build alone does not cover all branches. +- Keep raw source architecture separate from effective installation architecture. Architecture-conditional source entries are resolved with the raw architecture; shared package paths may use an effective architecture. +- A plain build (`bits build`, or aliBuild with a local alidist, no new options) must keep its package hashes, build graph and outputs: no forced rebuild. New behavior is opt-in. New keys in manifests, `.meta.json` and other JSON are additive and read with a fallback. +- An option that changes what a recipe produces must change the package hash, for example through the defaults `env:`, which is hashed; when the option is off it adds nothing, so existing hashes stay the same. How the build host runs a build (sandbox, oversubscription) belongs under the defaults `system:`, which is not hashed. +- `bits-recipe-tools` is a build dependency of nearly every package: a change there rebuilds the whole stack. Fix one package in its own recipe. +- Reuse only what the signed store holds and a signed manifest vouches for. Do not add other reuse sources, such as caches a provider points to. +- Shell code runs on Linux and macOS: use what both GNU and BSD tools accept (`sed -i`, `stat`, `date`). +- Code that deletes state (store objects, buckets, spool directories) runs only on an explicit command or option, never as a side effect of another command. + +## Progress and source reporting + +- Report build progress through `bits_helpers/build_progress.py` and `bits_helpers.log`, so terminal, debug, parallel and redirected output stay consistent; its module docstring describes the stage markers and package numbering, and `tests/test_build_progress.py` covers them. +- Keep the full command output under `--debug`, and report where sources come from at the point they are retrieved, without changing fallback order, checksum checks or redistribution rules. + +## Making changes + +- Keep it simple: the smallest change that solves the problem, no design for cases nobody has. Code comments are at most three lines. +- Give long-lived things (modules, tables, environment variables, events) names that say what they are; temporary names only for what will be removed. +- One behavior per commit. Before changing what a file, variable or trigger means, find everything that reads it. +- A new test must be seen to fail without the change. Test code where it runs (a container, a privileged process, macOS); if you could not, say so in the commit message. +- Get an independent review of every code change, from a reviewer who sees the diff but not your reasoning. +- Measure before optimizing: `BITS_TIMING=1` times the phases of `bits load`/`enter`; time other paths the same way rather than guessing. +- Commit messages are at most five lines and say why the change was needed. Do not refer to ADRs: they are not kept in this repository. + +## Preparing changes for review + +A change in behavior updates the user documentation (`docs/USERGUIDE.md`, `docs/COOKBOOK.md`, `docs/REFERENCE.md`) and `CHANGELOG.md`: one entry under Unreleased, tagged **[Feature]**, **[Fix]** or **[Improvement]**, crediting a contribution as "(from #N, by @user)". New first-party source files start with the SPDX header used elsewhere (`GPL-3.0-or-later`, CERN), except the recipe helper snippets bits hashes. + +Check the current branch and the requested PR base before publishing. This repository can have substantial work on non-default branches, so compare against the intended base to keep unrelated commits out of a PR. Describe observed behavior, relevant validation, and any environment limits accurately; distinguish illustrative output from captured build output. diff --git a/CHANGELOG.md b/CHANGELOG.md index a7249f0b..dc8d535a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,20 +3,130 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ --- # Unreleased -- **[Feature]** `bits cvmfs release --pipeline `: makes a release (its links and merged view) from one build, at any time, from the S3 store alone; the BOMs now carry the build's `cvmfs_templates`, requested packages, `view:` rules and `requires`, and each pipeline's builds are indexed under `BUILDS/by-pipeline/`. -- **[Improvement]** A merged view is one build's packages again, replaced when it differs (no more merging across builds), and is independent of the release links: a group with `cvmfs_views_template` but no `cvmfs_releases_template` gets views. It records its top-level packages; one made from others is replaced only with `--force-view`. `bits cvmfs publish --wait` waits for every job, the packages before the release. +## Scheduled builds and CI on push +How to set them up: WORKFLOWS, Phase 4, steps 4 and 5. They need the orchestrator's build queue (`BITS_ORCH_QUEUE=1`). +- **[Feature]** Nightly slots, made on the console's Schedules tab (**+ New slot**, or **Save as pipeline** in the Build dialog): a package with its defaults chain, variables and repositories, built on each of its platforms × variants on given days at a given time, with one passkey approval for 30 nights of certified builds. The defaults decide where the builds go; a nightly layout's `{day}` is the night's weekday. +- **[Feature]** CI on push: a slot's **Build on each commit to** lists the branches it follows (public https repositories, 8 at most). The orchestrator reads their heads every 2 minutes and, once a burst of pushes has settled (5 minutes, 30 at most), builds every cell as a nightly would, to the store only: never certified nor published. +- **[Feature]** Results: nightly and CI builds apart, in the console's Nightlies and CI builds pages and the phone app's Nightlies and CI tabs (Security moves into More): a row per slot, a column per night or per CI build (with the heads it fired on), a row per cell; the app's slot page has Run now and Run again of the failed cells. + +## Configuration +- **[Change]** Site configuration: the defaults bits had built in (stores, container images, the console, the providers repository, the SBOM namespace, paths) are now in `NN-.yaml` files: `00-bits.yaml` (site-neutral) and `10-CERN.yaml` (CERN), shipped with bits, then `/etc/bits/site.d`, `~/.config/bits/site.d` and `$BITS_SITE_DIR`, merged in name order. A setting is taken from the command line, the `bits use` profile, the environment, the community's `system:` defaults, then these files. At CERN nothing changes; **elsewhere, an empty `/etc/bits/site.d/10-CERN.yaml` turns CERN's values off** and a file of your own sets yours. `bits doctor` shows the files in effect (USERGUIDE "Where defaults come from"). +- **[Improvement]** `bits certify` and `bits compliance --enforce` take the console URL from the site configuration's `console.backend_url` too, so at CERN they need no `--console`. + +## Orchestrator (console-backend) +- **[Feature]** console-backend orchestrator, milestone M0 (`console_backend/orch/`): a worker process (`python -m console_backend.orch.worker`) and a shared SQLite database; a hash-chained event log; typed actions with JSON schemas, approvals (an advisor can only propose), authority re-checked at run time, retries with backoff, one action at a time per subject; `pipeline.retry` and `pipeline.cancel` as the first kinds; `/ops/actions` and `/ops/events`. +- **[Feature]** Orchestrator M1: the worker records every build pipeline from GitLab (outcome decided from its jobs) and the builds it published from the S3 build index; `GET /ops/builds`, `GET /ops/builds/{pipeline}`; CI jobs' hints `POST /ops/ci/published|failed` with an ID token of its own audience. `ci_auth.verify_ci_token` takes the audience to check. +- **[Feature]** Orchestrator M2: failed jobs are recorded from their logs (or the `bits-build.log` artifact when GitLab cut the log), redacted, and deduplicated into issues; seeded signatures name known causes; an optional backfill (`BITS_ORCH_BACKFILL_DAYS`) reads older failures; `GET /ops/issues`, `/ops/issues/{id}`, `/ops/failures`, `/ops/signatures`. +- **[Feature]** Orchestrator M3: approved pre-approvals live in the orchestrator database (with their passkey assertion; they survive a restart; signs counted atomically; expiry and revocation); standing approvals for schedules (passkey-bound to the schedule's sha256 and groups, 1-100 runs or up to 30 days); `GET /approvals`, `POST /approvals/{id}/revoke`; a daily database copy (`BITS_ORCH_BACKUP_DIR`). +- **[Feature]** Orchestrator M4: certification in shadow: the worker recomputes each bits-manifests certification (`certify.build_by_arch`, the signing half of `certify_by_arch` split out; `certify.s3_probe` with a digest cache) and compares it with what was signed; `GET /ops/certifications`, `/ops/certifications/summary`. The outside copy of the audit chain: events are sent to the audit-log service (bits-services), checked against its head every pass; `GET /ops/outside`, `python -m console_backend.orch.outside verify|status|clear`. +- **[Feature]** Orchestrator M5: the certified set: the worker keeps each certified BOM's bytes and follows bits-manifests' main pipelines as versions of the set, every addition and removal an event; BOMs are matched to their store copies; the shadow also computes each certification from the set; `GET /ops/certified`, `/ops/certified/bom/{sha256}`. +- **[Feature]** Orchestrator M6: the backend and the worker can hold no GitLab token or store key: with `BITS_CRED_PROXY_AGENT_SOCKET` they reach GitLab and the store through the credentials proxy (bits-services), with its gate tokens (`credproxy.py`); `forge_ops.GitLabForge` resolves its URL and token per request. +- **[Feature]** The credentials proxy (bits-services `credentials-proxy`): the services' GitLab tokens and store keys in a second security-proxy instance, pushed by bits-services' `make unlock` from an encrypted bundle. `BITS_ORCH_STORE_ANONYMOUS=1` reads a public store with unsigned requests. +- **[Feature]** Orchestrator M8a: the worker replays each CI certification as it would certify it (pre-approval, BOMs, group, admins, key policy), then certifies pre-approved builds itself (`BITS_ORCH_CERTIFY=1`) through a worker-only store writer, first under `CANARY/`, compared with the CI's certification of the same build. +- **[Feature]** Orchestrator M8b (`BITS_ORCH_OWNS_SET=1`): the worker owns the certified set. It writes `MANIFESTS/` itself, numbering each architecture's manifests (`sequence`), mirrors every version of the set to bits-manifests, and puts `MANIFESTS/orchestrator.json` in the store; the backend's `/sign` routes are gone. Admins request `decertify` (a group's BOMs of a build) and `decertify-package` (a group's BOMs without one package), after which the worker withdraws them from the signed manifests; a half-written manifest is repaired; with an outside audit copy it signs only while that copy is in step, and `python -m console_backend.orch.outside recover` rebuilds a restored database from it. +- **[Feature]** Builds are certified as soon as they are in the store (their pipeline running or succeeded), not after their CVMFS publish; `/ops/certify` also serves community admins and says `owns_set`. +- **[Feature]** The build queue (`BITS_ORCH_QUEUE=1`, `POST /ops/queue`): the console's builds wait in the orchestrator, not in GitLab, and start one per free build host of `config/build-hosts.yaml`. A host whose runner is offline is not free (runners found by tag, jobs traced to hosts by runner id), a build GitLab drops unstarted is queued again, and `GET /ops/queue` shows GitLab's refusals as a pause. One passkey approves a queued batch's certifications. +- **[Feature]** For the console: what each build job spent its time on (queued, setup, build, and per package prepub and gateway; `phases` in `GET /ops/builds`), and the CVMFS status document `cvmfs-status.json`, now written by the worker (`BITS_ORCH_CVMFS_STATUS`). +- **[Feature]** `bits trace PATH`: who built, published and approved a file on CVMFS, from its package's `.meta.json` and the orchestrator (`GET /ops/trace`, prepub's records via `BITS_ORCH_PREPUB_URLS`). +- **[Feature]** SBOM scans (`BITS_ORCH_SCANNER_URL`, the bits-services sbom-scanner, grype): every published build is scanned, again when the vulnerability database is newer and when a newer pipeline publishes the build. The scans index which builds contain a package and which packages a scanner can identify. `GET /ops/builds/{pipeline}/sbom` serves a build's SBOM to any signed-in user, with its findings for the admins of its community. +- **[Fix]** A build made with the CLI (`bits publish`, certified from its pre-approval) is recorded as a build once certified, as pipeline 0's with the approved group as its community (database migration 18 reserves pipeline 0). It is scanned, listed with the vulnerabilities and searched like a pipeline's; in upgrade plans and licence analyses it has a release line of its own, without a platform. +- **[Feature]** Risk over time: once a day the worker records each release's open findings and their risk, by severity (a release: a community's top-level package; database migration 19), and its earlier builds at their publish days, with their counts when recorded; `GET /ops/sbom/history?community=&days=` serves it for the Vulnerabilities page's chart. +- **[Feature]** Triage of findings (VEX: `not_affected`, `false_positive`, `will_not_fix`; with a reason, an expiry, revocable), for one version, every version, every package, or every vulnerability of a package (`vuln_id` `*`); triaged findings, and those of host tools and of what only they or the build need, leave the counts. +- **[Feature]** Upgrade plans from the scans, per release line and converged across communities, in waves (known exploited, within the major version, major), minding version pins; licence and dependency analysis: licence conflicts along runtime paths, unknown licences, copyleft reach, the most depended-on, build-only and unneeded packages, and what changed since a line's previous build. +- **[Improvement]** The console backend uses only public `bits_helpers` functions (`cvmfs_release.get_json`, `list_keys`, `is_bom`; `certify.norm_sha`, `now_iso`), and its README states that it is part of bits, deployed from the commit bits-console pins. +- **[Change]** Any signed-in user reads, for the community named (`?community=`), its builds (`/ops/builds`, without who approved their certification, a build without its variables and events), its failed jobs (`/ops/failures`) and its queued builds (`/ops/queue`, without who queued them or the build hosts): the phone app's read-only view. Admins read as before; vulnerabilities, approvals and actions stay theirs. +- **[Change]** The build hosts are the build project's runners tagged `bits-host-` in GitLab (architecture from `bits-build-`), not `config/build-hosts.yaml` (`BITS_ORCH_BUILD_HOSTS` is gone): a runner registered, removed or retagged changes them within 5 minutes. A host whose runners are all paused in GitLab is `paused` in `GET /ops/queue`, not offline; a build whose host or architecture is not among them waits an hour before it is refused, and a runner the token may not see is left out and logged. +- **[Feature]** Nightly schedules in the orchestrator (ADR bits 0007; database migration 20): `GET/POST /ops/schedules`, `PUT/DELETE /ops/schedules/{id}`, `/pause`, `/resume`, in the instance's time zone (site `console.time_zone`). +- **[Feature]** The worker fires the nightly schedules into the build queue (its `schedules` step, with the queue on): one build per platform, the GitLab fan-out's variables, missed nights within 6 hours, a platform still running skipping a night, a night that cannot be built as configured not run, certification under the schedule's standing approval (`schedule:@`). +- **[Feature]** What became of each night: the `nightly_results` step follows each run's build to `running`, `passed`, `failed` (and where) or `not_run` (taken out of the queue after 12 hours without a host); `GET /ops/nightlies` is the nightly matrix, `POST /ops/schedules/{id}/run` Run now or Run again. +- **[Feature]** An overall admin pauses and resumes a build host's runners in GitLab and refreshes its builder images, as actions (`host.pause`, `host.resume`, `host.refresh`). +- **[Feature]** An overall admin prunes a build host's sw until it has the free space asked for, as the action `host.prune` (bits-console's `prune-host` job on that host); the app asks for the one saved for the host (`PUT /ops/hosts/{host}/prune-target`, 200 GiB unless set). +- **[Feature]** Where a failed build job stopped (`phases.failed_in` in `GET /ops/builds`: `build`, `certify`, `publish`, `finish`; database migration 21), from the `build_ok` and `finish_start` marks bits-console's build job writes, so a failed publish is told from a failed build. +- **[Feature]** `GET /ops/admin/ci-var?key=BITS_DESIRED_REF`: the bits version the runners are set to, for any admin. +- **[Improvement]** A build's phases name the build host it was pinned to (`phases.host`), not its runner's description, so the timeline's rows are the hosts. +- **[Fix]** A nightly build's defaults are its schedule's with the community's `defaults_overlay` (without it a Testbed nightly would publish to production's repository), sent as `DEFAULTS` too (the build job resolved its CVMFS target with `release`), and it carries its night's weekday as `--day`, so a nightly layout's `{day}` is the night's even when the queue starts it after midnight UTC. A schedule's extra arguments may not give `--defaults`. +- **[Feature]** A schedule's variants (ADR bits 0009; database migration 22): defaults chains put before its own, so one schedule's night builds platforms × variants, each cell a run of the matrix; Run again takes cells. +- **[Feature]** Commit CI (ADR bits 0009; database migration 23): a schedule may follow branches (`watch`); the worker reads their heads (git protocol v2 over HTTPS, every `BITS_ORCH_WATCH_S`) and, once a moved head has held still for 5 minutes (30 at most) and none of the schedule's commit runs is still going, builds every cell of the newest heads, neither certified nor published; the matrix shows them as commit rows. +- **[Feature]** `GET /ops/worker`: the worker's last pass and which of its steps fail since when, for admins (the errors: overall admins). + +## Signed reuse, trust and certification +- **[Fix]** Trust manifests are fetched afresh by every build (a cached copy was never refreshed), revalidated with their ETag, and verified and parsed from the same bytes; a failed fetch trusts nothing from that manifest instead of an older copy. +- **[Fix]** A manifest entry names an architecture and a hash: a hash alone names no tarball. An object two verified manifests certify with different sha256 is trusted by neither. +- **[Feature]** `system: trust_manifest_prefix` takes several prefixes (e.g. `STAGING/common-manifest,MANIFESTS/common-manifest`): for each architecture the first that has a manifest wins. +- **[Feature]** Anti-rollback: a node remembers the highest `sequence` of each issuer's manifests per architecture (`~/.bits/trust-sequence.json`, `BITS_TRUST_SEQUENCE_STATE`) and refuses an older one put back. +- **[Change]** Where the orchestrator owns the store's certified set (`MANIFESTS/orchestrator.json`), `bits certify` stops after the pre-approval, `bits sign` refuses the store (also when its marker cannot be read), and `bits store rm --stale-boms` and `bits compliance --enforce` request decertify actions; `--enforce` takes out only the restricted packages, asks for the groups first, and fails when you administer none. +- **[Fix]** `bits certify` records who built a package from the build that compiled it, not from one that reused it; BOMs record the build's architecture, not the publishing host's. +- **[Improvement]** `bits publish` checks packages 8 at a time (`BITS_PUBLISH_JOBS`) and `bits certify` reuses the BOM the publish step wrote; waiting out GitLab's mergeability recheck, certification MRs no longer fail with 405 when several merge at once. +- **[Improvement]** Without the Python module `qrcode`, `bits certify` says why its passkey approval shows no QR code; the link is printed as before. + +## Stores +- **[Feature]** Several read and write stores: `--remote-store` and `--write-store` take lists (reads in order, uploads to every write store, revisions from all); each `b3://` bucket has its own credentials (`BITS_S3__*` or a `[bucket NAME]` section). `publish`, `gc` and `compliance` work over all of them, and `certify` publishes to each and certifies the first (issue #130). +- **[Fix]** `bits build --dry-run` and `bits status --check-store` see an `s3://` (s3cmd) store: its objects are listed with `s3cmd ls` (also by a build, for a hash it has not got locally, as for http(s):// and b3:// stores). +- **[Fix]** Offline builds: a read store that cannot be reached is dropped with one warning instead of crashing the build. +- **[Feature]** `fetch_cache: true`: a recipe's own downloads (`$BITS_FETCH_CACHE`) are restored from and saved to the store when the defaults name a cache under `system:` (issue #129). +- **[Change]** CERN builds read `lcgapp-bits-testing` (the site's `store.read`, not with `--write-store` alone nor for aliBuild). + +## Building +- **[Feature]** A build plan before every build (as `--dry-run`), replacing the build order banner: where each package comes from, and build-only dependencies of reused packages marked not needed and neither fetched nor built (issue #141; `--fetch-build-requires` restores the old behaviour). The store is checked 16 hashes at a time with progress, not for packages already here; certified listings are remembered in `/.store-cache/`; hashes changed by packages built here are listed in rounds. The plan says why a package in the store is not reused, and only what the plan fetches or builds is prefetched. +- **[Feature]** An end-of-build reuse summary: how many packages were already installed, taken from the store and built, the built ones with the reason (`rebuilt_because` in the build manifest), also pushed to the build-host metrics. +- **[Feature]** `--symlink-packages-from ROOT` and `--symlink-policy strict|relaxed` (`bits build`, `bits status`): packages of a deployed tree without modulefiles (a CMS `sw` area) are linked into the work directory with private, relocated `init.sh`, and the packages built on them have the deployment's paths rewritten to the work directory. Strict links only packages whose `.build-hash` matches and, with signed reuse on, that the signed manifest certifies; relaxed has loose provenance and refuses `--write-store`. A recipe repository's `SYMLINK_STAGE` hook (not hashed) makes what else needs a private copy (cms.bits: SCRAM). Linked packages are not listed in the store nor prefetched, and their manifest entries carry `symlinked_from` and `symlink_policy` (issue #131, by Akrit Kumar Behera, @akritkbehera). +- **[Fix]** Files to relocate are listed without following symlinks (`grep -r`), which relocation rewrote through or replaced with copies (by Akrit Kumar Behera). `BUILD_EPOCH` stays 0: existing tarballs remain valid. +- **[Feature]** `sandbox` on macOS: with `BITS_SANDBOX` or the defaults' `sandbox: on` a build may write only its own files (`sandbox-exec`; the profile kept as `sandbox.sb`), adopted from aliBuild PR #1062. +- **[Feature]** A defaults variable with `export: false` keeps a `--set` of it out of every hash: only the recipes that use it see it. +- **[Fix]** A defaults chain with a compiler but no build type builds `opt` (`key4hep::gcc15` built as `…-gcc15` and missed every certified `-opt` tarball of the same hash). +- **[Fix]** A patched source tree records the patches' strip levels, so a tree patched at another level is not reused. +- **[Fix]** A store tarball is relocated with this bits' `relocate-me.sh`, not its own; latest links are made with `/bin/ln`, also under the macOS sandbox. +- **[Fix]** A missing Python module says how to install the requirements, for every sub-command, instead of an ImportError traceback. +- **[Fix]** An override key keeps its `:matcher` as written, so one naming a variable in capitals matches; only the package regex is lowercased (#139, by Maria Folina, @m-fol). +- **[Improvement]** A failed package's build log is shown with its absolute path (#136, by Juan Miguel Carceller, @jmcarcell). +- **[Improvement]** The `bits enter` hint after a build names the built version (#142, by Maria Folina, @m-fol), and the family's `latest` alias too. +- **[Improvement]** `bits build` shows each package as `[n/total] name@version` and its stages as they happen (fetching sources, with where they come from; configuring; compiling; installing), in parallel builds too; `--debug` output and compiler percentages are kept (from #134, by Juan Miguel Carceller, @jmcarcell). - **[Feature]** `BUILD_EPOCH` (`bits_helpers/build_epoch.py`): bumping it rebuilds every package once, for changes to how bits builds that the package hashes do not see. At 0 (now) it is not hashed, so no hash changed. A test fails when the build template, `init.sh` generation or relocation scripts change, until their checksum is updated and the epoch considered. -- **[Fix]** `bits q`, `enter`, `load`, `printenv` and `setenv` find packages installed under a `package_family` (`sw////`): they are listed, their modulefiles point at them, and `--dev` sources their `init.sh`. -- **[Fix]** `bits --help`, `bits version` and `bits architecture` no longer create an empty `sw/`; the work directory is created when a command runs, in its `-C` directory. -- **[Improvement]** A recipe repository's `cvmfs.yaml` is read with the first `--defaults` profile taken from that repository, beneath that profile's keys, and never overrides a more specific repository's file (the group's, the repository of the first profile after `release`, then `-c` and `BITS_PATH`): `--defaults atlas::gcc15` publishes with atlas.bits' layout, also from a stacks.bits branch that still has its layout in its defaults. A modules template `bits q` cannot use (e.g. with `{install_dir}`) no longer prints a note. - **[Feature]** A defaults profile with `expand_recipe: true` expands every recipe body strictly (an unknown `%(name)s` is an error); `bits build` and `bits status` both honour it (from #125). - **[Fix]** The `Python` recipe itself gets the `python_*` variables (its body still expanded softly); non-string recipe variables no longer crash (a float warns: quote versions such as `"3.10"`); `relocate-me.sh` no longer rewrites an install base that contains the build directory twice (from #125). +- **[Improvement]** Faster build manifests (SBOMs): the tarball's sha256 is computed while it is packed (a `.sha256` sidecar), so the manifest and the store upload do not read it again; a store object stamped with its sha256 by a server-side managed copy, which also works above 5 GB, so large legacy objects are no longer re-hashed on every publish. +- **[Fix]** A provider's tag that names no branch, tag or commit stops the build with that (`Repository provider 'stacks.bits': no branch or tag 'LCG_110b' in …`); a tag newer than the local mirror is looked up upstream. +- **[Fix]** One bits at a time checks out a provider into the work area (others wait), and a checkout left unfinished by a failed or killed run is redone instead of stopping every later build. + +## SBOM +- **[Feature]** Recipe `references:` (`cpe:2.3:` CPEs and `pkg:` purls, hash-excluded) for vulnerability scanners; `PythonPipRecipe` packages get `pkg:pypi` automatically; further CPEs as `syft:cpe23`; component versions are the upstream ones (the revision is `bits:revision`). +- **[Feature]** System packages are recorded with their versions and a distro purl (one `rpm`/`dpkg` query per build); recipes may name them with `system_package:`. +- **[Feature]** `sbom: tool` marks a host tool (`bits:sbom=tool`): run, never shipped; version pins are recorded (`bits:pinned_by`). + +## bits status +- **[Fix]** `bits status` takes `--set` and resolves the defaults' variables as `bits build` does (the release from `--set` or the recipe branch, exported flavours in the hash), so it reports the build's packages and hashes. +- **[Fix]** `bits status` picks a recipe's source form as `bits build` does (tarballs with `source_mode` tar), and with `--docker` takes the system packages and the toolchain's container fingerprint as a `--docker` build: its hashes now match the build's. + +## bits upgrade +- **[Feature]** `bits upgrade` sets new package versions in place, in the file that sets each (recipe, defaults override, `requires` pin; a provider's recipe copied first), from `--to` or an upgrade plan exported as CSV from the console (`--wave N`). With no tarball of the new version it takes the GitHub release's source tarball (or GNOME's), else a git tag, with `autoreconf -fi` for autotools recipes taken from git. + +## CVMFS publishing and releases +- **[Fix]** `bits cvmfs publish`: paths to a package's dependencies point at the dependencies' own CVMFS paths (from `.meta.json` and the build's templates), not under the package; a dependency it names that has no CVMFS path stops the publish, and files naming other build-area paths are reported. +- **[Feature]** `bits cvmfs release --pipeline `: makes a release (its links and merged view) from one build, at any time, from the S3 store alone; the BOMs now carry the build's `cvmfs_templates`, requested packages, `view:` rules and `requires`, and each pipeline's builds are indexed under `BUILDS/by-pipeline/`. +- **[Improvement]** A merged view is one build's packages again, replaced when it differs (no more merging across builds), and is independent of the release links: a group with `cvmfs_views_template` but no `cvmfs_releases_template` gets views. It records its top-level packages; one made from others is replaced only with `--force-view`. `bits cvmfs publish --wait` waits for every job, the packages before the release. +- **[Feature]** A merged view's `setup.sh` sets what the release's modules set (`_ROOT`, the recipes' setenvs, entries outside the view, read from the modulefiles without Tcl); a release's top-level module points to its view when that covers all, and `bits setenv/enter/load` then source the view's `setup.sh` (no modulecmd, no Python). +- **[Fix]** No release is made for a build of the toolchain alone; `bits cvmfs release` takes the build's architecture from what its BOMs record. +- **[Fix]** The publish upload is streamed from disk instead of built in memory (four workers with a 25 GB package ran a 64 GB host out of memory). +- **[Improvement]** A recipe repository's `cvmfs.yaml` is read with the first `--defaults` profile taken from that repository, beneath that profile's keys, and never overrides a more specific repository's file (the group's, the repository of the first profile after `release`, then `-c` and `BITS_PATH`): `--defaults atlas::gcc15` publishes with atlas.bits' layout, also from a stacks.bits branch that still has its layout in its defaults. A modules template `bits q` cannot use (e.g. with `{install_dir}`) no longer prints a note. - **[Fix]** `bits cvmfs publish`: the release view adds only the links that are not published yet, instead of resending all of them (a shared root fails on existing ones). - **[Feature]** `bits cvmfs publish --replace-on-conflict` replaces only content another build published: a package whose published hash differs, or a merged view whose fingerprint (now in its `.meta.json`) differs, is sent with `replace` and prepub deletes the old subtree first. Needs cvmfs-prepub with `replace_on_conflict`, checked before uploading. Modulefiles, release views and aliases are never replaced, and the package identity is now always sent (no more UNIQUE failures on existing modulefiles). - **[Improvement]** The merged view's `setup.sh`/`setup.csh` put the view's man pages on `MANPATH` when it is set (unset, `man` already finds them from `PATH`); `share/man` stays a real directory in the view. - **[Improvement]** `bits cvmfs publish`: with a packages and a modules template, modulefiles are no longer a commit each: the package carries its modulefile, and one job at the end links `//` to it for every package that lacks one (about a quarter of a large publish). -- **[Improvement]** The tarball's sha256 is computed while it is packed (a `.sha256` sidecar), so the build manifest and the store upload no longer read the tarball again; a legacy store object over 5 GB is now stamped with its sha256 (multipart copy) instead of being re-hashed on every publish. -- **[Improvement]** Faster build manifests (SBOMs): the tarball's sha256 is computed while it is packed (a `.sha256` sidecar), so the manifest and the store upload do not read it again; a store object stamped with its sha256 by a server-side managed copy, which also works above 5 GB, so large legacy objects are no longer re-hashed on every publish. + +## Environment commands (`bits q`, `enter`, `load`, `setenv`) +- **[Fix]** `bits q/enter/load/setenv`: a community's CVMFS modules come from the trees of the given build type (and of none: the toolchain), never the other one; no build type is `-opt`; `bits q` notes the other build type when published. +- **[Fix]** `bits q`, `enter`, `load`, `printenv` and `setenv` find packages installed under a `package_family` (`sw////`): they are listed, their modulefiles point at them, and `--dev` sources their `init.sh`. +- **[Feature]** Without a work directory or Python: the community's CVMFS module trees (`$BITS_CVMFS_PREFIX` or a `cvmfs.yaml`), the architecture for this host's CPU and OS, `bits --community NAME` before the sub-command, a `bits use` profile read only when the directory has one. `bitsenv` is removed: `bits --community NAME setenv …` replaces it. +- **[Feature]** CVMFS trees by install directory (a modules template with `{install_dir}`, alice.bits), and `--archive` also searches `Modules/archive`, as alienv does. +- **[Feature]** `bits overlay lcg --closure META`: the manifest lists only the packages of a view package's dependency closure, at their versions, not everything in the work dir; members not found under the arch are reported. + +## Command line +- **[Fix]** `bits --help`, `bits version` and `bits architecture` no longer create an empty `sw/`; the work directory is created when a command runs, in its `-C` directory. +- **[Improvement]** `bits -h` lists the common options and the commands by group; `bits help COMMAND` is `COMMAND --help`; usage names `bits`, help goes to stdout, and a mistake gets one line. `bits -n prune` is a dry run again (the `-n` was taken by bits and ignored). + +## Project +- **[Improvement]** `AGENTS.md`: the project's rules for changes and builds (compatibility, hashing, signed reuse, portability, review, commits), with development and build validation guidance for agents (#138, by Juan Miguel Carceller, @jmcarcell). +- **[Fix]** CI uploads coverage to Codecov on bitsorg (the owner check named `bits`). # 0.6 — 2026-10-04 diff --git a/README.md b/README.md index 5a866ddd..85f63253 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,9 @@ bits build --parallel 4 ROOT # build up to 4 independent packages at once (--bui ## Basic Commands +`bits help` lists every command by group; `bits help COMMAND` (the same as +`bits COMMAND --help`) prints one command's options. + | Command | Description | |---------|-------------| | `bits build ` | Build a package and its dependencies (`--dry-run` prints the reuse plan instead). | @@ -85,8 +88,8 @@ bits build --parallel 4 ROOT # build up to 4 independent packages at once (--bui | `bits use [ ]` | Save options per directory (`.bitsuse`) so you do not repeat them; with no arguments, show the active profile. | | `bits brew` | macOS: write the Homebrew `Brewfile` for the stack (`sw//Brewfile`). | | `bits publish` | Relocate a built package (or, with `--release-view`, a release view) and hand it to CVMFS via cvmfs-prepub. | -| `bits certify` | Make a build trusted for reuse: upload what is missing, approve with a passkey via bits-console, open the certification MR. | -| `bits sign` | Merge build manifests into a signed common manifest (run by the manifests CI after `bits certify`). | +| `bits certify` | Make a build trusted for reuse: upload what is missing, approve with a passkey via bits-console, open the certification MR (where the orchestrator owns the store's certified set, it certifies the approved build and no MR is opened). | +| `bits sign` | Merge build manifests into a signed common manifest (run by the manifests CI after `bits certify`; refused for a store the orchestrator signs). | | `bits sbom ` | Export a build manifest as CycloneDX 1.6 and/or SPDX 2.3 JSON. | | `bits checksums` | Hash every tarball and patch of a recipe repository and pin git tags (`--write` records them). | | `bits overlay lcg` | Emit an LCG release view (`LCG_externals` + merged `setup.sh`) over a built closure (was `bits lcg-view`). | @@ -130,7 +133,7 @@ Global settings come from environment variables: | Variable | Related flag | Description | |----------|--------------|-------------| -| `$BITS_COMMUNITY` | — | Community name (uppercase), e.g. `LHCB` (`$BITS_ORGANISATION`, its former name, still works). Used only when `-c`/`--config-dir` names a directory that does not exist: bits then clones that community's recipe repository from the registry and uses it. The `aliBuild` wrapper sets `ALICE`. | +| `$BITS_COMMUNITY` | — | Community name (uppercase), e.g. `LHCB` (`$BITS_ORGANISATION`, its former name, still works). Used only when `-c`/`--config-dir` names a directory that does not exist: bits then clones that community's recipe repository from the registry and uses it. The `aliBuild` wrapper sets `ALICE`; `bits --community NAME ` sets it too. | | `$BITS_WORK_DIR` | `-w` / `--work-dir` | Output directory for built packages (default: `sw`). | | `$BITS_REPO_DIR` | `-c` / `--config-dir` | Root directory for recipe repositories. | | `$BITS_PROVIDERS` | — | URL of the bits-providers registry, optionally `@tag` (default `https://github.com/bitsorg/bits-providers`; off under the `aliBuild` wrapper). | @@ -144,6 +147,13 @@ internally when resolving the community recipe repository from bits-providers (e.g. `LHCB` → `lhcb.bits.sh` → `https://github.com/bitsorg/lhcb.bits`). Normally you do not need it: check out the community repository and run bits inside it. +Installation-wide defaults (stores, container images, the console) come from +**site configuration** files, `site.d/NN-.yaml`. bits ships CERN's as +`10-CERN.yaml`; elsewhere, mask it with an empty `/etc/bits/site.d/10-CERN.yaml` +and add your own. The command line, the `bits use` profile, the environment and the +community's `system:` defaults all come first +([where defaults come from](docs/USERGUIDE.md#where-defaults-come-from)). + [Configuration details](docs/USERGUIDE.md#4-configuration) --- diff --git a/bits b/bits index f2bfb52a..d29c1c34 100755 --- a/bits +++ b/bits @@ -15,13 +15,105 @@ BITSDIR="$(dirname "$0")" BITS_PY= [[ -x "$BITSDIR/runtime/bin/python3" ]] && BITS_PY="$BITSDIR/runtime/bin/python3" +function printHelp() { + # The commands, by group. Every command gives its own help with --help. + cat <<'EoF' +bits: build, publish and use software stacks. + +Usage: bits [--community NAME] COMMAND [OPTIONS] + bits help [COMMAND...] the same as: bits COMMAND... --help + bits --version + + --community NAME first, for every command: the community ($BITS_COMMUNITY) + -h, --help this help; after a command, that command's help + -d, --debug debug output; not for the module commands, cvmfs, store, + use, overlay and preload + +Using software (bits enter --help: their options, e.g. -a ARCH, -w DIR): + q, query [REGEXP] list the modules: local builds, then the community's + enter MODULES open a shell with MODULES loaded + setenv MODULES -c CMD run CMD with MODULES loaded + load, printenv MODULES print the shell code loading MODULES (to eval) + unload MODULES print the shell code unloading MODULES (to eval) + list, avail the loaded modules; all modules + modulecmd ARGS run modulecmd with ARGS + shell-helper a bits shell function for your shell startup file + +Building: + build PACKAGE build a package and its dependencies + status PACKAGE what bits build would do + deps PACKAGE the dependency graph + doctor [PACKAGE] check this machine (and the recipes) for a build + init [PACKAGE] check out PACKAGE to develop it here + use [SECTION] ARGS save options for the commands run in this directory + checksums check (--write: record) a repository's checksums + upgrade PACKAGE set new versions of packages in the recipes + stats the resource report of a monitored build + brew a Homebrew Brewfile from the recipes (macOS) + clean remove temporary build trees and unused versions + prune evict old packages from a persistent work directory + architecture, version the detected architecture; the bits version + +CVMFS: + cvmfs SUBCOMMAND a deployed tree (platforms, show, summary), or + publishing a build (stage, publish, release) + publish [PACKAGE] publish packages of the work directory + cvmfs-path where a package is published + import a foreign CVMFS deployment as a reuse overlay + overlay FORMAT the built packages in another format (e.g. lcg) + preload CVMFS prefetch files for deployed packages + +Trust and compliance: + certify make a build trusted (passkey approval) + sign sign the common manifest (manifests CI) + verify check a deployment against its build manifest + sbom a build manifest as an SBOM (CycloneDX, SPDX) + trace FILE who built, published and approved a file on CVMFS + compliance audit licence metadata and the store + +Binary store: + store SUBCOMMAND ls, cat, verify, stats, upload, rm, gc + +Documentation: https://github.com/bitsorg/bits/tree/main/docs +EoF +} + +# `bits --community NAME …` (first, before the sub-command): the +# same as $BITS_COMMUNITY=NAME, e.g. for a grid job's command line. +while [[ ${1:-} == --community || ${1:-} == --community=* ]]; do + if [[ $1 == --community ]]; then _bits_c=${2-}; shift $(( $# > 1 ? 2 : 1 )); else _bits_c=${1#*=}; shift; fi + [[ $_bits_c =~ ^[A-Za-z0-9][A-Za-z0-9_-]*$ ]] || { echo "bits: --community needs a community name, not '$_bits_c'" >&2; exit 1; } + export BITS_COMMUNITY=$(printf %s "$_bits_c" | tr '[:lower:]' '[:upper:]') + unset _bits_c +done + +# bits, bits -h|--help|help: the commands; bits help COMMAND: COMMAND --help. +case "${1:-}" in + "") printHelp >&2; exit 1 ;; + -h|--help) printHelp; exit 0 ;; + help) shift; (( $# )) || { printHelp; exit 0; } + set -- "$@" --help ;; +esac + # `.bitsuse` saved-arg profile (see `bits use`): inject the [common] + [] # tokens right after the action, BEFORE anything is parsed, so both the build # path and the module commands (q/enter/…, which read ARGV below) see them. -# The profile is a local ./.bitsuse (or legacy ./.bitscmd) or a per-directory -# record under ~/.bits/use; the guard fires for any of them and python resolves -# which applies (a no-op when none does). `use` opts out; explicit args stay last. -if [[ -f .bitsuse || -f .bitscmd || -d "$HOME/.bits/use" ]]; then +# The profile is a local ./.bitsuse (or legacy ./.bitscmd) or this directory's +# record under ~/.bits/use (~/.bits/use/.use, +# as bits_use names it); python reads it only when one is there, so the +# environment commands start no Python otherwise. `use` opts out; explicit args +# stay last. +function bitsUseProfile() { + # Whether a profile may apply here (also when the record's name cannot be + # computed: python then decides). + [[ -f .bitsuse || -f .bitscmd ]] && return 0 + [[ -d "$HOME/.bits/use" ]] || return 1 + local d k + d=$(pwd -P) || return 0 + k=$( { printf %s "$d" | sha256sum || printf %s "$d" | shasum -a 256; } 2> /dev/null | cut -c1-16) + [[ ${#k} != 16 || -f "$HOME/.bits/use/$k.use" ]] +} +if bitsUseProfile; then _bits_prof=() while IFS= read -r -d '' _bits_t; do _bits_prof+=("$_bits_t"); done \ < <(BITS_SELF="${BITSDIR}" "${BITS_PY:-python3}" ${BITS_PY:+-E -s} -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' --rewrite0 -- "$@") @@ -60,88 +152,46 @@ fi # else GNU date; no overhead when unset. [[ -n "${BITS_TIMING:-}" ]] && _BT_PREV="${EPOCHREALTIME/,/.}" && _BT_PREV="${_BT_PREV:-$(date +%s.%N 2>/dev/null || echo 0)}" -function printHelp() { -cat >&2 <${EZ} - ${ET}eval "\`bits shell-helper\`"${EZ} - - ${EM}build${EZ} Build a package. Use ${ET}bits build --help${EZ} for details. - ${EM}checksums${EZ} Compute/record a recipe repository's checksums. Use ${ET}bits checksums --help${EZ} for details. - ${EM}clean${EZ} Clean build artifacts. Use ${ET}bits clean --help${EZ} for details. - ${EM}compliance${EZ} Audit licence metadata and the store. Use ${ET}bits compliance --help${EZ} for details. - ${EM}deps${EZ} Show dependency tree. Use ${ET}bits deps --help${EZ} for details. - ${EM}doctor${EZ} Verify build environment. Use ${ET}bits doctor --help${EZ} for details. - ${EM}init${EZ} Initialise a local package. Use ${ET}bits init --help${EZ} for details. - ${EM}prune${EZ} Evict stale packages from a persistent workDir. Use ${ET}bits prune --help${EZ} for details. - ${EM}sbom${EZ} Export a build manifest as a CycloneDX/SPDX SBOM. Use ${ET}bits sbom --help${EZ} for details. - ${EM}store${EZ} Inspect/verify/clean the S3 content store. Use ${ET}bits store --help${EZ} for details. - ${EM}version${EZ} Display bits version: tag, commit and date (also ${ET}bits --version${EZ}). - +function printModuleHelp() { + cat <<'EoF' +bits: the module commands, for the packages built here and those a community +published on CVMFS ($BITS_CVMFS_PREFIX, set by --community on CVMFS, or a +recipe repository's cvmfs.yaml). Local builds come first. + +Usage: bits [OPTIONS] COMMAND [MODULES] ... MODULES: PKG[/VERSION][,PKG...] + + q, query [REGEXP] list the modules (matching REGEXP) + enter MODULES open a shell with MODULES loaded; leave it with exit + setenv MODULES -c CMD run CMD (all after -c) with MODULES loaded + load, printenv MODULES print the shell code loading MODULES: + eval "$(bits load ROOT/latest)" + unload MODULES print the shell code unloading MODULES + list the loaded modules + avail all modules, through modulecmd + modulecmd ARGS run modulecmd with ARGS (e.g. zsh load ROOT/v6) + shell-helper a bits function for load/unload in your shell: + eval "$(bits shell-helper)" + +Options: + -a, --architecture ARCH default: the one installed, else the community's for + this host, else the host's + -w, --work-dir DIR default: $BITS_WORK_DIR, else sw or ../sw; not needed + for a community's CVMFS modules alone + --no-refresh do not refresh the modules directory + -q no module-load messages + --view collapse the paths onto the merged view (automatic + in enter and setenv for a package with view: true) + --archive also the retired modules (Modules/archive, as alienv) + --dev enter: the packages' own init.sh, for development + --shellrc enter: read your shell startup file too + +The shell is the one you run, or $MODULES_SHELL. EoF -[[ -z "$1" ]] || printf "${ER}ERROR: $*${EZ}\n" >&2 +} + +function usageError() { + printf "${ER}bits: %s${EZ}\n" "$*" >&2 + echo "Run 'bits help' for the commands, 'bits help COMMAND' for one of them." >&2 } function installHint() { @@ -216,6 +266,7 @@ function installDir() { } function collectModules() { + [[ -z "$NO_WORK_DIR" ]] || return 0 # CVMFS modules only if [[ -n "$NO_REFRESH" ]]; then printf "${EY}WARNING: not updating modulefiles${EZ}\n" >&2 return 0 @@ -295,7 +346,7 @@ function listAvailableModules() { # `modulecmd avail` just to enumerate — the two costs that make `q` slow on a # real filesystem. Output is unsorted; callers pipe through sort. local base="${WORK_DIR}/${ARCHITECTURE}" P pkgver pkgname - [[ -d "$base" ]] || return 0 + [[ -z "$NO_WORK_DIR" && -d "$base" ]] || return 0 while read -r P; do [[ -n "$P" ]] || continue pkgver=${P##*/} @@ -336,27 +387,52 @@ function publishedArchs() { for d in "$pre"*; do [[ -d "$d$post" ]] && echo "${d#"$pre"}"; done } -function hostCvmfsArch() { - # hostCvmfsArch TREE: the published build arch (without -opt/-dbg) for this - # host's CPU and OS, when exactly one matches; as the CVMFS entry point picks - # its view: the CPU token, and el (or slc/alma/rhel/rocky/centos) or - # ubuntu from /etc/os-release. - local cpu os a b bases=() - cpu=$(uname -m) - os=$( . /etc/os-release 2> /dev/null +function archTokens() { + # archTokens ARCH: its CPU (x86_64, aarch64, ...) and OS (el, from + # el/slc/alma/rhel/rocky/centos, or ubuntu), as "CPU OS". + local cpu= os= + [[ "$1" =~ (x86[-_]64|aarch64|arm64|ppc64le|riscv64) ]] && cpu=${BASH_REMATCH[1]/x86-64/x86_64} + if [[ "$1" =~ (^|[-_])(el|slc|alma|rhel|rocky|centos)([0-9]+)([-_]|$) ]]; then + os=el${BASH_REMATCH[3]} + elif [[ "$1" =~ (^|[-_])(ubuntu[0-9]+)([-_]|$) ]]; then + os=${BASH_REMATCH[2]} + fi + echo "$cpu $os" +} + +function hostMatches() { + # hostMatches: the architectures on stdin built for this host's CPU and OS + # (from /etc/os-release), as the CVMFS entry point picks its view. + tokensMatch "$(uname -m)" "$( . /etc/os-release 2> /dev/null if [[ ${PLATFORM_ID:-} =~ ^platform:(el[0-9]+)$ ]]; then echo "${BASH_REMATCH[1]}" elif [[ ${ID:-} == ubuntu ]]; then echo "ubuntu${VERSION_ID//./}" - fi ) - [[ -n "$os" ]] || return 0 + fi )" +} + +function tokensMatch() { + # tokensMatch CPU OS: the architectures on stdin with that CPU token and that + # OS (el also as slc/alma/rhel/rocky/centos); none without an OS. + local cpu=$1 os=$2 a + [[ -n "$cpu" && -n "$os" ]] || return 0 while read -r a; do [[ "$a" == *"$cpu"* || "$a" == *"${cpu/_/-}"* ]] || continue case $os in el*) [[ "$a" =~ (^|[-_])(el|slc|alma|rhel|rocky|centos)${os#el}([-_]|$) ]] || continue ;; *) [[ "$a" =~ (^|[-_])$os([-_]|$) ]] || continue ;; esac + echo "$a" + done + return 0 +} + +function hostCvmfsArch() { + # hostCvmfsArch TREE: the published build arch (without -opt/-dbg) for this + # host (hostMatches), when exactly one matches. + local a b bases=() + while read -r a; do b=${a%-opt}; b=${b%-dbg} [[ " ${bases[*]} " == *" $b "* ]] || bases+=("$b") - done < <(publishedArchs "$1") + done < <(publishedArchs "$1" | hostMatches) (( ${#bases[@]} == 1 )) && echo "${bases[0]}" return 0 } @@ -368,10 +444,23 @@ function isModuleAction() { function cvmfsTrees() { # cvmfsTrees TREE ARCH: add to CVMFS_TREES the trees (TREE with {arch}) of - # ARCH, of ARCH without its -opt/-dbg, then of that with -opt and -dbg. - local a b t + # ARCH and of ARCH without its -opt/-dbg (the toolchain); ARCH without one is + # -opt. Never the other build type. With install dirs in place of archs + # (CVMFS_SLOT), ARCH if it is one, else the one with ARCH's CPU and OS. + local a b t cpu os cands b=${2%-opt}; b=${b%-dbg} - for a in "$2" "$b" "$b-opt" "$b-dbg"; do + if [[ "$b" == "$2" ]]; then cands=("$2" "$2-opt"); else cands=("$2" "$b"); fi + if [[ -n "$CVMFS_SLOT" ]]; then + cands=("$2"); t=${1//"{arch}"/"$2"} + if [[ ! -d "$t" ]]; then + read -r cpu os <<< "$(archTokens "$2")" + a=$(publishedArchs "$1" | tokensMatch "$cpu" "$os") + [[ -n "$a" && "$a" != *$'\n'* ]] || return 0 + cands=("$a") + fi + fi + CVMFS_ARCH=$2 + for a in "${cands[@]}"; do t=${1//"{arch}"/"$a"} [[ -d "$t" ]] || continue # Physical, as BASE/1.0 resolves its BASEDIR (file normalize): the same @@ -392,22 +481,27 @@ function catalogListing() { function listCvmfsModules() { # Emit / for every modulefile in the CVMFS module trees (no - # BASE), with find, or first the serving catalog (catalogListing). - local t bm= + # BASE), with find, or first the serving catalog (catalogListing), and in + # their archives with --archive. + local t i bm= (( ${#CVMFS_TREES[@]} )) || return 0 if ! catalogListing; then : elif [[ -n "$BITS_PY" ]]; then bm="${BITSDIR}/bitsModules" else bm=$(command -v bitsModules 2> /dev/null || echo "${BITSDIR}/bitsModules"); fi - for t in "${CVMFS_TREES[@]}"; do + for ((i = 0; i < ${#CVMFS_TREES[@]}; i++)); do + t=${CVMFS_TREES[i]} { [[ -n "$bm" && -x "$bm" ]] && ${BITS_PY:+"$BITS_PY" -E -s} "$bm" "$t" 2> /dev/null; } || ( cd "$t" && find . -mindepth 2 -maxdepth 2 \( -type f -o -type l \) | sed -e 's|^\./||' ) + t=${CVMFS_ARCHIVES[i]} + [[ -z "$t" ]] || ( cd "$t" && find . -mindepth 2 -maxdepth 2 \( -type f -o -type l \) | sed -e 's|^\./||' ) done | grep -E '^[^/.][^/]*/[^/.][^/]*$' | grep -v '^BASE/' || true } function resolveModules() { # Find the module tree of each module [/]: the local MODULEPATH # (the modulefile cache, reuse overlays, the user's own entries), then the - # CVMFS trees in order; the first that has it wins. / is a + # CVMFS trees in order (each with its archive, with --archive); the first that + # has it wins. / is a # file there, a bare a directory. Testing that directly is instant; # `modulecmd avail` (a full scan) runs only when no tree has it, so # default-version aliases still resolve locally. The `..` guard keeps the @@ -415,7 +509,7 @@ function resolveModules() { # Sets GROUP_TREE (a CVMFS tree, or "" for the local MODULEPATH) and # GROUP_MODS (its modules, space-separated), one entry per tree used, in tree # order. - local M t i found mp trees=("" "${CVMFS_TREES[@]}") mods=() + local M t i found mp trees=("" "${CVMFS_TREES[@]}") arcs=("" "${CVMFS_ARCHIVES[@]}") mods=() IFS=: read -r -a mp <<< "$MODULEPATH" for t in "${trees[@]}"; do mods+=(""); done for M in "$@"; do @@ -423,7 +517,7 @@ function resolveModules() { if [[ "$M" != *..* ]]; then for t in "${mp[@]}"; do [[ -n "$t" && -e "$t/$M" ]] && { found=0; break; }; done for ((i = 1; i < ${#trees[@]} && ${#found} == 0; i++)); do - [[ -e "${trees[i]}/$M" ]] && found=$i + [[ -e "${trees[i]}/$M" || ( -n "${arcs[i]}" && -e "${arcs[i]}/$M" ) ]] && found=$i done fi [[ -z "$found" ]] && $MODULECMD bash -t avail 2>&1 | grep -qFx "$M" && found=0 @@ -442,14 +536,16 @@ function resolveLoaded() { # _LMFILES_ (//); any other module, e.g. one no tree has # any more, goes with the local MODULEPATH, as before. Sets GROUP_TREE and # GROUP_MODS as resolveModules does. - local M f t i found lm trees=("" "${CVMFS_TREES[@]}") mods=() + local M f t a i found lm trees=("" "${CVMFS_TREES[@]}") arcs=("" "${CVMFS_ARCHIVES[@]}") mods=() IFS=: read -r -a lm <<< "${_LMFILES_:-}" for t in "${trees[@]}"; do mods+=(""); done for M in "$@"; do found=0 for ((i = 1; i < ${#trees[@]} && found == 0; i++)); do for f in "${lm[@]}"; do - [[ "$f" == "${trees[i]}/$M" || "$f" == "${trees[i]}/$M/"* ]] && { found=$i; break; } + for t in "${trees[i]}" ${arcs[i]:+"${arcs[i]}"}; do + [[ "$f" == "$t/$M" || "$f" == "$t/$M/"* ]] && { found=$i; break 2; } + done done done mods[found]+="${mods[found]:+ }$M" @@ -461,6 +557,37 @@ function resolveLoaded() { return 0 } +function releaseView() { + # releaseView [auto]: the view of the requested release, when its setup.sh + # can stand in for loading the modules (no modulecmd, no Python): one module + # / from a CVMFS tree whose modules directory points to the + # view (..view, a link to the view's marker for that module, see + # bits cvmfs publish), and a view to use: --view or, with auto, a view: true + # package (its modulefile sets BITS_VIEW). Prints the view directory, or + # nothing. + local t m p v link d + [[ $DEVOPT != 1 && ${#GROUP_TREE[@]} == 1 && -n "${GROUP_TREE[0]}" ]] || return 0 + m=${GROUP_MODS[0]} + [[ $m == */* && $m != *" "* ]] || return 0 + t=${GROUP_TREE[0]}; p=${m%%/*}; v=${m#*/} + link="$t/$p/.$v.view" + [[ -f "$link" && -f "$t/$m" ]] || return 0 + [[ $VIEWOPT == 1 ]] || { [[ ${1:-} == auto ]] && + grep -q '^setenv BITS_VIEW 1$' "$t/$m" 2> /dev/null; } || return 0 + d=$(readlink "$link") || return 0 + [[ $d == /* ]] || d="$t/$p/$d" + d=$(cd "$(dirname "$d")" 2> /dev/null && pwd -P) || return 0 + [[ -f "$d/setup.sh" && -f "$d/setup.csh" ]] && echo "$d" + return 0 +} + +function sourceReleaseView() { + # The release view's setup.sh in this shell; BITS_VIEW, which it sets to + # the view, is cleared as after loading the modules. + . "$1/setup.sh" + unset BITS_VIEW +} + function localOnly() { # --dev sources the packages' own init.sh from the work directory: local # modules only. @@ -479,11 +606,18 @@ function groupModulecmd() { # dependencies come from the same tree. BASEDIR is set to the tree's packages: # bits modulefiles find their package through it, and those of a second tree # skip their BASE/1.0 when the first tree's is loaded already. - local t="${GROUP_TREE[$3]}" - if [[ -z "$t" ]]; then + # A tree's archive (--archive) comes after it, with the same packages. + local t="${GROUP_TREE[$3]}" mp i + mp=$t + for ((i = 0; i < ${#CVMFS_TREES[@]}; i++)); do + [[ "${CVMFS_TREES[i]}" == "$t" && -n "${CVMFS_ARCHIVES[i]}" ]] && mp+=":${CVMFS_ARCHIVES[i]}" + done + if [[ -z "$t" && -n "$NO_WORK_DIR" ]]; then + $MODULECMD "$1" "$2" ${GROUP_MODS[$3]} # from the user's own MODULEPATH + elif [[ -z "$t" ]]; then BASEDIR="${WORK_DIR}/${ARCHITECTURE}" $MODULECMD "$1" "$2" ${GROUP_MODS[$3]} else - MODULEPATH="$t" BASEDIR="${t%/Modules/modulefiles}/Packages" $MODULECMD "$1" "$2" ${GROUP_MODS[$3]} + MODULEPATH="$mp" BASEDIR="${t%/Modules/modulefiles}/Packages" $MODULECMD "$1" "$2" ${GROUP_MODS[$3]} fi } @@ -496,7 +630,7 @@ function evalGroups() { for ((i = 0; i < ${#GROUP_TREE[@]}; i++)); do eval "$(groupModulecmd bash "$1" "$i" 2> >(grep -v "$IGNORE_ERR" >&2))" done - [[ -z "${GROUP_TREE[*]}" ]] || export BASEDIR="${WORK_DIR}/${ARCHITECTURE}" + [[ -z "${GROUP_TREE[*]}" || -n "$NO_WORK_DIR" ]] || export BASEDIR="${WORK_DIR}/${ARCHITECTURE}" } function printGroups() { @@ -572,6 +706,13 @@ if [[ "${1:-}" == "use" ]]; then BITS_SELF="${BITSDIR}" exec "${BITS_PY:-python3}" ${BITS_PY:+-E -s} -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' "$@" fi +# `bits trace FILE`: its package, and from the console backend who built, +# published and approved it. No work dir: it reads CVMFS and the backend. +if [[ "${1:-}" == "trace" ]]; then + shift + BITS_SELF="${BITSDIR}" exec "${BITS_PY:-python3}" ${BITS_PY:+-E -s} -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.trace import main; sys.exit(main())' "$@" +fi + # `bits cvmfs {platforms,show,summary,stage,publish} …` → the CVMFS group. # platforms/show/summary read a deployed tree (no work dir); stage/publish are # the producer-side ops, delegated inside cvmfs_inspect.main to their @@ -585,7 +726,8 @@ fi for arg in "$@" do case $arg in - architecture|brew|build|certify|checksums|clean|cleanup|prune|compliance|cvmfs-path|deps|doctor|import|init|publish|sbom|sign|stats|status|verify|version|-debug|-d) + architecture|brew|build|certify|checksums|clean|cleanup|prune|compliance|cvmfs-path|deps|doctor|import|init|publish|sbom|sign|stats|status|upgrade|verify|version|--debug|-d) + # (the same list in the unknown-command check below) # bitsBuild creates the work directory when a command runs (not for # --help, version or architecture). ${BITS_PY:+"$BITS_PY" -E -s} "$BITSDIR/bitsBuild" "$@" @@ -628,6 +770,8 @@ COMMAND_IN_ENV=() CLEAN_ENV=1 DEVOPT=0 VIEWOPT=0 +ARCHIVE= +_HELP= VERBOSE=1 ARGS=() @@ -649,8 +793,10 @@ do DEVOPT=1;; --view) VIEWOPT=1;; - -h|--help|help) - printHelp; exit 0 ;; + --archive|-archive) + ARCHIVE=1;; + -h|--help) + _HELP=1;; -c) i=$((i+1)); COMMAND_IN_ENV=("${ARGV[@]:${i}}"); break ;; @@ -663,6 +809,17 @@ ACTION="${ARGS[0]}" ARGS=("${ARGS[@]:1}") +case "$ACTION" in + q|query|enter|setenv|load|printenv|unload|list|avail|modulecmd|shell-helper) + [[ -z "$_HELP" ]] || { printModuleHelp; exit 0; } ;; + help) printHelp; exit 0 ;; + "") [[ -z "$_HELP" ]] || { printHelp; exit 0; } + usageError "no command given"; exit 1 ;; + architecture|brew|build|certify|checksums|clean|cleanup|prune|compliance|cvmfs-path|deps|doctor|import|init|publish|sbom|sign|stats|status|upgrade|verify|version|store|use|cvmfs|cvmfs-stage|cvmfs-publish|store-stats|preload|overlay|lcg-view) + usageError "$ACTION takes its options after it: bits $ACTION [OPTIONS]"; exit 1 ;; + *) usageError "unknown command: $ACTION"; exit 1 ;; +esac + export BITSLVL=$((BITSLVL+1)) if [[ "$ACTION" == "shell-helper" ]]; then @@ -677,6 +834,7 @@ bits() { echo "bits not found" return 1 fi + case " $* " in *" -h "*|*" --help "*) "$BITS_ENV__" "$@"; return $? ;; esac for A in "$@"; do if ! test `echo "$A" | cut -b1` = -; then if test "$A" = load || test "$A" = unload; then @@ -701,42 +859,93 @@ if [[ -z "$WORK_DIR" ]]; then for WORK_DIR in "${DEFAULT_WORK_DIRS[@]}"; do [[ -d "$WORK_DIR" ]] && break || WORK_DIR= done - [[ -z "$WORK_DIR" ]] && { printHelp "No default work dir can be accessed:" \ - "export \$BITS_WORK_DIR or run bits from a directory" \ - "containing the \"sw\" dir"; false; } fi -[[ ! -d "$WORK_DIR" ]] && { printHelp "Work dir $WORK_DIR cannot be accessed"; false; } -WORK_DIR=$(cd "$WORK_DIR"; pwd) +# The CVMFS module trees of a community: with $BITS_CVMFS_PREFIX (which the +# CVMFS entry point sets; empty turns them off), //Modules/modulefiles; +# else, with a cvmfs.yaml in the recipe repository (here, or beside the work +# dir), the directory of its cvmfs_modules_template. For each deployed arch that +# is $ARCHITECTURE or that without its -opt/-dbg (own_hash packages, the +# toolchain); an $ARCHITECTURE without a build type is -opt. Never the other +# build type: bits q notes when it is published. +# With cvmfs.yaml, no -a and no tree for $ARCHITECTURE, the arch is the one +# published for this host's CPU and OS, if there is exactly one (a compiler); +# else a note lists those published. They +# come after the local MODULEPATH and stay off it: each tree is used on its own +# (groupModulecmd), as its BASE/1.0 sets BASEDIR for it alone. +CVMFS_TREES=(); CVMFS_ARCHIVES=(); CVMFS_SLOT=; CVMFS_ARCH= +_tree=; _layout= +if [[ -n "${BITS_CVMFS_PREFIX+set}" ]]; then + [[ -n "$BITS_CVMFS_PREFIX" ]] && _tree="${BITS_CVMFS_PREFIX%/}/{arch}/Modules/modulefiles" +else + for _layout in "$PWD/cvmfs.yaml" ${WORK_DIR:+"$(dirname "$WORK_DIR")/cvmfs.yaml"} ""; do + [[ -f "$_layout" ]] && break + done + if [[ -n "$_layout" ]]; then + _p=$(layoutValue "$_layout" prefix) + _tree=$(layoutValue "$_layout" cvmfs_modules_template) + _tree=${_tree:-"{prefix}/{platform}/Modules/modulefiles/{pkg}"} + _tree=${_tree//"{prefix}"/"${_p%/}"}; _tree=${_tree//"{platform}"/"{arch}"} + _d=${_tree%"/{pkg}"} + # The install dir in place of the arch (alice.bits: el9-x86_64, as on + # /cvmfs/alice.cern.ch): the one published with the arch's CPU and OS. + if [[ "$_d" != *"{arch}"* && "$_d" == *"{install_dir}"* ]]; then + _d=${_d//"{install_dir}"/"{arch}"}; CVMFS_SLOT=install_dir + fi + # Usable: a prefix, and /{pkg}, absolute with {arch} once, its only slot. + _r=${_d//"{arch}"/} + if [[ -z "$_p" || "$_tree" != *"/{pkg}" || "$_d" != /* || "$_r" == "$_d" || "$_r" == *"{"* || + "${_d#*"{arch}"}" == *"{arch}"* ]]; then + _tree=; CVMFS_SLOT= # no module trees for q/enter/load + else + _tree=$_d + fi + fi +fi +# The environment commands work on a community's CVMFS modules alone (e.g. in a +# grid job): then no work dir is needed. +NO_WORK_DIR= +if [[ -z "$WORK_DIR" ]]; then + if [[ -n "$_tree" ]] && { isModuleAction || [[ $ACTION == unload ]]; }; then + NO_WORK_DIR=1 + [[ $DEVOPT != 1 ]] || { printf "${ER}ERROR: --dev needs a work dir${EZ}\n" >&2; exit 1; } + else + usageError "no work dir: export \$BITS_WORK_DIR, or run bits where the \"sw\" dir is"; false + fi +else + [[ ! -d "$WORK_DIR" ]] && { usageError "work dir $WORK_DIR cannot be accessed"; false; } + WORK_DIR=$(cd "$WORK_DIR"; pwd) +fi # Remember whether the architecture was given explicitly (-a); if so, respect it # verbatim and never second-guess it below. _bt_mark startup # script start → here (configBits, option/work-dir parsing) _ARCH_GIVEN=; [[ -n "$ARCHITECTURE" ]] && _ARCH_GIVEN=1 -# Architecture autodetection (when -a not given). MODULES-first: for q/load/enter -# the installed module trees are the source of truth, so choose from them -# directly and skip the bitsBuild (python) host-arch spawn entirely in the common -# single-arch case. A real arch has a $WORK_DIR/MODULES/ dir; sw/ also holds -# non-arch dirs (wrapper-scripts) which are excluded by going via MODULES. Fall -# back to python only to disambiguate several installed arches (prefer the host's) -# or when nothing is installed yet (e.g. a first build). +# Architecture autodetection (when -a not given), without Python where it can: +# the installed architectures (their MODULES/, else in the work +# dir); of several, the one for this host's CPU and OS (hostMatches); with none, +# the one the community's CVMFS module trees publish for this host. Python's +# host architecture only when that does not settle it. if [[ -z "$_ARCH_GIVEN" ]]; then # A real arch dir carries a CPU token (x86-64/aarch64/arm64/ppc64...). The # reuse-from-modules overlays live under MODULES//, so MODULES/ # can also hold dirs with no CPU token — exclude those so an overlay # is never mistaken for an architecture. _present=() - for _a in $(ls -1t "$WORK_DIR/MODULES" 2> /dev/null); do - [[ -d "$WORK_DIR/MODULES/$_a" ]] || continue - [[ "$_a" =~ (x86[-_]64|aarch64|arm64|ppc64le|ppc64) ]] || continue - _present+=("$_a") + [[ -n "$NO_WORK_DIR" ]] || for _d in "$WORK_DIR/MODULES" "$WORK_DIR"; do + for _a in $(ls -1t "$_d" 2> /dev/null); do + [[ -d "$_d/$_a" && "$_a" =~ (x86[-_]64|aarch64|arm64|ppc64le|ppc64|riscv64) ]] && _present+=("$_a") + done + (( ${#_present[@]} )) && break done if [[ ${#_present[@]} -eq 1 ]]; then ARCHITECTURE="${_present[0]}" # sole installed arch — no python spawn elif [[ ${#_present[@]} -gt 1 ]]; then - # Several present: prefer the host's native arch if installed. - _host="$( { [[ -z "$BITS_PY" ]] && "bitsBuild" architecture 2> /dev/null; } || - ${BITS_PY:+"$BITS_PY" -E -s} "$BITSDIR/bitsBuild" architecture 2> /dev/null || true)" + # Several present: the host's, if exactly one is; else Python's host arch. ARCHITECTURE= - if [[ -n "$_host" ]]; then + _host=$(printf '%s\n' "${_present[@]}" | hostMatches) + [[ -n "$_host" && "$_host" != *$'\n'* ]] && ARCHITECTURE=$_host + if [[ -z "$ARCHITECTURE" ]]; then + _host="$( { [[ -z "$BITS_PY" ]] && "bitsBuild" architecture 2> /dev/null; } || + ${BITS_PY:+"$BITS_PY" -E -s} "$BITSDIR/bitsBuild" architecture 2> /dev/null || true)" for _a in "${_present[@]}"; do [[ "$_a" == "$_host" ]] && { ARCHITECTURE="$_a"; break; }; done fi # No host match (or the host-arch probe momentarily failed → empty $_host): @@ -757,6 +966,17 @@ if [[ -z "$_ARCH_GIVEN" ]]; then printf "${EY}NOTE: several architectures present; selected most recent: %s${EZ}\n" "$ARCHITECTURE" >&2 printf "${EY} (override with bits -a %s)${EZ}\n" "${ACTION:-q}" >&2 fi + elif [[ -n "$_tree" && ( -z "$CVMFS_SLOT" || -n "$NO_WORK_DIR" ) ]] && + _a=$(hostCvmfsArch "$_tree") && [[ -n "$_a" ]]; then + # Nothing installed: the community's, for this host (an install dir is no + # arch to build for: then only without a work dir). + ARCHITECTURE=$_a; [[ -n "$CVMFS_SLOT" ]] || ARCHITECTURE+=-opt + elif [[ -n "$NO_WORK_DIR" ]]; then + _p=$(publishedArchs "$_tree" | tr '\n' ' ' | sed 's/ $//') + _s=${CVMFS_SLOT:-arch}; _d=${_tree//"{arch}"/"{$_s}"} + printf "${ER}ERROR: no single %s published for this host in %s; published: %s. Choose one: bits -a <%s> %s${EZ}\n" \ + "$_s" "$_d" "${_p:-none}" "$_s" "$ACTION" >&2 + exit 1 else # Nothing installed yet → host arch from python (e.g. for a first build). ARCHITECTURE="$([[ -z "$BITS_PY" ]] && "bitsBuild" architecture 2> /dev/null || true)" @@ -764,7 +984,7 @@ if [[ -z "$_ARCH_GIVEN" ]]; then fi fi _bt_mark archdetect # arch autodetect (python spawn only when needed) -[[ -z "$ARCHITECTURE" ]] && { printHelp "Cannot autodetect architecture"; false; } +[[ -z "$ARCHITECTURE" ]] && { usageError "cannot detect the architecture: give it with -a ARCH"; false; } # Look for modulecmd (v3) or modulecmd-compat (>= v4): bits' own runtime first MODULECMD="$BITSDIR/runtime/bin/modulecmd" @@ -773,7 +993,7 @@ MODULECMD="$BITSDIR/runtime/bin/modulecmd" [[ -x "$MODULECMD" ]] || MODULECMD="$(brew --prefix modules 2> /dev/null || true)/libexec/modulecmd-compat" [[ -x "$MODULECMD" ]] || { installHint; false; } -if [[ -d "${WORK_DIR}/MODULES/${ARCHITECTURE}" ]]; then +if [[ -z "$NO_WORK_DIR" && -d "${WORK_DIR}/MODULES/${ARCHITECTURE}" ]]; then touch "${WORK_DIR}/MODULES/${ARCHITECTURE}/.testwrite" 2> /dev/null || NO_REFRESH=1 rm -f "${WORK_DIR}/MODULES/${ARCHITECTURE}/.testwrite" 2> /dev/null || true fi @@ -783,48 +1003,16 @@ fi # package's `prereq ` cannot resolve at enter/load time without it. # Append every such overlay for this arch. Flat tree stays first (a local build # wins over a reused copy); no overlays -> MODULEPATH is unchanged. -_bits_overlays="" -for _ov in "$WORK_DIR"/MODULES/*/"$ARCHITECTURE"; do - [[ -d "$_ov" ]] && _bits_overlays="${_bits_overlays:+$_bits_overlays:}$_ov" -done -export MODULEPATH="$WORK_DIR/MODULES/$ARCHITECTURE${_bits_overlays:+":$_bits_overlays"}${MODULEPATH:+":$MODULEPATH"}" -unset _bits_overlays _ov -MODULEPATH=$(echo "$MODULEPATH" | sed -e 's/::*/:/g; s/^://; s/:$//') - -# The CVMFS module trees of a community: with $BITS_CVMFS_PREFIX (which the -# CVMFS entry point sets; empty turns them off), //Modules/modulefiles; -# else, with a cvmfs.yaml in the recipe repository (here, or beside the work -# dir), the directory of its cvmfs_modules_template. For each deployed arch that -# is $ARCHITECTURE but for a trailing -opt/-dbg, in this order: the exact one, -# the one without build type (own_hash packages, the toolchain), -opt, -dbg. -# With cvmfs.yaml, no -a and no tree for $ARCHITECTURE, the arch is the one -# published for this host's CPU and OS, if there is exactly one (a compiler); -# else a note lists those published. They -# come after the local MODULEPATH and stay off it: each tree is used on its own -# (groupModulecmd), as its BASE/1.0 sets BASEDIR for it alone. -CVMFS_TREES=() -_tree=; _layout= -if [[ -n "${BITS_CVMFS_PREFIX+set}" ]]; then - [[ -n "$BITS_CVMFS_PREFIX" ]] && _tree="${BITS_CVMFS_PREFIX%/}/{arch}/Modules/modulefiles" -else - for _layout in "$PWD/cvmfs.yaml" "$(dirname "$WORK_DIR")/cvmfs.yaml" ""; do - [[ -f "$_layout" ]] && break +if [[ -z "$NO_WORK_DIR" ]]; then + _bits_overlays="" + for _ov in "$WORK_DIR"/MODULES/*/"$ARCHITECTURE"; do + [[ -d "$_ov" ]] && _bits_overlays="${_bits_overlays:+$_bits_overlays:}$_ov" done - if [[ -n "$_layout" ]]; then - _p=$(layoutValue "$_layout" prefix) - _tree=$(layoutValue "$_layout" cvmfs_modules_template) - _tree=${_tree:-"{prefix}/{platform}/Modules/modulefiles/{pkg}"} - _tree=${_tree//"{prefix}"/"${_p%/}"}; _tree=${_tree//"{platform}"/"{arch}"} - # Usable: a prefix, and /{pkg} with {arch} once in , its only slot. - _d=${_tree%"/{pkg}"}; _r=${_d//"{arch}"/} - if [[ -z "$_p" || "$_d" == "$_tree" || "$_r" == "$_d" || "$_r" == *"{"* || - "${_d#*"{arch}"}" == *"{arch}"* ]]; then - _tree= # e.g. alice.bits' {install_dir}: no module trees for q/enter/load - else - _tree=$_d - fi - fi + export MODULEPATH="$WORK_DIR/MODULES/$ARCHITECTURE${_bits_overlays:+":$_bits_overlays"}${MODULEPATH:+":$MODULEPATH"}" + unset _bits_overlays _ov fi +MODULEPATH=$(echo "$MODULEPATH" | sed -e 's/::*/:/g; s/^://; s/:$//') + if [[ -n "$_tree" ]]; then cvmfsTrees "$_tree" "$ARCHITECTURE" # cvmfs.yaml, no -a, and nothing for the local arch: the one for this host. @@ -832,14 +1020,32 @@ if [[ -n "$_tree" ]]; then _a=$(hostCvmfsArch "$_tree") [[ -z "$_a" ]] || cvmfsTrees "$_tree" "$_a-opt" fi - if [[ -n "$_layout" && ${#CVMFS_TREES[@]} == 0 ]] && isModuleAction; then + if [[ -n "$_layout$NO_WORK_DIR" && ${#CVMFS_TREES[@]} == 0 ]] && isModuleAction; then _p=$(publishedArchs "$_tree" | tr '\n' ' ' | sed 's/ $//') _a=$ARCHITECTURE; [[ -n "$_ARCH_GIVEN" ]] || _a="this host" - [[ -z "$_p" ]] || printf "${EY}NOTE: no CVMFS module tree %s (%s) for %s; published: %s. Choose one: bits -a %s${EZ}\n" \ - "$_tree" "$_layout" "$_a" "$_p" "$ACTION" >&2 + _s=${CVMFS_SLOT:-arch}; _d=${_tree//"{arch}"/"{$_s}"} + [[ -z "$_p" ]] || printf "${EY}NOTE: no CVMFS module tree %s%s for %s; published: %s. Choose one: bits -a <%s> %s${EZ}\n" \ + "$_d" "${_layout:+ ($_layout)}" "$_a" "$_p" "$_s" "$ACTION" >&2 fi + # bits q: the other build type of this arch, if published, is only noted. + if [[ -z "$CVMFS_SLOT" && ( "$ACTION" == q || "$ACTION" == query ) && -n "$CVMFS_ARCH" ]]; then + _b=${CVMFS_ARCH%-opt}; _b=${_b%-dbg}; _o=$_b-dbg; [[ "$CVMFS_ARCH" != *-dbg ]] || _o=$_b-opt + _od=${_tree//"{arch}"/"$_o"} # not inside "...": bash 3.2 keeps the quotes + [[ ! -d "$_od" ]] || + printf "${EY}NOTE: also published: %s (bits -a %s %s)${EZ}\n" "$_o" "$_o" "$ACTION" >&2 + fi + # Retired modulefiles, as alienv --archive finds them: Modules/archive beside + # Modules/modulefiles. With --archive (and for unload, which finds a module + # where it was loaded from). + for _d in "${CVMFS_TREES[@]}"; do + _a= + if [[ -n "$ARCHIVE" || $ACTION == unload ]] && [[ "$_d" == */Modules/modulefiles && -d "${_d%/modulefiles}/archive" ]]; then + _a=${_d%/modulefiles}/archive + fi + CVMFS_ARCHIVES+=("$_a") + done fi -unset _a _d _r _p _tree _layout +unset _a _b _d _o _od _r _p _s _tree _layout IGNORE_ERR="Unable to locate a modulefile for 'Toolchain/" # -q (which unsets VERBOSE) silences environment-modules' "Loading " / # "Loading requirement:" progress lines so setenv/printenv/load/enter output can @@ -862,6 +1068,10 @@ _bitsView() { } applyView() { [[ $VIEWOPT == 1 ]] || return 0 + if [[ -n "$NO_WORK_DIR" ]]; then + printf "${EY}NOTE: no work dir: the path variables are not collapsed onto a view${EZ}\n" >&2 + return 0 + fi eval "$(_bitsView --work-dir "${WORK_DIR}" --arch "${ARCHITECTURE}" 2> /dev/null)" } @@ -876,7 +1086,11 @@ case "$ACTION" in resolveModules $MODULES _bt_mark resolveModules PS1DEV= - if [[ $DEVOPT == 1 ]];then + RELEASE_VIEW=$(releaseView auto) + if [[ -n "$RELEASE_VIEW" ]]; then + sourceReleaseView "$RELEASE_VIEW" + VIEWOPT=0 # the view's own setup is the view + elif [[ $DEVOPT == 1 ]];then localOnly PS1DEV=" (dev)" for MODULE in $MODULES; do @@ -910,11 +1124,15 @@ case "$ACTION" in zsh) export PS1="[$MODULES]$PS1DEV"' %~ %#> ' ;; esac fi - $MODULECMD bash list - if [[ $DEVOPT == 1 ]];then - echo "*** Development env loaded for '$MODULES' ***" >&2 + if [[ -n "$RELEASE_VIEW" ]]; then + printf "${ET}Set up from the release view %s (no modules loaded). " "$RELEASE_VIEW" >&2 else - printf "${ET}Use ${EM}bits list${ET} to list loaded modules. " >&2 + $MODULECMD bash list + if [[ $DEVOPT == 1 ]];then + echo "*** Development env loaded for '$MODULES' ***" >&2 + else + printf "${ET}Use ${EM}bits list${ET} to list loaded modules. " >&2 + fi fi printf "Use ${EM}exit${ET} to exit this environment.${EZ}\n" >&2 exec ${CLEAN_ENV:+env $SHELL_NORC_ENV} $MODULES_SHELL ${CLEAN_ENV:+$SHELL_NORC_PARAM} -i @@ -923,13 +1141,15 @@ case "$ACTION" in [[ $ACTION == printenv ]] && ACTION=load MODULES=$(normModules "${ARGS[@]}") _bt_mark preamble + RELEASE_VIEW= if [[ $ACTION == load ]]; then collectModules; _bt_mark collectModules resolveModules $MODULES; _bt_mark resolveModules + RELEASE_VIEW=$(releaseView) # --view only, as the modules path else resolveLoaded $MODULES fi - [[ $VERBOSE == 1 ]] && printf "${ET}Use ${EM}bits list${ET} to list loaded modules.${EZ}\n" >&2 + [[ $VERBOSE == 1 && -z "$RELEASE_VIEW" ]] && printf "${ET}Use ${EM}bits list${ET} to list loaded modules.${EZ}\n" >&2 if [[ $DEVOPT == 1 ]];then localOnly echo "Warning: Development mode for '$ACTION' is not supported via bits. Please remove '--dev' or run manually shown below" >&2 @@ -943,12 +1163,19 @@ case "$ACTION" in exit 0 fi detectShell + if [[ -n "$RELEASE_VIEW" ]]; then + case "$MODULES_SHELL" in + csh|tcsh) printf 'source "%s/setup.csh";\nunsetenv BITS_VIEW;\n' "$RELEASE_VIEW" ;; + *) printf '. "%s/setup.sh";\nunset BITS_VIEW;\n' "$RELEASE_VIEW" ;; + esac + exit 0 + fi printGroups "$ACTION" # --view: emit the view-collapsed path overrides AFTER the module setup so they # win on eval. printenv/load only print (they don't load in-process), so the # closure is loaded in a subshell to compute the collapse; output is rendered # in the caller's shell syntax (sh export vs csh setenv). - if [[ $VIEWOPT == 1 && $ACTION == load ]]; then + if [[ $VIEWOPT == 1 && $ACTION == load && -z "$NO_WORK_DIR" ]]; then _vshell=sh; case "$MODULES_SHELL" in csh|tcsh) _vshell=csh;; esac ( evalGroups load 2> /dev/null _bitsView --work-dir "${WORK_DIR}" --arch "${ARCHITECTURE}" --shell "$_vshell" 2> /dev/null ) @@ -959,10 +1186,15 @@ case "$ACTION" in ;; setenv) [[ $DEVOPT == 1 ]] && echo "Warning: Development mode for '$ACTION' is not supported. Please remove '--dev'" >&2 - [[ -z "${COMMAND_IN_ENV[*]}" ]] && { printHelp "No command specified with -c"; false; } + [[ -z "${COMMAND_IN_ENV[*]}" ]] && { usageError "setenv needs a command after -c"; false; } MODULES=$(normModules "${ARGS[@]}") collectModules resolveModules $MODULES + RELEASE_VIEW=$(releaseView auto) + if [[ -n "$RELEASE_VIEW" ]]; then + sourceReleaseView "$RELEASE_VIEW" + exec "${COMMAND_IN_ENV[@]}" + fi evalGroups add [[ $UNAME == Darwin ]] && eval $(stripDyld) [[ -n "${BITS_VIEW:-}" ]] && VIEWOPT=1 # view: true package auto-enables --view @@ -995,12 +1227,6 @@ case "$ACTION" in collectModules exec "$MODULECMD" "${ARGS[@]}" ;; - '') - printHelp "What do you want to do?" - ;; - *) - printHelp "Unknown command: $1" - ;; esac false diff --git a/bitsBuild b/bitsBuild index 80d8c3e0..53dc3687 100755 --- a/bitsBuild +++ b/bitsBuild @@ -12,28 +12,53 @@ wrappers that exec this script with the matching sub-command prepended. import atexit import logging import os +import shlex import sys import traceback from os.path import exists, expanduser + +def _missing_module(exc): + """Exit with how to install bits' Python requirements: a module missing here + stops every sub-command, doctor too, before it can say so itself.""" + mod = (exc.name or "").split(".")[0] + req = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt") + try: + with open(req) as f: + wanted = f.read().split() + except OSError: + wanted = [] + if {"yaml": "pyyaml"}.get(mod, mod) not in wanted: # not one of ours: as before + raise exc + py = shlex.quote(sys.executable or "python3") + user = "" if sys.prefix != sys.base_prefix or sys.flags.no_user_site else " --user" + sys.exit("bits: Python module '%s' is missing for %s. As the user that runs bits, " + "install its requirements with:\n %s -m pip install%s -r %s\n" + "(no pip: install python3-pip with the system's package manager), then run: bits doctor" + % (mod, py, py, user, shlex.quote(req))) + + # Internal -from bits_helpers import __version__, _VERSION_INFO -from bits_helpers.version import version_line -from bits_helpers.args import doParseArgs -from bits_helpers.build import doBuild -from bits_helpers.clean import doClean -from bits_helpers.cleanup import doCleanup -from bits_helpers.deps import doDeps -from bits_helpers.brew import doBrew -from bits_helpers.cvmfs_import_cmd import doImport -from bits_helpers.doctor import doDoctor -from bits_helpers.init import doInit -from bits_helpers.publish import doPublish -from bits_helpers.verify import doVerify -from bits_helpers.status import doStatus -from bits_helpers.stats import doStats -from bits_helpers.log import debug, error, info, logger, warning -from bits_helpers.arch import detectArch +try: + from bits_helpers import __version__, _VERSION_INFO + from bits_helpers.version import version_line + from bits_helpers.args import doParseArgs + from bits_helpers.build import doBuild + from bits_helpers.clean import doClean + from bits_helpers.cleanup import doCleanup + from bits_helpers.deps import doDeps + from bits_helpers.brew import doBrew + from bits_helpers.cvmfs_import_cmd import doImport + from bits_helpers.doctor import doDoctor + from bits_helpers.init import doInit + from bits_helpers.publish import doPublish + from bits_helpers.verify import doVerify + from bits_helpers.status import doStatus + from bits_helpers.stats import doStats + from bits_helpers.log import debug, error, info, logger, warning + from bits_helpers.arch import detectArch +except ModuleNotFoundError as _exc: + _missing_module(_exc) def doMain(args, parser): @@ -142,6 +167,10 @@ def doMain(args, parser): from bits_helpers.checksums_cmd import doChecksums sys.exit(doChecksums(args, parser)) + if args.action == "upgrade": + from bits_helpers.upgrade_cmd import doUpgrade + doUpgrade(args, parser) + if args.action == "sbom": from bits_helpers.sbom import doSbom sys.exit(doSbom(args, parser)) diff --git a/bitsStore b/bitsStore index 4c41a93b..edc4217a 100755 --- a/bitsStore +++ b/bitsStore @@ -28,6 +28,12 @@ # instead of MANIFESTS/ — that is the copy certify reads, # so rm there (then commit + MR) is what fixes certify; # rm without it only tidies the store's MANIFESTS/ copies. +# In a store bits-orchestrator certifies (MANIFESTS/ +# orchestrator.json), rm --manifests-dir requests a +# decertify action for each stale BOM instead (--console, +# a GitLab token), and rm never deletes the orchestrator's +# signed manifests (nor its marker, unless named exactly +# with --raw and --force). # --deep hashes objects that carry no recorded sha256. # # NOTE (hash-only store): new builds write ONLY the content-addressed @@ -37,13 +43,11 @@ # selectors now only match LEGACY pointer objects left by older builds and # are retained for one-time migration cleanup (`bits store rm --links …`). # -# Credentials: ~/.bits/s3keys (override $BITS_AWS_KEYS_FILE). -# Store: CERN S3 by default (override $BITS_S3_STORE). +# Credentials: $BITS_AWS_KEYS_FILE, else the site's paths.s3_keys_file. +# Store: $BITS_S3_STORE, else the site's store.operations. set -euo pipefail PROG=$(basename "$0") -STORE="${BITS_S3_STORE:-https://s3.cern.ch/lcgapp-bits-testing}" -S3KEYS="${BITS_AWS_KEYS_FILE:-$HOME/.bits/s3keys}" # bitsStore lives next to the bits_helpers package (bits/), so it imports the # helpers from its own directory — no dependency on `bits` being on PATH. @@ -53,18 +57,18 @@ SELFDIR="$(cd "$(dirname "$0")" && pwd)" BITS_PY=(python3) [[ -x "$SELFDIR/runtime/bin/python3" ]] && BITS_PY=("$SELFDIR/runtime/bin/python3" -E -s) -# Load creds if present; a missing file is not fatal here (--help needs no S3). -# The Python side checks for credentials before it actually touches S3. -if [ -f "$S3KEYS" ]; then - set +u # a creds file may reference vars we don't set - # shellcheck disable=SC1090 - . "$S3KEYS" - set -u - export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY - [ -n "${AWS_SESSION_TOKEN:-}" ] && export AWS_SESSION_TOKEN || true +S3KEYS="${BITS_AWS_KEYS_FILE:-}" +if [[ -z "$S3KEYS" ]]; then + S3KEYS=$(BITSSTORE_SELF="$SELFDIR" "${BITS_PY[@]}" -c 'import os, sys; sys.path.insert(0, os.environ["BITSSTORE_SELF"]); from bits_helpers import site; site.main(["paths.s3_keys_file"])') + S3KEYS="${S3KEYS:-~/.bits/s3keys}" + S3KEYS="${S3KEYS/#\~/$HOME}" fi -BITSSTORE_STORE="$STORE" BITSSTORE_SELF="$SELFDIR" \ +# The Python side reads the creds file as bits build does ([bucket NAME] sections +# included); a missing file is not fatal (--help needs no S3). +export BITS_AWS_KEYS_FILE="$S3KEYS" + +BITSSTORE_STORE="${BITS_S3_STORE:-}" BITSSTORE_SELF="$SELFDIR" \ PYTHONPATH="${SELFDIR}${PYTHONPATH:+:$PYTHONPATH}" \ exec "${BITS_PY[@]}" - "$@" <<'PY' import argparse, datetime, fnmatch, hashlib, json, os, sys, tempfile @@ -72,20 +76,26 @@ if sys.flags.ignore_environment: # bits' own runtime: PYTHONPATH is not read sys.path.insert(0, os.environ["BITSSTORE_SELF"]) from bits_helpers.publish import _normalize_s3_store -from bits_helpers.sync import remote_from_url +from bits_helpers.sync import remote_from_url, resolve_and_export_s3_config +resolve_and_export_s3_config() # the keys file and environment, as bits build reads them from bits_helpers.utilities import resolve_store_path, resolve_links_path -from bits_helpers import stale_boms +from bits_helpers import orchestrated, stale_boms try: from bits_helpers import trust except Exception: trust = None -STORE = os.environ["BITSSTORE_STORE"] -PROG = "bitsStore" +from bits_helpers import site +STORE = os.environ.get("BITSSTORE_STORE") or str(site.get("store.operations") or "") +if not STORE and not set(sys.argv[1:]) & {"-h", "--help"}: + sys.exit("bits store: no store: set $BITS_S3_STORE, or store.operations in a site " + "configuration file (see bits doctor)") +PROG = "bits store" NOW = datetime.datetime.now(datetime.timezone.utc) # Default bits work dir (S3 client + default MANIFESTS) for the gc/stats verbs; # mirrors bits_helpers.args without importing that heavy module on every call. -_WORKDIR = os.environ.get("BITS_WORK_DIR") or os.environ.get("ALICE_WORK_DIR") or "sw" +_WORKDIR = (os.environ.get("BITS_WORK_DIR") or os.environ.get("ALICE_WORK_DIR") + or site.get("paths.work_dir") or "sw") # ── S3 helpers ──────────────────────────────────────────────────────────────── @@ -538,7 +548,7 @@ def stale_bom_report(store, s3, bucket, f): "Set $BITS_S3_STORE, or pass --force (rm) to select them anyway.", file=sys.stderr) return {} - return {m["key"]: None for m, _c in full} + return {m["key"]: (m if f.manifests_dir else None) for m, _c in full} def delete_local_files(paths, f): @@ -574,6 +584,59 @@ def delete_local_files(paths, f): return 1 if failed else 0 +def decertify_local(boms, f, marker): + """In a store bits-orchestrator certifies, the BOM files of its bits-manifests + mirror (--manifests-dir) are not removed: each stale one is asked out of the + certified set by a decertify action, whose group is the file's directory + (manifests//). Same dry-run / confirmation rules as a delete.""" + from bits_helpers import forge + wanted, bad = stale_boms.decertify_targets(boms) + for path in bad: + print(f"{PROG}: {path} is not a BOM under manifests//; skipped", file=sys.stderr) + if not wanted: + print("Nothing matched; nothing to decertify.") + return 1 if bad else 0 + print(f"{orchestrated.describe(marker)} owns the certified set: the stale BOMs are " + "decertified, not removed") + for (group, bid, arch), paths in sorted(wanted.items()): + print(f" {group:<12} {bid[:40]:<40} {arch:<22} {len(paths)} BOM(s)") + if f.dry_run: + print(f"[dry-run] would request {len(wanted)} decertify action(s) (nothing requested)") + return 0 + url = orchestrated.console_url(f.console) + token = forge.resolve_gitlab_token(None) + if not (url and token): + sys.exit(f"{PROG}: decertifying needs the bits-console URL (--console, " + "$BITS_CONSOLE_URL) and a GitLab token (~/.bits/gitlab-token)") + if not f.yes: + try: + with open("/dev/tty") as tty: + sys.stderr.write(f"Ask {url} to decertify {len(wanted)} build(s)? [y/N] ") + sys.stderr.flush() + if tty.readline().strip().lower() not in ("y", "yes"): + print("Aborted."); return 1 + except OSError: + print("Refusing to decertify without confirmation (no tty). Use --yes.", + file=sys.stderr) + return 1 + console = orchestrated.Console(url, token, cafile=f.console_cafile, + insecure=f.console_insecure) + failed = len(bad) + for (group, bid, arch) in sorted(wanted): + try: + action = console.settle(console.decertify( + group, bid, arch, "stale BOM: the store no longer backs it"), timeout=30) + except orchestrated.RequestError as exc: + print(f"ERROR decertify {bid} ({arch}) in {group}: {exc}", file=sys.stderr) + failed += 1 + continue + ok, words = console.outcome(action) + print(f"decertify {bid} ({arch}) in {group}: action {action.get('id')} {words}", + file=sys.stdout if ok else sys.stderr) + failed += 0 if ok else 1 + return 1 if failed else 0 + + # ── presentation ───────────────────────────────────────────────────────────── def present_manifests(store, f): store.load() @@ -742,6 +805,20 @@ def do_delete(s3, bucket, keys, store, f): return 0 protected = set() if f.force else store.referenced_keys(verified_only=True) hits = sorted(keys) + marker = getattr(f, "marker", None) + if marker: + # The orchestrator's own manifests, always; its marker unless named exactly + # with --force, as going back to the manifests CI does: + # `rm --raw MANIFESTS/orchestrator.json --force`. + named = f.force and orchestrated.MARKER in (f.raw or []) + owned = [k for k in hits if k.startswith(marker["manifests"]) + or (k == orchestrated.MARKER and not named)] + if owned: + print(f"{PROG}: {len(owned)} object(s) are {orchestrated.describe(marker)}'s " + "signed manifests or its marker — not deleted:", file=sys.stderr) + for k in owned[:10]: + print(" ", k, file=sys.stderr) + hits = [k for k in hits if k not in owned] blocked = [k for k in hits if k in protected] deletable = [k for k in hits if k not in protected] if blocked: @@ -841,6 +918,14 @@ def main(): sp = sub.add_parser(verb, help="list" if verb == "ls" else "delete") add_selection(sp) if verb == "rm": + sp.add_argument("--console", metavar="URL", default=None, + help="with --stale-boms --manifests-dir in a store bits-orchestrator " + "certifies: the bits-console to request the decertify actions " + "from (default: $BITS_CONSOLE_URL)") + sp.add_argument("--console-cafile", dest="console_cafile", metavar="PEM", default=None, + help="CA bundle for the console when it is not in this host's trust store") + sp.add_argument("--console-insecure", dest="console_insecure", action="store_true", + help="allow http:// / skip TLS verification to the console (testbed only)") sp.add_argument("-n", "--dry-run", action="store_true", help="show what would be deleted") sp.add_argument("--yes", action="store_true", help="skip the confirmation prompt") sp.add_argument("--force", action="store_true", @@ -860,9 +945,11 @@ def main(): # S3 client, so the dest names below must match what they read. gp = sub.add_parser("gc", help="reachability GC of the shared store") gp.add_argument("--trust-manifest", dest="trustManifest", required=True, metavar="PATH", - help="signed common manifest whose hashes are the GC roots (must verify)") + help="signed common manifest whose hashes are the GC roots (must verify); " + "with several stores one per store, comma-separated, in the same order") gp.add_argument("--remote-store", "--store", dest="gcStore", metavar="URL", default=STORE, - help="S3 store URL/bucket to sweep (default: %(default)s)") + help="S3 store URL/bucket to sweep; several, comma-separated, each in " + "turn (default: %(default)s)") gp.add_argument("-a", "--architecture", dest="architecture", metavar="ARCH", default=None, help="architecture store tree to sweep (default: detected)") gp.add_argument("-w", "--work-dir", dest="workDir", metavar="WORKDIR", default=_WORKDIR, @@ -914,9 +1001,17 @@ def main(): # do NOT use the ls/rm selection machinery — handle them before the # selection-attr normalisation below (which would csv-mangle package/version). if args.verb in ("gc", "stats", "upload"): - if not os.environ.get("AWS_ACCESS_KEY_ID"): - sys.exit(f"{PROG}: no S3 credentials — put them in ~/.bits/s3keys " - "(or set $BITS_AWS_KEYS_FILE / AWS_ACCESS_KEY_ID).") + from bits_helpers.publish import _normalize_s3_store + from bits_helpers.sync import resolve_and_export_s3_config, s3_settings, store_urls + resolve_and_export_s3_config() + stores = store_urls(getattr(args, {"gc": "gcStore", "stats": "storeStatsStore", + "upload": "uploadStore"}[args.verb], "")) + for url in stores: # each bucket's own credentials, else the run's + bucket = _normalize_s3_store(url).split("://", 1)[-1].split("/", 1)[0] + if not s3_settings(bucket)["access_key"]: + sys.exit(f"{PROG}: no S3 credentials for {url} — put them in ~/.bits/s3keys " + "(or set $BITS_AWS_KEYS_FILE / AWS_ACCESS_KEY_ID, or " + "BITS_S3__ACCESS_KEY_ID for that bucket).") if not getattr(args, "architecture", None): from bits_helpers.arch import detectArch args.architecture = detectArch() @@ -927,9 +1022,10 @@ def main(): if args.verb == "stats": from bits_helpers.store_stats import doStoreStats return doStoreStats(args, ap) or 0 - from bits_helpers.publish import _publish_s3 # upload - _publish_s3(args.package, args.version, args.architecture, args.workDir, - args.uploadStore, ap, dry_run=args.dryRun) + from bits_helpers.publish import _publish_s3 # upload, to every store given + for url in stores: + _publish_s3(args.package, args.version, args.architecture, args.workDir, + url, ap, dry_run=args.dryRun) return 0 # Normalise multi-value filters; give safe defaults for attrs absent on a verb. @@ -942,7 +1038,8 @@ def main(): ("bits_version", None), ("bits_dist", None), ("built_before", None), ("only_signed", False), ("only_boms", False), ("long", False), ("dry_run", False), - ("yes", False), ("force", False), ("deep", False)): + ("yes", False), ("force", False), ("deep", False), + ("console", None), ("console_cafile", None), ("console_insecure", False)): if not hasattr(args, name): setattr(args, name, dflt) args.before_dt = None @@ -999,6 +1096,15 @@ def main(): if args.verb == "cat": return do_cat(s3, bucket, args.key) + args.marker = None + if args.verb == "rm": + # Whether bits-orchestrator owns the certified set: rm then leaves its + # manifests alone and decertifies stale BOMs rather than removing them. + try: + args.marker = orchestrated.read(s3, bucket) + except orchestrated.Unknown as exc: + sys.exit(f"{PROG}: {exc}") + # Resolve the absolute --before cutoff now that S3 is reachable (it may name a # manifest whose upload time we look up). Fail CLOSED: an unresolved cutoff # must never fall through to "no age filter" and widen a delete. @@ -1035,8 +1141,14 @@ def main(): keys = stale_bom_report(store, s3, bucket, args) if args.verb != "rm": return 0 + if args.manifests_dir and args.marker: + return decertify_local(keys, args, args.marker) if args.manifests_dir: return delete_local_files(keys, args) + if args.marker and keys: + print(f"# the certified set ({orchestrated.describe(args.marker)}) still holds what " + "these copies certified: `--manifests-dir ` asks " + "for it to be decertified", file=sys.stderr) return do_delete(s3, bucket, keys, store, args) # Resolve the object set for the chosen view. diff --git a/bits_helpers/args.py b/bits_helpers/args.py index 29256adf..9a815b7f 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -8,6 +8,7 @@ normalise_arch_key, detectArchComponents, apply_arch_template) from bits_helpers.workarea import cleanup_git_log +from bits_helpers import site import multiprocessing import re @@ -19,13 +20,9 @@ from os.path import abspath, dirname, basename import sys -# Default workdir: fall back on "sw" if env is not set or empty -DEFAULT_WORK_DIR = os.environ.get("BITS_WORK_DIR") or os.environ.get("ALICE_WORK_DIR") or "sw" -# Default container registry for --docker image selection (the bits-containers -# minimal-OS toolchain images). Overridable per invocation via BITS_DOCKER_REGISTRY -# or per community via a `docker_registry:` field in defaults-release; bypassed -# entirely by BITS_LEGACY_REGISTRY=1 (legacy alisw builders). -DEFAULT_DOCKER_REGISTRY = "gitlab-registry.cern.ch/bits/containers" +# Default workdir: the environment, else the site configuration (paths.work_dir). +DEFAULT_WORK_DIR = (os.environ.get("BITS_WORK_DIR") or os.environ.get("ALICE_WORK_DIR") + or site.get("paths.work_dir") or "sw") def _add_s3_connection_opts(group): @@ -43,7 +40,7 @@ def _add_s3_connection_opts(group): help=("S3 endpoint URL for b3:// stores (e.g. " "https://s3.amazonaws.com or http://minio.local:9000). " "Overrides $S3_ENDPOINT_URL / $AWS_ENDPOINT_URL_S3. " - "Default: https://s3.cern.ch. Set this to use a non-CERN bucket.")) + "Default: the site configuration's store.s3_endpoint (CERN: https://s3.cern.ch).")) group.add_argument("--s3-access-key", dest="s3AccessKey", metavar="KEY", default=None, help=("S3 access key id. Overrides $AWS_ACCESS_KEY_ID. Prefer the " "env var (a CI/CD variable or gitlab-runner `environment` entry) " @@ -156,10 +153,9 @@ def _docker_memory_args(): # cd to this directory before start DEFAULT_CHDIR = os.environ.get("BITS_CHDIR") or "." -# Default S3 content store for the store-operating actions (certify, compliance, -# gc, store-stats, publish). Precedence: CLI --remote-store (or a .bitsuse-recorded -# one) > $BITS_S3_STORE > this literal. Same env var the bitsStore launcher reads. -DEFAULT_S3_STORE = os.environ.get("BITS_S3_STORE") or "https://s3.cern.ch/lcgapp-bits-testing" +# Store of the store-operating actions (certify, compliance, gc, store-stats, +# publish): --remote-store > $BITS_S3_STORE > the site's store.operations. +DEFAULT_S3_STORE = os.environ.get("BITS_S3_STORE") or site.get("store.operations") or "" # Worker count assumed when --parallel/--builders is given with no number. BUILDERS_AUTO = 4 @@ -313,8 +309,8 @@ def add_architecture_arguments(subparsers, ctx): def add_version_arguments(subparsers, ctx): """`bits version` — display the version and architecture (no options).""" - return subparsers.add_parser("version", help="display %(prog)s version", - description="Display the %(prog)s version (tag, commit and " + return subparsers.add_parser("version", help="display the bits version", + description="Display the bits version (tag, commit and " "date) and the architecture. Same as --version.") @@ -517,9 +513,10 @@ def add_doctor_arguments(subparsers, ctx): be rebuilt from scratch. rsync://, https://, b3:// and s3:// remote stores are recognised. Some of these require credentials: s3:// remotes require an ~/.s3cfg; b3:// remotes require AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - environment variables. A useful remote store is - 'https://s3.cern.ch/swift/v1/alibuild-repo'. It requires no credentials and - provides tarballs for the most common supported architectures. + environment variables. Without --remote-store, the site configuration's + store.read_by_arch / store.read (at CERN: ALICE's public alibuild-repo for its + architectures, else lcgapp-bits-testing) or the community's system: remote_store + is used. """) doctor_remote.add_argument("--no-remote-store", action="store_true", help="Disable the use of the remote store, even if it is enabled by default.") @@ -681,8 +678,8 @@ def add_init_arguments(subparsers, ctx): description="With no PACKAGE, 'bits init' records the supplied options as a " "'bits use' profile (./.bitsuse or a ~/.bits/use record) so you do not " "repeat them on every build, then exits. --architecture goes to [common], " - "the rest to [build]. community/providers have no build flag — set " - "$BITS_COMMUNITY / $BITS_PROVIDERS for those.") + "the rest to [build]. community/providers are not saved — set " + "$BITS_COMMUNITY (or 'bits --community') / $BITS_PROVIDERS for those.") init_cfg.add_argument("--providers", dest="providers", default=None, metavar="URL", help="URL of the bits-providers repository. Has no build-time flag; " "set the BITS_PROVIDERS environment variable instead.") @@ -694,9 +691,10 @@ def add_init_arguments(subparsers, ctx): "'--write-store' in the [build] profile).") init_cfg.add_argument("--community", "--organisation", dest="organisation", default=None, metavar="NAME", - help="Community selecting the registry/provider 'home' repo. Has no " - "build-time flag; set the BITS_COMMUNITY environment variable " - "instead (the aliBuild wrapper sets it).") + help="Community selecting the registry/provider 'home' repo. Not " + "saved; set the BITS_COMMUNITY environment variable or give " + "'bits --community NAME' before the sub-command (the aliBuild " + "wrapper sets it).") # version takes no options; the architecture is auto-detected for display. return init_parser @@ -714,6 +712,7 @@ def add_status_arguments(subparsers, ctx): "from_store (local tarball), from_remote_store (remote tarball, requires " "--check-store), local_checkout (development package, will rebuild), " "local_checkout_unchanged (development package, nothing changed), " + "symlink_from_deployment (matching deployed package), " "build_from_source (will compile), or hash_unknown (git refs not cached; " "e.g. with --no-fetch-repos)." ), @@ -742,6 +741,11 @@ def add_status_arguments(subparsers, ctx): help=("Directory where reference git repos are cached. " "'%%(workDir)s' will be substituted. Default '%(default)s'."), ) + status_parser.add_argument( + "--flavour", "--flavor", "--set", dest="flavours", action="append", + default=[], metavar="NAME[=VALUE]", + help="Same as `bits build --set`; pass the build's values (e.g. " + "release=LCG_110) for the same package hashes.") status_parser.add_argument( "--no-local", dest="noDevel", metavar="PACKAGE", default=[], action="append", @@ -792,10 +796,33 @@ def add_status_arguments(subparsers, ctx): help=("Probe the remote store to detect tarballs not yet mirrored " "locally. Implies a network round-trip per package."), ) + status_parser.add_argument( + "--symlink-packages-from", dest="symlinkPackagesFrom", metavar="DEPLOYMENT-ROOT", + help="Report packages that bits build would link from this deployment.", + ) + status_parser.add_argument( + "--symlink-policy", dest="symlinkPolicy", choices=["strict", "relaxed"], + default="strict", help="Match deployed packages by hash (strict) or path (relaxed).", + ) + status_parser.add_argument( + "--build-local", dest="buildLocal", metavar="PKG[,PKG...]", default="", + help="Exclude these packages from deployment symlink selection.", + ) status_parser.add_argument( "--json", dest="json_output", action="store_true", default=False, help="Emit a machine-readable JSON report instead of the human-readable table.", ) + status_docker = status_parser.add_argument_group( + title="Use a Docker container", + description="As bits build --docker: the system packages and the container " + "fingerprint (in the toolchain's hash) come from the container.") + status_docker.add_argument("--docker", dest="docker", action="store_true", + help="Check for system packages inside a Docker container.") + status_docker.add_argument("--docker-image", dest="dockerImage", metavar="IMAGE", default=None, + help=("The Docker image to use, as for the build. Implies --docker. " + "By default, an image is chosen from the architecture.")) + status_docker.add_argument("--docker-extra-args", metavar="ARGLIST", default="", + help="Command-line arguments to pass to 'docker run'. Implies --docker.") return status_parser @@ -1040,8 +1067,8 @@ def add_publish_arguments(subparsers, ctx): "WORKDIR/MANIFESTS. Use --store to pick the target.")) ctx.remote_store(publish_parser, dest="publishStore", help=("S3 store URL/bucket for --from-manifest. Accepts an https URL " - "(https:///), b3://, or s3://. " - "Default: %(default)s")) + "(https:///), b3://, or s3://; " + "several, comma-separated, each get the build. Default: %(default)s")) # cvmfs-prepub direct-upload path (replaces the spool + bits-ingest + bits-publisher flow). _prepub = publish_parser.add_argument_group( @@ -1114,7 +1141,8 @@ def add_certify_arguments(subparsers, ctx): help=("passkey (default): wait for a passkey approval via bits-console before the MR. " "none: open the MR only — for builds approved elsewhere (a console build; needs --certifier).")) p.add_argument("--console", dest="console", metavar="URL", default=None, - help="bits-console URL for the approval (default: $BITS_CONSOLE_URL, then `system: console_url:`).") + help=("bits-console URL for the approval (default: $BITS_CONSOLE_URL, `system: console_url:`, " + "then the site's console.backend_url).")) p.add_argument("--console-cafile", dest="consoleCafile", metavar="PEM", default=None, help="CA bundle for the console when it is not in this host's trust store (e.g. the CERN CA).") p.add_argument("--console-insecure", dest="consoleInsecure", action="store_true", default=False, @@ -1123,7 +1151,8 @@ def add_certify_arguments(subparsers, ctx): help=("Record USER as certified_by (when a bot opens the MR for a human already " "verified upstream, e.g. bits-console). Default: $GITLAB_USER_LOGIN.")) ctx.remote_store(p, dest="publishStore", - help="S3 store the build is uploaded to / checked in. Default: %(default)s") + help="S3 store the build is uploaded to / checked in; several, comma-" + "separated, each get it, the first is certified. Default: %(default)s") ctx.work_dir(p, help="Work directory holding MANIFESTS/ and the build. Default: %(default)s") ctx.architecture(p, help="Architecture of the build. Default: detected") return p @@ -1276,14 +1305,29 @@ def add_compliance_arguments(subparsers, ctx): help=("ADMIN: remove non-compliant packages from the store — delete their " "TARS objects, rev-index markers and SOURCES archives, rewrite the " "per-build BOMs without them, and (with --key) re-certify the affected " - "architectures. Requires S3 write credentials. Combine with --dry-run " + "architectures; in a store whose certified set bits-orchestrator owns, " + "ask it to take the packages out of the set's BOMs instead (--console, " + "a GitLab token). Requires S3 write credentials. Combine with --dry-run " "to preview every action first.")) compliance_parser.add_argument("--dry-run", dest="dryRun", action="store_true", default=False, help="With --enforce: print every deletion/rewrite without touching anything.") compliance_parser.add_argument("--key", dest="enforceKey", metavar="PEM", default=None, help=("With --enforce: Ed25519 release key to re-sign the affected " "architectures' common manifests after the purge. Without it the next " - "CI certification heals them (removed objects are dropped as missing).")) + "CI certification heals them (removed objects are dropped as missing). " + "Refused for a store whose manifests bits-orchestrator signs.")) + compliance_parser.add_argument("--console", dest="console", metavar="URL", default=None, + help=("With --enforce, in a store bits-orchestrator certifies: the bits-console " + "to request the decertify actions from (default: $BITS_CONSOLE_URL, then the " + "site's console.backend_url).")) + compliance_parser.add_argument("--console-cafile", dest="consoleCafile", metavar="PEM", default=None, + help="CA bundle for the console when it is not in this host's trust store.") + compliance_parser.add_argument("--console-insecure", dest="consoleInsecure", action="store_true", + default=False, + help="Allow http:// / skip TLS verification to the console (trusted testbed only).") + compliance_parser.add_argument("--gitlab-token", dest="gitlabToken", metavar="PAT", default=None, + help=("GitLab token the decertify requests are made with (default: " + "$BITS_CERTIFIER_TOKEN / $GITLAB_TOKEN / ~/.bits/gitlab-token).")) return compliance_parser @@ -1353,6 +1397,45 @@ def add_checksums_arguments(subparsers, ctx): return cp +def add_upgrade_arguments(subparsers, ctx): + """`bits upgrade` — set new package versions in the recipes, in place.""" + up = subparsers.add_parser( + "upgrade", + help="set new versions of packages in the recipes, in place", + description=( + "Resolve PACKAGE as bits build would, find the file that sets each chosen " + "package's version (its recipe, the defaults profile whose overrides pin it, " + "or the recipe whose requires pin it) and set the new version, and a tag " + "that spells it. A file of a repository provider's cached checkout is first " + "copied, with its patches, into CONFIGDIR. The versions come from a plan exported " + "as CSV from bits-console (by default the newest upgrade-plan-*.csv in the current " + "directory) and --to. New tarball URLs, git tags and PyPI versions are checked; a " + "version with no tarball is built from a git tag instead when one is found. Exit " + "status 1 when a package is left for a hand edit or to check."), + ) + up.add_argument("pkgname", nargs="+", metavar="PACKAGE", + help="The package(s) the build builds; a .csv file among them is the plan (as --plan).") + up.add_argument("--to", dest="to", action="append", default=[], metavar="PACKAGE=VERSION", + help="A package and its new version (repeatable).") + up.add_argument("--plan", metavar="FILE", default=None, + help=("An upgrade plan exported as CSV from the console's Plan view (default: the " + "newest upgrade-plan-*.csv in the current directory, unless only --to is given).")) + up.add_argument("--wave", dest="waves", type=int, action="append", default=[], metavar="N", + help="A wave of a release line's plan to take (repeatable; default: every wave).") + up.add_argument("--line", metavar="LINE", default=None, + help="The release line to take from a converged plan.") + ctx.defaults(up, help="Defaults profile(s), as for the build; use :: to combine. Default: %(default)s.") + ctx.architecture(up, help="Architecture, as for the build. Default: %(default)s.") + ctx.config_dir(up, help="The recipe directory, as for the build; copies go there. Default: %(default)s.") + ctx.work_dir(up, help="Work directory, as for the build. Default: %(default)s.") + up.add_argument("--set", dest="variables", action="append", default=[], metavar="NAME=VALUE", + help="As for the build (e.g. release=LCG_110) (repeatable).") + ctx.search_path(up) + up.add_argument("--no-check", dest="no_check", action="store_true", default=False, + help="Do not check the new sources (URLs, git tags, PyPI).") + return up + + def add_build_arguments(subparsers, ctx): """`bits build` — build a package.""" build_parser = subparsers.add_parser("build", help="build a package", @@ -1519,11 +1602,11 @@ def add_build_arguments(subparsers, ctx): Builds can be done inside a Docker container, to make it easier to get a common, usable environment. The Docker daemon must be installed and running on your system. With --docker and no --docker-image, the image is derived from - the architecture as /-[-cuda]:latest, where the - registry is BITS_DOCKER_REGISTRY, else a `docker_registry:` field in - defaults-release, else the built-in default (BITS_DOCKER_TAG overrides the - tag). Set BITS_LEGACY_REGISTRY=1 for the legacy alisw/-builder images - (the aliBuild wrapper sets this). Images are fetched if unavailable. + the architecture as /-[-cuda]:. The registry is + BITS_DOCKER_REGISTRY, else system: docker_registry in defaults-release, else + the site configuration's containers.registry; the tag likewise BITS_DOCKER_TAG, + system: docker_tag, containers.tag. BITS_LEGACY_REGISTRY=1 (set by the aliBuild + wrapper) uses containers.legacy_image. Images are fetched if unavailable. """) build_docker.add_argument("--docker", dest="docker", action="store_true", help="Build inside a Docker container.") @@ -1573,14 +1656,17 @@ def add_build_arguments(subparsers, ctx): (no VM, no overhead). On Linux, podman (rootless) is used only when --docker is active (a nested podman container inside the builder image) or when requested explicitly with --sandbox=podman; a plain local Linux build is not - sandboxed and never invokes podman. + sandboxed and never invokes podman. In the macOS sandbox a build may write + only its own files in the work dir, the temp dirs and a few caches. """) build_sandbox.add_argument( - "--sandbox", dest="sandbox", metavar="MODE", default="off", + "--sandbox", dest="sandbox", metavar="MODE", default=None, choices=["off", "auto", "podman", "sandbox-exec"], help=( "Recipe sandbox mode. " - "'off' (default): no sandboxing. " + "'off' (default): no sandboxing, unless $BITS_SANDBOX=1 or `sandbox: on` " + "in the active defaults asks for sandbox-exec on a macOS build without " + "--docker ($BITS_SANDBOX=0 overrides the defaults). " "'auto': sandbox-exec on macOS, nested podman when --docker is active " "and podman is present in the builder image (else off), and off on a " "local Linux build. " @@ -1588,6 +1674,10 @@ def add_build_arguments(subparsers, ctx): "'sandbox-exec': macOS only." ), ) + build_sandbox.add_argument( + "--sandbox-allow-write", dest="sandboxAllowWrite", metavar="PATH", default=[], + action="append", + help="With sandbox-exec, also let builds write under PATH (repeatable).") build_sandbox.add_argument( "--sandbox-image", dest="sandboxImage", metavar="IMAGE", default=None, help=( @@ -1616,9 +1706,10 @@ def add_build_arguments(subparsers, ctx): be rebuilt from scratch. rsync://, https://, b3:// and s3:// remote stores are recognised. Some of these require credentials: s3:// remotes require an ~/.s3cfg; b3:// remotes require AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - environment variables. A useful remote store is - 'https://s3.cern.ch/swift/v1/alibuild-repo'. It requires no credentials and - provides tarballs for the most common supported architectures. To archive to + environment variables. Without --remote-store, the site configuration's + store.read_by_arch / store.read (at CERN: ALICE's public alibuild-repo for its + architectures, else lcgapp-bits-testing; store.read not with --write-store + alone) or the community's system: remote_store is used. To archive to and reuse from your own bucket, use a b3:// store (append ::rw to --remote-store, or add --write-store) and set the connection via the --s3-* options / env vars below; a non-CERN endpoint (AWS, MinIO, Ceph) is supported @@ -1692,6 +1783,19 @@ def add_build_arguments(subparsers, ctx): "not configured. A trailing '::relaxed' or '::strict' also sets the reuse " "policy (e.g. 'cvmfs::relaxed'); it must agree with --reuse-policy if both " "are given. Defaults to $BITS_REUSE_FROM; an empty value turns reuse off.")) + build_remote.add_argument("--symlink-packages-from", dest="symlinkPackagesFrom", + metavar="DEPLOYMENT-ROOT", default=None, + help=("Find installed packages under ROOT//[family/]/" + ", or ROOT/share for shared packages. " + "Link matches into the work directory with private relocated " + "initialization files. No modulefiles or --reuse-from are needed.")) + build_remote.add_argument("--symlink-policy", dest="symlinkPolicy", + choices=["strict", "relaxed"], default="strict", + help=("Package symlink selection: strict (default) requires .build-hash " + "to match a computed package hash (with signed reuse, a certified one); " + "relaxed matches the architecture, " + "family and version path without checking the hash. " + "Independent of --reuse-policy.")) build_remote.add_argument("--build-local", dest="buildLocal", metavar="PKG[,PKG...]", default="", help=("Comma-separated packages to always build locally even under " "--reuse-policy relaxed (e.g. a package you need patched), rather than " @@ -1720,12 +1824,15 @@ def add_build_arguments(subparsers, ctx): metavar="N", help="""\ Start N background threads that pre-download pre-built tarballs and source - archives for all packages in the build graph before they are needed, so that + archives for the packages the build needs before it needs them, so that downloads overlap the (serial) preparation loop instead of blocking it. A .downloading sentinel file coordinates with the build loop so no file is fetched twice. Default: -1 (auto = min(builders, 4)); 0 disables prefetch. Works in all build modes. """) + build_remote.add_argument("--fetch-build-requires", dest="fetchBuildRequires", action="store_true", + help=("Fetch or build the build-only dependencies of packages that are " + "reused from a store too (by default they are skipped: nothing needs them).")) build_remote.add_argument("--parallel-downloads", dest="parallelDownloads", type=int, default=2, metavar="N", help="""\ @@ -1861,9 +1968,11 @@ def doParseArgs(): # `ctx` carries the shared cross-cutting adders (architecture, work_dir, …). ctx = _ArgCtx(detectedArch) - parser = argparse.ArgumentParser(epilog="""\ - For help about each option, specify --help after the option itself. For - complete documentation please refer to https://alisw.github.io/alibuild. + # The command people type (bitsBuild is run through it). + _prog = "aliBuild" if os.environ.get("BITS_BRANDING", "").strip().lower() == "alibuild" else "bits" + parser = argparse.ArgumentParser(prog=_prog, epilog="""\ + Each command lists its options with --help. Documentation: + https://github.com/bitsorg/bits/tree/main/docs """) from bits_helpers import _VERSION_INFO @@ -1899,6 +2008,7 @@ def __call__(self, parser, namespace, values, option_string=None): verify_parser = add_verify_arguments(subparsers, ctx) add_sbom_arguments(subparsers, ctx) add_checksums_arguments(subparsers, ctx) + add_upgrade_arguments(subparsers, ctx) stats_parser = add_stats_arguments(subparsers, ctx) import_parser = add_import_arguments(subparsers, ctx) @@ -1955,14 +2065,20 @@ def optionOrder(x): # have their own (see _own_dry_run above). if x in ["--debug", "-d"]: return 0 - if x in ["-n", "--dry-run"] and not _own_dry_run: - return 0 + if x in ["-n", "--dry-run"]: + return 3 if _own_dry_run else 0 # their own -n goes after them # if x in ["build", "init", "clean", "analytics", "doctor", "deps"]: if x in ["build", "init", "clean", "doctor", "deps"]: return 1 return 2 rest.sort(key=optionOrder) sys.argv = [prog] + rest + if "upgrade" in sys.argv[1:4]: # its plan.csv may follow the options: put it first + k = sys.argv.index("upgrade") + 1 + tail = sys.argv[k:] + first = [i for i, t in enumerate(tail) + if t.lower().endswith(".csv") and (i == 0 or tail[i - 1] != "--plan")] + sys.argv[k:] = [tail[i] for i in first] + [t for i, t in enumerate(tail) if i not in first] # For "bits init" config mode: record which flags were explicit on the CLI so # that doInitConfig() can write only the settings the user actually specified. @@ -2066,24 +2182,17 @@ def _defaults_architecture_template(args): Apple Silicon: osx_arm64 """ -# When updating this variable, also update docs/docs/user.md! -S3_SUPPORTED_ARCHS = "slc7_x86-64", "slc8_x86-64", "ubuntu2004_x86-64", "ubuntu2204_x86-64", "ubuntu2404_x86-64", "slc9_x86-64", "slc9_aarch64" -# Match S3 support by (distro, machine) rather than exact string, so an -# equivalent layout (e.g. ubuntu2404_x86_64) still resolves to the same entry. -_S3_SUPPORTED_ARCH_KEYS = {normalise_arch_key(a) for a in S3_SUPPORTED_ARCHS} - -def _defaults_docker_registry(args): - """Return the `docker_registry:` value from the merged defaults chain, or None. +def _defaults_docker(args, key): + """A container setting (docker_registry, docker_tag) of the merged defaults + chain, under system: or, the old spelling, top-level; else None. - A community pins its container registry in defaults-release either as a bare - top-level `docker_registry:` or under `system: docker_registry:`. Read - defensively: a malformed/unreadable defaults set must not break arg parsing. + Read defensively: a malformed/unreadable defaults set must not break arg parsing. """ try: meta, _ = readDefaults(args.configDir, args.defaults, lambda *a, **k: None, args.architecture) sysd = meta.get("system", {}) or {} - val = sysd.get("docker_registry", meta.get("docker_registry")) + val = sysd.get(key, meta.get(key)) return val.strip() if isinstance(val, str) and val.strip() else None except (Exception, SystemExit): # readDefaults calls sys.exit(1) on a malformed defaults file; don't let this @@ -2135,11 +2244,127 @@ def _with_release_base(defaults): return defaults +def _finalise_docker(args, parser): + """The container options of build, doctor, deps and status: --docker implied by + an image or extra args, the docker run arguments, the image and the platform.""" + if args.dockerImage or args.docker_extra_args: + args.docker = True + + args.docker_extra_args = shlex.split(args.docker_extra_args) + args.docker_extra_args.append("--network=host") + # Pin the build container to the full set of online host CPUs so that + # make -j sees the real core count rather than the cgroup + # quota inherited from the GitLab runner process. + # /sys/devices/system/cpu/online gives the kernel-reported online CPU + # list (e.g. "0-7") which reflects actual hardware, not the caller's + # cgroup CPU quota. Only inject if the user hasn't already specified + # --cpuset-cpus in --docker-extra-args. + # Rootless podman can only apply limits whose cgroup controller is delegated + # to the user; skip the others rather than fail to start the container. + # (build.py also runs such containers with --userns=keep-id, not --user.) + args.rootless_podman = bool(args.docker) and _is_rootless_podman() + _ctrls = _rootless_podman_controllers() if args.rootless_podman else None + # Podman labels containers for SELinux (docker-ce does not), which blocks + # the bind-mounted home-dir workdir/configdir; :z relabelling is too slow on + # SOURCES and impossible on /cvmfs, so turn labelling off instead. + if args.rootless_podman and not any(a.startswith("--security-opt") + for a in args.docker_extra_args): + args.docker_extra_args.append("--security-opt=label=disable") + _skipped = [] + if not any(a.startswith("--cpuset-cpus") for a in args.docker_extra_args): + if _ctrls is None or "cpuset" in _ctrls: + args.docker_extra_args.append("--cpuset-cpus=" + _host_online_cpus()) + else: + _skipped.append("cpuset") + + # Hard memory cap on the build container so that no single build can OOM + # the HOST (see _docker_memory_args). Skipped when the user passes any + # --memory* themselves. + if not any(a.startswith("--memory") for a in args.docker_extra_args): + if _ctrls is None or "memory" in _ctrls: + args.docker_extra_args.extend(_docker_memory_args()) + else: + _skipped.append("memory") + if _skipped: + from bits_helpers.log import warning + warning("rootless podman: cgroup controller(s) %s not available to this user, " + "so the build container is not limited by them. To enable (root): put " + "'[Service]' and 'Delegate=cpu cpuset io memory pids' in " + "/etc/systemd/system/user@.service.d/delegate.conf, run " + "'systemctl daemon-reload', then log in again.", + ", ".join(_skipped)) + + if args.docker and args.architecture.startswith("osx"): + parser.error("cannot use `-a %s` and --docker" % args.architecture) + + if args.docker and commands.getstatusoutput("which docker")[0]: + parser.error("cannot use --docker as docker executable is not found") + + # If specified, used the docker image requested, otherwise, if running + # in docker the docker image is given by the first part of the + # architecture we want to build for. + if args.docker and not args.dockerImage: + # The image from the architecture: /-[-cuda]:, + # each from the environment, the defaults' system: or the site configuration; + # BITS_LEGACY_REGISTRY (the aliBuild wrapper) or no registry: legacy_image. + # Only the base OS and the -cuda axis pick the image; the compiler axes + # (-gccNN/-clang) and build type (-opt/-dbg) are resolved inside the + # container by the entrypoint shim, so they do not change it. + distro_token = arch_distro_token(args.architecture) or args.architecture.split("_")[0] + _legacy = os.environ.get("BITS_LEGACY_REGISTRY", "").strip().lower() in ("1", "true", "yes", "on") + _registry = None if _legacy else ( + os.environ.get("BITS_DOCKER_REGISTRY", "").strip() + or _defaults_docker(args, "docker_registry") + or site.get("containers.registry")) + _machine = (arch_machine_token(args.architecture) or "").replace("-", "_") + if _registry and _machine and distro_token: + _name = "%s-%s" % (_machine, distro_token) + if re.search(r"(^|-)cuda(-|$)", args.architecture): + _name += "-cuda" + _tag = (os.environ.get("BITS_DOCKER_TAG", "").strip() + or _defaults_docker(args, "docker_tag") or site.get("containers.tag") or "latest") + args.dockerImage = "%s/%s:%s" % (str(_registry).rstrip("/"), _name, _tag) + elif site.get("containers.legacy_image"): + args.dockerImage = str(site.get("containers.legacy_image")).replace("{distro}", distro_token) + else: + parser.error("--docker: no image for %s; give --docker-image, or set containers.registry " + "in a site configuration file" % args.architecture) + + # ── --docker-platform / cross-compilation ───────────────────────────────── + # Derive the Docker --platform value from --architecture when the user has + # not set it explicitly. If the target architecture matches the host we + # leave it as None (no --platform flag → daemon uses native image variant, + # zero overhead). If they differ we inject the matching platform string so + # that Docker pulls the correct image variant and QEMU transparently + # emulates the target ISA inside the builder container. + # + # The special sentinel value "native" lets users opt out of automatic + # injection even when cross-compiling (useful on native ARM runners that + # happen to run an x86-64 bits client, or for testing). + if args.docker: + if getattr(args, "dockerPlatform", None) == "native": + args.dockerPlatform = None + elif not getattr(args, "dockerPlatform", None): + from bits_helpers.arch import docker_platform_for_arch, detectArch as _detectArch + target_plat = docker_platform_for_arch(args.architecture) + host_plat = docker_platform_for_arch(_detectArch()) + if target_plat and target_plat != host_plat: + args.dockerPlatform = target_plat + else: + args.dockerPlatform = None + + # --sandbox-image implies --sandbox=podman. Promote from the mode-unset + # states ("off" is the default, "auto" the legacy default); an explicit + # "podman"/"sandbox-exec" is left as chosen. + if getattr(args, "sandboxImage", None) and getattr(args, "sandbox", None) in (None, "off", "auto"): + args.sandbox = "podman" + + def finaliseArgs(args, parser): # Nothing to finalise for version, architecture, or verify # if args.action in ["version", "analytics", "architecture"]: - if args.action in ["version", "architecture", "verify", "stats", "sbom", "checksums"]: + if args.action in ["version", "architecture", "verify", "stats", "sbom", "checksums", "upgrade"]: return args # Minimal finalisation for cvmfs-path: only the defaults profile is loaded @@ -2159,6 +2384,7 @@ def finaliseArgs(args, parser): args.noDevel = normalise_multiple_options(args.noDevel) args.disable = normalise_multiple_options(args.disable) args.force_rebuild = normalise_multiple_options(args.force_rebuild) + args.flavours = _parse_flavours(getattr(args, "flavours", None)) args.referenceSources = args.referenceSources % {"workDir": args.workDir} # Repository-provider configuration, mirroring the general path below: # `bits status` reports what `bits build` WOULD do, so it must resolve @@ -2167,11 +2393,12 @@ def finaliseArgs(args, parser): _alibuild = os.environ.get("BITS_BRANDING", "").strip().lower() == "alibuild" args.bits_providers = ( os.environ.get("BITS_PROVIDERS") - or ("" if _alibuild else "https://github.com/bitsorg/bits-providers")) + or ("" if _alibuild else site.get("providers.url") or "")) if args.bits_providers: os.environ.setdefault("BITS_PROVIDERS", args.bits_providers) args.provider_policy = _parse_provider_policy( getattr(args, "providerPolicy", None) or "") + _finalise_docker(args, parser) return args # compliance group mode rides the general finalisation: it needs the @@ -2195,9 +2422,8 @@ def finaliseArgs(args, parser): # child processes inherit it. Under the aliBuild wrapper (BITS_BRANDING=aliBuild) # the built-in default is off (classic aliBuild uses a local alidist checkout); # native `bits` defaults to the provider path. - _BITS_PROVIDERS_DEFAULT = "https://github.com/bitsorg/bits-providers" _alibuild_mode = os.environ.get("BITS_BRANDING", "").strip().lower() == "alibuild" - _providers_default = "" if _alibuild_mode else _BITS_PROVIDERS_DEFAULT + _providers_default = "" if _alibuild_mode else site.get("providers.url") or "" args.bits_providers = os.environ.get("BITS_PROVIDERS") or _providers_default if args.bits_providers: os.environ.setdefault("BITS_PROVIDERS", args.bits_providers) @@ -2271,120 +2497,7 @@ def finaliseArgs(args, parser): cleanup_git_log(args.referenceSources) if args.action in ("build", "doctor", "deps"): - if args.dockerImage or args.docker_extra_args: - args.docker = True - - args.docker_extra_args = shlex.split(args.docker_extra_args) - args.docker_extra_args.append("--network=host") - # Pin the build container to the full set of online host CPUs so that - # make -j sees the real core count rather than the cgroup - # quota inherited from the GitLab runner process. - # /sys/devices/system/cpu/online gives the kernel-reported online CPU - # list (e.g. "0-7") which reflects actual hardware, not the caller's - # cgroup CPU quota. Only inject if the user hasn't already specified - # --cpuset-cpus in --docker-extra-args. - # Rootless podman can only apply limits whose cgroup controller is delegated - # to the user; skip the others rather than fail to start the container. - # (build.py also runs such containers with --userns=keep-id, not --user.) - args.rootless_podman = bool(args.docker) and _is_rootless_podman() - _ctrls = _rootless_podman_controllers() if args.rootless_podman else None - # Podman labels containers for SELinux (docker-ce does not), which blocks - # the bind-mounted home-dir workdir/configdir; :z relabelling is too slow on - # SOURCES and impossible on /cvmfs, so turn labelling off instead. - if args.rootless_podman and not any(a.startswith("--security-opt") - for a in args.docker_extra_args): - args.docker_extra_args.append("--security-opt=label=disable") - _skipped = [] - if not any(a.startswith("--cpuset-cpus") for a in args.docker_extra_args): - if _ctrls is None or "cpuset" in _ctrls: - args.docker_extra_args.append("--cpuset-cpus=" + _host_online_cpus()) - else: - _skipped.append("cpuset") - - # Hard memory cap on the build container so that no single build can OOM - # the HOST (see _docker_memory_args). Skipped when the user passes any - # --memory* themselves. - if not any(a.startswith("--memory") for a in args.docker_extra_args): - if _ctrls is None or "memory" in _ctrls: - args.docker_extra_args.extend(_docker_memory_args()) - else: - _skipped.append("memory") - if _skipped: - from bits_helpers.log import warning - warning("rootless podman: cgroup controller(s) %s not available to this user, " - "so the build container is not limited by them. To enable (root): put " - "'[Service]' and 'Delegate=cpu cpuset io memory pids' in " - "/etc/systemd/system/user@.service.d/delegate.conf, run " - "'systemctl daemon-reload', then log in again.", - ", ".join(_skipped)) - - if args.docker and args.architecture.startswith("osx"): - parser.error("cannot use `-a %s` and --docker" % args.architecture) - - if args.docker and commands.getstatusoutput("which docker")[0]: - parser.error("cannot use --docker as docker executable is not found") - - # If specified, used the docker image requested, otherwise, if running - # in docker the docker image is given by the first part of the - # architecture we want to build for. - if args.docker and not args.dockerImage: - # Choose the builder image from the architecture. Resolution order: - # 1. BITS_LEGACY_REGISTRY truthy -> legacy alisw/-builder - # (ALICE guard, mirrors BITS_LEGACY_INITDOTSH; the aliBuild wrapper - # sets it so ALICE keeps its images regardless of any registry here). - # 2. otherwise -> a bits-containers-style image from the configured - # registry: BITS_DOCKER_REGISTRY env, else a `docker_registry:` field - # in defaults-release, else the built-in DEFAULT_DOCKER_REGISTRY. - # Image = /-[-cuda]: (BITS_DOCKER_TAG, - # default latest). If the arch cannot be decomposed, fall back to the - # legacy alisw builder. - # Only the base OS and the -cuda axis pick the image; the compiler axes - # (-gccNN/-clang) and build type (-opt/-dbg) are resolved inside the - # container by the entrypoint shim, so they do not change it. - distro_token = arch_distro_token(args.architecture) or args.architecture.split("_")[0] - _legacy = os.environ.get("BITS_LEGACY_REGISTRY", "").strip().lower() in ("1", "true", "yes", "on") - _registry = None if _legacy else ( - os.environ.get("BITS_DOCKER_REGISTRY", "").strip() - or _defaults_docker_registry(args) - or DEFAULT_DOCKER_REGISTRY) - _machine = (arch_machine_token(args.architecture) or "").replace("-", "_") - if _registry and _machine and distro_token: - _name = "%s-%s" % (_machine, distro_token) - if re.search(r"(^|-)cuda(-|$)", args.architecture): - _name += "-cuda" - _tag = os.environ.get("BITS_DOCKER_TAG", "").strip() or "latest" - args.dockerImage = "%s/%s:%s" % (_registry.rstrip("/"), _name, _tag) - else: - args.dockerImage = "registry.cern.ch/alisw/%s-builder" % distro_token - - # ── --docker-platform / cross-compilation ───────────────────────────────── - # Derive the Docker --platform value from --architecture when the user has - # not set it explicitly. If the target architecture matches the host we - # leave it as None (no --platform flag → daemon uses native image variant, - # zero overhead). If they differ we inject the matching platform string so - # that Docker pulls the correct image variant and QEMU transparently - # emulates the target ISA inside the builder container. - # - # The special sentinel value "native" lets users opt out of automatic - # injection even when cross-compiling (useful on native ARM runners that - # happen to run an x86-64 bits client, or for testing). - if args.docker: - if getattr(args, "dockerPlatform", None) == "native": - args.dockerPlatform = None - elif not getattr(args, "dockerPlatform", None): - from bits_helpers.arch import docker_platform_for_arch, detectArch as _detectArch - target_plat = docker_platform_for_arch(args.architecture) - host_plat = docker_platform_for_arch(_detectArch()) - if target_plat and target_plat != host_plat: - args.dockerPlatform = target_plat - else: - args.dockerPlatform = None - - # --sandbox-image implies --sandbox=podman. Promote from the mode-unset - # states ("off" is the default, "auto" the legacy default); an explicit - # "podman"/"sandbox-exec" is left as chosen. - if getattr(args, "sandboxImage", None) and getattr(args, "sandbox", "off") in ("off", "auto"): - args.sandbox = "podman" + _finalise_docker(args, parser) if "annotate" in args: for comment_assignment in args.annotate: @@ -2412,17 +2525,23 @@ def finaliseArgs(args, parser): # otherwise system.remote_store overrides the built-in arch default below. args.remoteStoreExplicit = bool(args.remoteStore) - # A public read store is enabled by default on selected platforms. Unlike + # Without one given, the site's read store for the architecture. Unlike # aliBuild, activating a store does NOT force --no-system (reuse is # content-hash addressed); use --no-system for a self-contained build. - if normalise_arch_key(args.architecture) in _S3_SUPPORTED_ARCH_KEYS and not args.preferSystem and not args.no_remote_store: - if not args.remoteStore: - args.remoteStore = "https://s3.cern.ch/swift/v1/alibuild-repo" - elif args.no_remote_store: + if args.no_remote_store: args.remoteStore = "" + elif not args.remoteStore and not args.preferSystem: + # store.read is the default only when no write store is given (a write + # store alone is also the read store) and not for aliBuild. + args.remoteStore = site.read_store(args.architecture, fallback=not ( + args.writeStore or _alibuild_mode)) if args.remoteStore.endswith("::rw") and args.writeStore: parser.error("cannot specify ::rw and --write-store at the same time") + if "::rw" in args.writeStore: + parser.error("::rw belongs to --remote-store, not --write-store") + if "::rw" in args.remoteStore and "," in args.remoteStore: + parser.error("::rw takes one store; give several write stores with --write-store") if args.remoteStore.endswith("::rw"): args.remoteStore = args.remoteStore[0:-4] @@ -2433,10 +2552,10 @@ def finaliseArgs(args, parser): from bits_helpers.sync import normalise_store_url args.normalisedStores = [] for _attr in ("remoteStore", "writeStore"): - _url = getattr(args, _attr) - if normalise_store_url(_url) != _url: - args.normalisedStores.append((_url, normalise_store_url(_url))) - setattr(args, _attr, normalise_store_url(_url)) + _urls = [u.strip() for u in getattr(args, _attr).split(",") if u.strip()] + args.normalisedStores += [(u, normalise_store_url(u)) for u in _urls + if normalise_store_url(u) != u] + setattr(args, _attr, ",".join(normalise_store_url(u) for u in _urls)) if args.action in ["build", "init"]: if "develPrefix" in args and args.develPrefix is None: diff --git a/bits_helpers/bits_use.py b/bits_helpers/bits_use.py index a4b88bb8..6aa190fc 100644 --- a/bits_helpers/bits_use.py +++ b/bits_helpers/bits_use.py @@ -234,7 +234,7 @@ def merged_argv(command, user_args, path=None): # store, version, help, cvmfs-stage/publish) and `verify` (accepts neither # --architecture nor --defaults) take a different option set and are excluded. INJECT_ACTIONS = { - "build", "deps", "doctor", "status", "clean", "prune", "cleanup", "gc", + "build", "deps", "doctor", "status", "upgrade", "clean", "prune", "cleanup", "gc", "import", "publish", "certify", "sign", "compliance", "q", "query", "enter", "setenv", "printenv", "load", "unload", } diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 7065bd8f..34321a65 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -7,6 +7,7 @@ from bits_helpers import __version__ from bits_helpers.log import debug, info, banner, warning from bits_helpers.log import dieOnError +from bits_helpers import site from bits_helpers.repo_provider import (fetch_repo_providers_iteratively, load_always_on_providers, announce_providers, MAX_PROVIDER_ITERATIONS, no_recipes_hint) @@ -21,7 +22,7 @@ from bits_helpers.utilities import resolve_store_path, resolve_links_path, ver_rev, is_virtual_package from bits_helpers.arch import detectArch, effective_arch, SHARED_ARCH, compute_combined_arch, compute_own_hash_arch from bits_helpers.defaults import parseDefaults, readDefaults -from bits_helpers.matchers import resolve_variables +from bits_helpers.matchers import resolve_variables, exported_flavours from bits_helpers.packages import getPackageList from bits_helpers.initdotsh import generate_initdotsh from bits_helpers.hashing import storeHashes @@ -36,7 +37,10 @@ from bits_helpers.scm import SCMError from bits_helpers.sync import remote_from_url from bits_helpers.build_config import BuildConfig -from dataclasses import dataclass +from bits_helpers import plan as _plan +from bits_helpers import fetch_cache +from bits_helpers.build_progress import BuildProgress, package_info, package_label, set_package_progress +from dataclasses import dataclass, field from bits_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources try: from bits_helpers.resource_monitor import run_monitor_on_command @@ -131,6 +135,8 @@ def defaults_store_url(defaults_meta): return "", False url = str(url).strip() writable = url.endswith("::rw") + dieOnError("::rw" in url and "," in url, + "system: remote_store: ::rw takes one store, not a list (%s)" % url) from bits_helpers.sync import normalise_store_url return normalise_store_url(url[:-4] if writable else url), writable @@ -243,7 +249,17 @@ def prepare_hash_inputs(spec, specs, defaults, default_vars, config_dir, spec["version"] = devel_version -def _prefetch_package(spec, sync_helper, work_dir, build_arch, source_arch=None) -> None: +def prefetch_wanted(build_order, states): + """[(package, with its sources)] worth prefetching, from the plan *states*: an + installed or linked package needs neither tarball nor sources, one from the + store no sources. Without a plan, everything.""" + return [(p, states.get(p) != _plan.REMOTE) for p in build_order + if states.get(p) not in (_plan.INSTALLED, _plan.LOCAL_TARBALL, _plan.OVERLAY, + _plan.SYMLINK)] + + +def _prefetch_package(spec, sync_helper, work_dir, build_arch, source_arch=None, + sources=True) -> None: """Background task: prefetch the prebuilt tarball + all source archives. Uses the sentinel-file mechanism (``.downloading`` files; see @@ -303,6 +319,8 @@ def _prefetch_package(spec, sync_helper, work_dir, build_arch, source_arch=None) # Another thread is already fetching this tarball; just wait. _wait_for_sentinel(tar_hash_dir) + if not sources: # reused from the store: its sources are not needed + return # --- Source archive prefetch ------------------------------------------------ # Warms the download cache (SOURCES/cache) only; checkout_sources copies from # there. download() uses _acquire_download/_wait_for_sentinel internally, so @@ -335,7 +353,8 @@ def _prefetch_package(spec, sync_helper, work_dir, build_arch, source_arch=None) def readHashFile(fn): try: - return open(fn).read().strip("\n") + with open(fn) as f: + return f.read().strip("\n") except OSError: return "0" @@ -343,21 +362,13 @@ def readHashFile(fn): def _localize_manifest(src, work_dir): """Return a local filesystem path for a manifest *src*. - A local path is returned unchanged; a URL is fetched (with its detached - ``.sig``) into ``MANIFESTS/trust`` and the local path returned. Note - ``downloadUrllib2`` keeps a pre-existing file of the same name, so repeated - calls within one build (e.g. from both trust-index and trust-records loaders) - resolve to the same local manifest. + A local path is returned unchanged; a URL is fetched afresh (with its detached + ``.sig``) into ``MANIFESTS/trust``, once per build, so the trust-index and + trust-records loaders verify the same copy. A failed fetch raises: the + callers then reuse nothing from that manifest (never an older cached copy). """ - if "://" not in src: - return src - from bits_helpers.download import downloadUrllib2 - dest = os.path.join(work_dir, "MANIFESTS", "trust") - os.makedirs(dest, exist_ok=True) - name = os.path.basename(src.split("?")[0]) or "trust-manifest.json" - downloadUrllib2(src, dest, work_dir, dest_filename=name) - downloadUrllib2(src + ".sig", dest, work_dir, dest_filename=name + ".sig") - return os.path.join(dest, name) + from bits_helpers.download import fetch_signed_manifest + return fetch_signed_manifest(src, os.path.join(work_dir, "MANIFESTS", "trust"), work_dir) _TRUST_IMPORT_WARNED = False @@ -387,7 +398,7 @@ def _try_import_trust(): def _load_trusted_index(src, work_dir, accept_groups): - """Verify one signed manifest (local path or URL) -> (key_id, {hash: sha256}). + """Verify one signed manifest (local path or URL) -> (key_id, {(arch, hash): sha256}). Returns ``(None, {})`` on any failure (missing/unreachable/untrusted, or the signing library not installed), so a caller can degrade to a rebuild rather @@ -397,8 +408,8 @@ def _load_trusted_index(src, work_dir, accept_groups): if trust is None: return None, {} try: - return trust.trusted_index(_localize_manifest(src, work_dir), - accept_groups=accept_groups) + return trust.trusted_object_index(_localize_manifest(src, work_dir), + accept_groups=accept_groups) except Exception as exc: warning("--require-signed-reuse: could not fetch/verify %s (%s); those " "entries will not be reused.", src, exc) @@ -406,14 +417,15 @@ def _load_trusted_index(src, work_dir, accept_groups): def trusted_reuse_index(args, work_dir): - """Return the verified {hash: tarball_sha256} index for --require-signed-reuse. - - --trust-manifest may name several signed manifests (comma-separated): a build - node fetches its own ``common-manifest-.json`` plus the always-relevant - ``common-manifest-shared.json`` and this merges their indexes. Each is - signature-verified independently; content hashes are globally unique so the - merge is collision-free. Loaded once, memoised on *args*. An empty map means - nothing is attested, so every remote reuse fails closed. + """Return the verified {(effective_architecture, hash): tarball_sha256} index + for --require-signed-reuse. + + --trust-manifest may name several signed manifests (comma-separated), one per + architecture plus ``common-manifest-shared.json``; this merges their indexes. + Each is signature-verified independently. A hash alone is not unique (it does + not include the architecture), the stored object (architecture, hash) is. + Loaded once, memoised on *args*. An empty map means nothing is attested, so + every remote reuse fails closed. """ cached = getattr(args, "_trustedReuseIndex", None) if cached is not None: @@ -429,6 +441,7 @@ def trusted_reuse_index(args, work_dir): "no remote tarball will be reused.") guessed = getattr(args, "_guessedTrustManifests", None) or set() absent = [] + conflicts = set() for src in sources: kid, part = _load_trusted_index(src, work_dir, accept_groups) if not kid and src in guessed and _url_not_found(src): @@ -440,7 +453,18 @@ def trusted_reuse_index(args, work_dir): warning("--require-signed-reuse: could not verify signed manifest %s; " "its tarballs will not be reused.", src) continue - index.update(part) + for key, sha in part.items(): + if key in conflicts: + continue + if key in index and index[key] != sha: + # Two verified manifests disagree about one stored object: the store is + # inconsistent somewhere, so neither is trusted for it. + warning("--require-signed-reuse: %s@%s is certified with two sha256 (%s, %s); " + "it will not be reused.", key[1], key[0], index[key], sha) + conflicts.add(key) + del index[key] + continue + index[key] = sha debug("Trusted reuse index from %s (signed by %s): %d entries%s", src, kid, len(part), "" if accept_groups is None else " (groups: %s + common)" % ",".join(accept_groups)) @@ -673,11 +697,43 @@ def _store_container_root(store, endpoint=None): if store.startswith(("b3://", "s3://")): bucket = store.split("://", 1)[1].split("/", 1)[0].split("::", 1)[0] if bucket: - ep = str(endpoint or "https://s3.cern.ch").rstrip("/") + ep = str(endpoint or site.get("store.s3_endpoint") or "").rstrip("/") + if not ep: + return None return "%s/swift/v1/%s" % (ep, bucket) return None +def trust_manifest_srcs(store, prefixes, endpoint, work_dir, arch): + """``(sources, guessed)``: the signed manifests a build trusts. + + *prefixes* is one prefix or several, comma-separated, in order of preference + (e.g. ``STAGING/common-manifest,MANIFESTS/common-manifest``): for each + architecture's manifest, the first prefix that has one is used, so a later + prefix only fills in the architectures the earlier ones lack. The one used is + not checked here: an expired or unverifiable one is not replaced by a later + prefix's. Prefer the manifests actually in the store (arch-independent + hashes) over names guessed from our own arch; *guessed* is the set of guessed + names (they may not exist). With guesses only (no listing found any), every + prefix's guesses are kept, so their indexes are merged, not preferred. + """ + order = [p.strip().lstrip("/") for p in str(prefixes or "").split(",") if p.strip()] \ + or ["MANIFESTS/common-manifest"] + listed = [(p, _list_store_manifest_srcs(store, p, endpoint, work_dir)) for p in order] + if not any(found for _p, found in listed): + # The store cannot be listed (or holds none): every prefix's guessed names, + # all kept, since a guess that does not exist is skipped later. + guessed = [s for p in order for s in derive_trust_manifest_srcs(store, p, arch, endpoint)] + return guessed, set(guessed) + chosen = {} + for prefix, found in listed: + leaf = os.path.basename(prefix) + for src in found: + name = os.path.basename(src) # -.json + chosen.setdefault(name[len(leaf):] if name.startswith(leaf) else name, src) + return sorted(chosen.values()), set() + + def derive_trust_manifest_srcs(store, prefix, arch, endpoint=None): """Construct the signed common-manifest URLs for *arch* + the shared one. @@ -698,12 +754,12 @@ def derive_trust_manifest_srcs(store, prefix, arch, endpoint=None): def _list_store_manifest_srcs(store, prefix, endpoint, work_dir): """Every signed common-manifest present in the store (swift JSON listing). - Content hashes are architecture-independent, so trusting all signed manifests - lets a build reuse a hash certified under a different arch than its own (e.g. a - bare x86_64-el9 build reusing a gcc15-qualified toolchain). This widens the - trust set from arch-scoped to every manifest under the prefix in the bucket; - each is still signature + group verified per manifest downstream. [] on any - failure -> the caller falls back to derive_trust_manifest_srcs. + A build's packages are stored under more than its own architecture name (an + own-hash toolchain under the arch without the build type, ``share`` packages + under ``share``), so every manifest under the prefix is loaded; each entry + still vouches only for its own (architecture, hash), and each manifest is + signature + group verified downstream. [] on any failure -> the caller falls + back to derive_trust_manifest_srcs. """ prefix = str(prefix or "MANIFESTS/common-manifest").lstrip("/") root = _store_container_root(store, endpoint) @@ -1195,6 +1251,9 @@ def _closure_untracked(): _provenance = "own_hash" else: _provenance = "loose" if _closure_untracked() else "pure" + if (getattr(args, "symlinkPackagesFrom", None) + and getattr(args, "symlinkPolicy", "strict") == "relaxed"): + _provenance = "loose" from bits_helpers.deps import deps_graph from bits_helpers.build_epoch import BUILD_EPOCH @@ -1220,6 +1279,7 @@ def _closure_untracked(): # never hashed — declared under system:, which is not part of any pkg hash). "cvmfs_templates": getattr(args, "cvmfsTemplates", None), "reuse_policy": getattr(args, "reusePolicy", "strict") or "strict", + "symlink_policy": getattr(args, "symlinkPolicy", "strict"), "provenance": _provenance, # Dependencies this package linked but excluded from its identity hash. "untracked_requires": _untracked, @@ -1304,6 +1364,22 @@ def _extract_error_excerpt(log_path, max_match=15, tail=12, scan_limit=20000): return "\n".join(out) +def _after_recipe(output, log_path, limit=8): + """The error lines bits' build script printed after the recipe's log ended: + the failure when the recipe itself succeeded (a missing rsync).""" + try: + with open(log_path, errors="replace") as f: + last = [ln.rstrip("\n") for ln in f if ln.strip()][-1:] + except OSError: + return [] # no recipe log to start after + output = list(output) + start = max((i + 1 for i, ln in enumerate(output) if [ln] == last), default=None) + if start is None: + return [] + return [ln for ln in output[start:] if not ln.startswith("+") + and (_ERROR_PATTERNS.search(ln) or re.search(r": line \d+: ", ln))][-limit:] + + def write_failure_summary(work_dir, scheduler, arch): """Write a concise per-run failure summary for a --builders build. @@ -1340,13 +1416,13 @@ def write_failure_summary(work_dir, scheduler, arch): # different effective platforms, and these run-level logs are not otherwise # arch-scoped, so write them under LOGS// to avoid cross-platform # clobbering. Fall back to the work-dir root if the directory can't be made. - log_dir = os.path.join(work_dir, "LOGS", arch or "") + log_dir = os.path.join(abspath(work_dir), "LOGS", arch or "") try: os.makedirs(log_dir, exist_ok=True) except OSError as exc: warning("Could not create log dir %s (%s); writing logs to %s instead", log_dir, exc, work_dir) - log_dir = work_dir + log_dir = abspath(work_dir) full_path = os.path.join(log_dir, "build-errors-full.log") try: with open(full_path, "w") as fh: @@ -1387,31 +1463,41 @@ def write_failure_summary(work_dir, scheduler, arch): return (path, full_path) +def report_failure_summary(work_dir, scheduler, arch): + """Write failure reports and print the log paths for each failed package.""" + summary_path, full_path = write_failure_summary(work_dir, scheduler, arch) + fails = getattr(scheduler, "buildFailures", None) or [] + if not (summary_path or full_path or fails): + return + info("=" * 70) + info("Build finished with errors. Where to look:") + if summary_path: + info(" Summary (start here): %s", summary_path) + if full_path: + info(" Full error log: %s", full_path) + if fails: + info(" Failed package build logs:") + for failure in sorted(fails, key=lambda x: x["package"].lower()): + info(" %s: %s", failure["package"], failure["log"]) + info("=" * 70) + + def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scriptDir, workDir, syncHelper): spec = specs[p] debug("Build command: %s", build_command) - progress = debug # Reuse marker (shown even without --debug): "[from store]" when a remote store # is configured (tarball from the shared cache), "[cached]" for a local-only # tarball. Blank when the package is compiled. _reuse_tag = ((" [from store]" if getattr(syncHelper, "remoteStore", "") else " [cached]") if cachedTarball else "") - if args.builders==1: - progress_msg = ("Unpacking %s@%s" + _reuse_tag) if cachedTarball else "Compiling %s@%s" - if not cachedTarball and not args.debug: - progress_msg += " (use --debug for full output)" - progress = ProgressPrint( - progress_msg % - (spec["package"], - args.develPrefix if "develPrefix" in args and spec["is_devel_pkg"] else spec["version"]) - ) - else: - scheduler.log ( - (("Unpacking %s@%s" + _reuse_tag) if cachedTarball else - "Compiling %s@%s (use --debug for full output)") % - (spec["package"], - args.develPrefix if "develPrefix" in args and spec["is_devel_pkg"] else spec["version"]) - ) + stage = "Unpacking" + _reuse_tag if cachedTarball else "Building" + if not cachedTarball and not args.debug: + stage += " (use --debug for full output)" + version = (args.develPrefix if "develPrefix" in args and spec["is_devel_pkg"] + else spec["version"]) + progress = BuildProgress(package_label(spec, version), stage, + emit=scheduler.log if args.builders > 1 else info, + interactive=args.builders == 1) # Report progress (no-op unless running under gitlab-runner): this package is # now starting. defaults-release is excluded to match the planned total. if spec["package"] != "defaults-release": @@ -1456,19 +1542,37 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr "in command); container build runs unthrottled this slot.") else: build_command = "nice -n %d /bin/sh -c %s" % (nice_level, shlex.quote(build_command)) + _cache = getattr(args, "fetchCache", "") + _cache = _cache if not cachedTarball and fetch_cache.enabled(spec, _cache) else "" + _arch = effective_arch(spec, args.architecture) + if _cache: + _cache_state = fetch_cache.restore(spec, syncHelper, workDir, _cache, _arch) from bits_helpers import monitor as _bits_monitor + from collections import deque + _output = deque(maxlen=400) # the end of the build script's output + + def _printer(fmt, *a): + _output.append(fmt % a if a else fmt) + progress(fmt, *a) _bits_monitor.note_build(p, getattr(args, "architecture", ""), True) try: if args.resourceMonitoring: - err = run_monitor_on_command(build_command, "{}/{}.json".format(scriptDir, p), printer=progress) + err = run_monitor_on_command(build_command, "{}/{}.json".format(scriptDir, p), printer=_printer) else: - err = execute(build_command, printer=progress) + err = execute(build_command, printer=_printer) + except BaseException: + progress.end(error=True) + raise finally: _bits_monitor.note_build(p, getattr(args, "architecture", ""), False) if nice_ladder is not None: nice_ladder.release(nice_token) - if args.builders==1: - progress.end("failed" if err else "done", err) + if err: + progress.end(error=True) + if _cache and not err: + _bwd = args.workDir if spec["is_devel_pkg"] else os.environ.get("BITS_BUILD_WORK_DIR", args.workDir) + fetch_cache.save(spec, syncHelper, workDir, _cache, _arch, _cache_state, + os.path.join(_bwd, "BUILD", spec["hash"], fetch_cache.LISTING)) # We do not use the override for devel packages, because we # want to avoid having to rebuild things when the /tmp gets cleaned. @@ -1479,8 +1583,7 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr # Determine paths devSuffix = "-" + args.develPrefix if "develPrefix" in args and spec["is_devel_pkg"] else "" - log_path = f"{buildWorkDir}/BUILD/{spec['package']}-latest{devSuffix}/log" - log_abs_path = log_path # keep the absolute path; log_path may become relative below + log_path = abspath(join(buildWorkDir, "BUILD", f"{spec['package']}-latest{devSuffix}", "log")) build_dir = f"{buildWorkDir}/BUILD/{spec['package']}-latest{devSuffix}/{spec['package']}" # Staging install prefix ($INSTALLROOT in the recipe): where the package's # files are installed before being tarred. Useful for inspecting a partial @@ -1492,10 +1595,10 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr except Exception: # pylint: disable=broad-except install_root = None - # Use relative paths if we're inside the work directory + # Use relative paths for the build and install directories, but keep the log + # path absolute so it can be opened directly from any working directory. try: from os.path import relpath - log_path = relpath(log_path, os.getcwd()) build_dir = relpath(build_dir, os.getcwd()) if install_root: install_root = relpath(install_root, os.getcwd()) @@ -1525,10 +1628,18 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr # Surface the proximate error so the user does not have to open the full log. excerpt = "" if err: - excerpt = _extract_error_excerpt(log_abs_path) or "" + excerpt = _extract_error_excerpt(log_path) or "" if excerpt: buildErrMsg += f"\n{bold}Error excerpt:{reset}\n" buildErrMsg += excerpt + "\n" + after = _after_recipe(_output, log_path) + if after: + buildErrMsg += f"\n{bold}After the recipe, bits' build script printed:{reset}\n" + buildErrMsg += "".join(" %s\n" % ln for ln in after) + from bits_helpers.sandbox import failure_hint + hint = failure_hint(build_command) + if hint: + buildErrMsg += f"\n{bold}Sandbox:{reset}\n {hint}\n" updatablePkgs = [dep for dep in spec["requires"] if specs[dep]["is_devel_pkg"]] if spec["is_devel_pkg"]: @@ -1659,7 +1770,14 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr else: dieOnError(err, buildErrMsg.strip()) - doFinalSync(spec, specs, args, syncHelper) + progress.update("Syncing package to store" if getattr(syncHelper, "writeStore", "") + else "Finalizing package", interactive=False) + try: + doFinalSync(spec, specs, args, syncHelper) + except BaseException: + progress.end(error=True) + raise + progress.end() def _doCheckout(spec, workDir, referenceSources, docker, enforce_mode, @@ -1686,6 +1804,42 @@ def _doCheckout(spec, workDir, referenceSources, docker, enforce_mode, return "" +def _report_reuse(args, main_package, metrics_url, instance): + """The end-of-build summary of what was reused and built (and why), and the + same counts as metrics. Best-effort: never fails a finished build.""" + try: + from bits_helpers import reuse_report + packages = args.manifest.packages() + text = reuse_report.format_summary(packages) + if text: + banner("%s", text) + if metrics_url: + from bits_helpers.metrics import push_prometheus + from bits_helpers.monitor import default_instance + push_prometheus(metrics_url, "\n".join(reuse_report.metric_lines( + packages, instance() or default_instance(), args.architecture, main_package)) + "\n", + timeout=5) + except Exception as exc: # pylint: disable=broad-except + debug("reuse summary: %s", exc) + + +def _rebuild_reason(spec, sync_helper): + """Why a package with no tarball to reuse is built here, in a few words.""" + from bits_helpers.sync import NoRemoteSync, binary_redistributable + if spec.get("is_devel_pkg"): + return "development package" + if spec.get("force_rebuild"): + return "forced rebuild" + if is_virtual_package(spec): + return "never stored" + if not binary_redistributable(spec): + return "licence: never stored" + if sync_helper is None or isinstance(sync_helper, NoRemoteSync): + return "no remote store" + # (Absent, or the fetch failed: either way nothing came from the store.) + return "not found in the store" + + def doFinalSync(spec, specs, args, syncHelper): # We need to create 2 sets of links, once with the full requires, # once with only direct dependencies, since that's required to @@ -1709,15 +1863,15 @@ def doFinalSync(spec, specs, args, syncHelper): # "no redistribution" clause covers the binary form. The package is still # built and usable locally; it just never leaves this host. if getattr(syncHelper, "writeStore", ""): - info("%s@%s [NOT uploaded — redistributable: %s]", - spec["package"], spec["version"], spec.get("redistributable")) + package_info("%s: not uploaded (redistributable: %s)", package_label(spec), + spec.get("redistributable")) elif not spec["revision"].startswith("local") and not is_virtual_package(spec): syncHelper.upload_symlinks_and_tarball(spec) # Log (info level) that a freshly built tarball was pushed to the write store. # Reused packages (cachedTarball) are already there and were marked # "[from store]" at build time, so they are skipped here. if getattr(syncHelper, "writeStore", "") and not spec.get("cachedTarball"): - info("%s@%s [uploaded]", spec["package"], spec["version"]) + package_info("%s: uploaded", package_label(spec)) # Record the tarball's SHA-256 in the local integrity ledger so that # future recalls from the store can be verified against it. # Only active when --store-integrity is set (or store_integrity = true @@ -1973,6 +2127,13 @@ class _BuildLoopCtx: prefetch_executor: object; cmake_prefix_env: bool specs_for_checksum_phase: object; monitoredDirs: dict build_work_dir: str = "" + # Issue #141: the plan's states, the packages it found not needed (build-only + # dependencies of reused packages), the live build order (they go back into it + # when such a package is built after all) and the full order they are sorted by. + planned: dict = field(default_factory=dict) + skipped: set = field(default_factory=set) + build_order: list = field(default_factory=list) + full_order: list = field(default_factory=list) def build_one_package(p, ctx): @@ -1997,6 +2158,7 @@ def build_one_package(p, ctx): monitoredDirs = ctx.monitoredDirs spec = specs[p] + debug("%s: Preparing (checking installed packages and cache)", package_label(spec)) log_current_package(p, mainPackage, specs, getattr(args, "develPrefix", None)) # Calculate the hashes. We do this in build order so that we can guarantee @@ -2012,11 +2174,58 @@ def build_one_package(p, ctx): debug("Hashes for recipe %s are %s (remote); %s (local)", p, ", ".join(spec["remote_hashes"]), ", ".join(spec["local_hashes"])) - # 4b: if the reuse overlay satisfies this package, set it up from modules - # instead of building. Its consumers 'module load' it (generate_initdotsh); - # it is neither built nor materialized locally, so we skip the whole - # build/unpack path here (this is what avoids the legacy tarball synthesis + - # relocate-me.sh). Strict = same remote hash (byte-identical, publishable); + # Direct deployment selection needs neither a modulefile nor an overlay. + # Resolve the effective architecture (including share) before looking up + # the package, and retain the selected revision even when it is empty. + from bits_helpers.reuse_symlinks import ( + select_symlink_package, matching_local_install, stage_reused_package, stage_hook_scripts) + try: + _linked = select_symlink_package( + spec, cfg, args.architecture, + trusted=(trusted_reuse_index(args, workDir) + if getattr(cfg, "symlink_packages_from", None) and cfg.require_signed_reuse + else None)) + if _linked: + _existing = matching_local_install(spec, _linked, workDir, args.architecture) + if _existing: + spec["revision"] = _linked["revision"] + spec["hash"] = _linked["hash"] + spec["cachedTarball"] = "" + spec.setdefault("deps_hash", "") + package_info("%s: Already installed at %s (%s)", package_label(spec), _existing, + _linked["hash"][:12]) + if getattr(args, "manifest", None) is not None: + args.manifest.add_package(spec, "already_installed", + effective_architecture=effective_arch(spec, args.architecture)) + return + _runtime_work = workDir + if args.docker: + _runtime_work = (cfg.cvmfs_prefix or + (workDir if args.containerUseWorkDir else "/container/bits/sw")) + _local_reuse = stage_reused_package( + spec, _linked["install_base"], workDir, args.architecture, + runtime_work_dir=_runtime_work, source_path=_linked["source"], + stage_hooks=stage_hook_scripts(spec, getattr(args, "configDir", ""))) + spec["revision"] = _linked["revision"] + spec["hash"] = _linked["hash"] + spec["reuse_symlinked"] = True + spec["symlink_source"] = _linked["source"] + spec["symlink_policy"] = cfg.symlink_policy + spec["cachedTarball"] = "" + spec.setdefault("deps_hash", "") + package_info("%s: Linked from %s (%s policy) to %s", package_label(spec), + _linked["source"], cfg.symlink_policy, _local_reuse) + if getattr(args, "manifest", None) is not None: + args.manifest.add_package(spec, "already_installed", + effective_architecture=effective_arch(spec, args.architecture)) + return + except (OSError, ValueError) as exc: + dieOnError(True, "Cannot symlink package %s: %s" % (p, exc)) + + # 4b: select reused packages from the module overlay. Consumers source init.sh + # from the deployment. Local symlink staging is selected independently by + # --symlink-packages-from above. + # Both paths skip building/unpacking. Strict = same remote hash (publishable); # relaxed = any version in the one-release overlay. defaults-*, --build-local # and development packages are never grafted. if (cfg.reuse_overlay and not spec["is_devel_pkg"] @@ -2043,7 +2252,7 @@ def build_one_package(p, ctx): spec["hash"] = spec.get("remote_revision_hash") or spec.get("hash", "") spec["cachedTarball"] = "" spec.setdefault("deps_hash", "") - info("Reuse: %s from CVMFS overlay as module %s (not built)", p, _mid) + package_info("%s: Reusing CVMFS overlay module %s (not built)", package_label(spec), _mid) if getattr(args, "manifest", None) is not None: args.manifest.add_package(spec, "already_installed", effective_architecture=effective_arch(spec, args.architecture)) @@ -2337,7 +2546,7 @@ def build_one_package(p, ctx): workDir, effective_arch(spec, args.architecture), spec) if spec["devel_hash"]+spec["deps_hash"] == spec["old_devel_hash"] \ and os.path.isdir(devel_install_dir): - info("Development package %s does not need rebuild", spec["package"]) + package_info("%s: Development package does not need rebuild", package_label(spec)) return if spec["devel_hash"]+spec["deps_hash"] == spec["old_devel_hash"]: debug("Devel package %s source unchanged but install dir %s is missing " @@ -2365,7 +2574,7 @@ def _handle_already_installed(hashPath): if fileHash == spec["hash"] and not spec["is_devel_pkg"]: # If we get here, the installed package matches the selected hash. It is # already usable, so we can register its dependency closure and skip work. - debug("Package %s is already installed with the expected hash.", spec["package"]) + debug("%s: Already installed", package_label(spec)) # own_hash arch bridge (reuse compatibility): an own_hash package (the # toolchain) is installed ONLY under its build-type-NEUTRAL arch. A consumer # whose init.sh was generated by the current initdotsh sources it from that @@ -2457,7 +2666,7 @@ def _handle_already_installed(hashPath): # otherwise-complete package, unlike a freshly built one in doFinalSync. try: syncHelper.upload_symlinks_and_tarball(spec) - info("%s@%s [uploaded from local cache]", spec["package"], spec["version"]) + package_info("%s: uploaded from local cache", package_label(spec)) except Exception as exc: warning("%s@%s store sync failed: %s", spec["package"], spec["version"], exc) else: @@ -2493,6 +2702,7 @@ def _handle_already_installed(hashPath): tar_hash_dir = os.path.join(workDir, resolve_store_path(effective_arch(spec, args.architecture), spec["hash"])) debug("Looking for cached tarball in %s", tar_hash_dir) spec["cachedTarball"] = "" + _remote_tars = set() # tarballs the fetch below brings from the remote store # Virtual packages (provides_repository, defaults-release) are never in the # store: skip the fetch/select/trusted-reuse scan entirely and build locally. if not spec["is_devel_pkg"] and not is_virtual_package(spec): @@ -2539,6 +2749,7 @@ def _handle_already_installed(hashPath): syncHelper.fetch_tarball(spec) tarballs = [t for t in glob(os.path.join(tar_hash_dir, "*gz")) if os.path.isfile(t)] # skip dangling symlinks + _remote_tars = set(tarballs) - _preFetchTars finally: if _hold_sentinel: try: @@ -2563,12 +2774,13 @@ def _handle_already_installed(hashPath): if (spec["cachedTarball"] and cfg.require_signed_reuse and spec["cachedTarball"] not in _preFetchTars): _idx = trusted_reuse_index(args, workDir) - _sha = _idx.get(spec["hash"]) + _sha = _idx.get((effective_arch(spec, args.architecture), spec["hash"])) if _sha is None: warning("Trusted reuse: %s@%s not vouched for by the signed manifest; " "discarding remote tarball and rebuilding.", spec["package"], spec["hash"]) spec["cachedTarball"] = "" + spec["reuse_note"] = "store tarball not certified" else: _actual = compute_checksum_file(spec["cachedTarball"]) if _actual != _sha: @@ -2579,6 +2791,7 @@ def _handle_already_installed(hashPath): "manifest (expected %s, got %s); discarding and rebuilding.", os.path.basename(spec["cachedTarball"]), _sha, _actual) spec["cachedTarball"] = "" + spec["reuse_note"] = "store tarball differs from the certified one" else: debug("Trusted reuse: %s@%s verified against signed manifest", spec["package"], spec["hash"]) @@ -2611,6 +2824,36 @@ def _handle_already_installed(hashPath): return shutil.rmtree(local_hash_path.encode("utf-8"), True) + # Planned as reused but built after all (e.g. its tarball failed the signed + # manifest check): the build-only dependencies the plan skipped for it come + # back first, then this package again. Its remote tarballs stay gated. + _back = [] + if ctx.skipped and not spec["cachedTarball"] and ctx.planned.get(p) in _plan.REUSED: + _want = _plan.needed(ctx.full_order, specs, ctx.planned, [p], build_targets={p}) + _back = [d for d in ctx.full_order if d in ctx.skipped and d in _want] + if ctx.planned.get(p) == _plan.REMOTE and not spec["cachedTarball"]: + try: # not in the store after all: asked for again next time + from bits_helpers.store_cache import StoreCache, store_id + _kept = StoreCache(workDir, store_id(syncHelper)) + _kept.forget(effective_arch(spec, args.architecture), spec.get("remote_hashes") or ()) + _kept.save() + except Exception as exc: # pylint: disable=broad-except + debug("could not update the store cache: %s", exc) + if _back: + warning("%s is built after all: fetching or building first what it needs to build: %s", + p, ", ".join(_back)) + ctx.skipped.difference_update(_back) + for d in _back: # their own loop pass decides afresh + for k in ("hash", "revision", "not_needed"): + specs[d].pop(k, None) + spec["prefetched_tarballs"] = sorted(set(spec.get("prefetched_tarballs", ())) | _remote_tars) + ctx.build_order[0:0] = _back + [p] + return + + # Why this package is built here (the end-of-build reuse summary says). + if not spec["cachedTarball"] and not spec.get("reuse_note"): + spec["reuse_note"] = _rebuild_reason(spec, syncHelper) + # The actual build script. fp = open(dirname(realpath(__file__))+'/build_template.sh') @@ -2677,7 +2920,29 @@ def _handle_already_installed(hashPath): # Packages base (an absolute /cvmfs path, identical on host and in the # container once /cvmfs is mounted). _reuse_cvmfs_base = cfg.reuse_cvmfs_base + _reuse_relocations = {} + _reuse_relocations_text = "" + if cfg.symlink_packages_from: + from bits_helpers.reuse_symlinks import serialize_relocation_table + for _reused_spec in specs.values(): + if _reused_spec.get("reuse_symlinked"): + _reuse_relocations.update(_reused_spec.get("reuse_reference_paths", {})) + if _reuse_relocations: + # Exact package mappings above preserve families and effective arches. + # The fallback maps other paths under the deployment's Packages root. + _reuse_relocations[cfg.symlink_packages_from.rstrip("/")] = abspath(init_workDir) + try: + _reuse_relocations_text = serialize_relocation_table(_reuse_relocations) + except ValueError as exc: + dieOnError(True, str(exc)) makedirs(scriptDir, exist_ok=True) + _reuse_relocations_file = "" + if _reuse_relocations_text: + _map_name = "reuse-relocations.tsv" + writeAll(join(scriptDir, _map_name), _reuse_relocations_text + "\n") + _reuse_relocations_file = join( + abspath(init_workDir), os.path.relpath(scriptDir, abspath(args.workDir)), + _map_name) # Remember where the resource monitor will write this package's trace so we # can aggregate build stats once the run finishes (P3). if args.resourceMonitoring: @@ -2711,6 +2976,7 @@ def _handle_already_installed(hashPath): # actual build script bits_dir = dirname(dirname(realpath(__file__))) buildEnvironment = [ + ("BITS_REUSE_RELOCATIONS_FILE", _reuse_relocations_file), ("ARCHITECTURE", raw_architecture), ("EFFECTIVE_ARCHITECTURE", effective_arch(spec, args.architecture)), ("BUILD_REQUIRES", " ".join(spec["build_requires"])), @@ -2761,6 +3027,9 @@ def _handle_already_installed(hashPath): # build_template.sh makes a private writable copy so the shared read-only # SOURCES tree is never mutated. Empty (off) for modern out-of-tree builds. ("BITS_PRIVATE_SOURCE", "1" if not getattr(args, "initdotshFromModules", True) else ""), + # fetch_cache: true (issue #129), relative to WORK_DIR; empty when off. + ("BITS_FETCH_CACHE", fetch_cache.relative_dir(spec, effective_arch(spec, args.architecture)) + if fetch_cache.enabled(spec, getattr(args, "fetchCache", "")) else ""), ] if "sources" in spec: for idx, src in enumerate(spec["sources"]): @@ -2787,6 +3056,9 @@ def _handle_already_installed(hashPath): buildEnvironment += [(key, value) for key, value in spec.get("track_env", {}).items()] + # What the macOS sandbox lets this package's build write is derived from it. + _pkg_env = {k: v for k, v in buildEnvironment if isinstance(v, str)} + # In case the --docker options is passed, we setup a docker container which # will perform the actual build. Otherwise build as usual using bash. if args.docker: @@ -2843,7 +3115,16 @@ def _handle_already_installed(hashPath): jobLabel=("--label bits-job=%s " % quote(_job_id)) if _job_id else "", # Mount /cvmfs read-only when reusing deployed components, so a reused # dep's init.sh (and its files under /cvmfs) resolve inside the container. - cvmfsMount=("-v /cvmfs:/cvmfs:ro " if cfg.reuse_cvmfs_base else ""), + cvmfsMount=(("-v /cvmfs:/cvmfs:ro " if cfg.reuse_cvmfs_base or + (cfg.symlink_packages_from and + (cfg.symlink_packages_from == "/cvmfs" or + cfg.symlink_packages_from.startswith("/cvmfs/"))) else "") + + "".join( + "-v %s " % quote("%s:%s:ro" % (root, root)) + for root in dict.fromkeys((cfg.reuse_cvmfs_base, + cfg.symlink_packages_from)) + if cfg.symlink_packages_from and root and root.rstrip("/") != "/cvmfs" + and not root.startswith("/cvmfs/"))), platformArg="--platform %s " % quote(_docker_platform) if _docker_platform else "", roSources=_ro_sources, image=quote(args.dockerImage), @@ -2894,6 +3175,10 @@ def _handle_already_installed(hashPath): docker_active=bool(getattr(args, "docker", False)), container_workdir=container_workDir if getattr(args, "docker", False) else None, docker_image=getattr(args, "dockerImage", None), + pkg_env=_pkg_env, + build_workdir=abspath(args.workDir if spec["is_devel_pkg"] + else os.environ.get("BITS_BUILD_WORK_DIR", args.workDir)), + script_dir=abspath(scriptDir), ) @@ -2918,6 +3203,121 @@ def _handle_already_installed(hashPath): scheduler.parallel("build:%s" % p, build_deps, "build", runBuildCommand, scheduler, p, specs, args, build_command,cachedTarball, scriptDir, workDir, syncHelper) +def enter_target(package, spec, build_family): + """The module to enter: the version just built, else the family's latest alias.""" + if spec is not None: + return "%s/%s" % (package, ver_rev(spec)) + return "%s/latest-%s" % (package, build_family) + + +def latest_alias_note(package, spec, build_family): + """A note naming the family's latest alias, when the hint names the version built.""" + if spec is None or not build_family: + return "" + return "\n\n(%s/latest-%s enters the newest build of this family.)" % (package, build_family) + + +def providers_and_defaults(args, packages, workDir, defaultsReader, _cli_disable, + overrides, taps, defaultsMeta, parse=parseDefaults): + """Clone the repository providers and re-read the defaults until both settle + (bits status uses this too, for the same defaults and hashes). Returns + (overrides, taps, defaultsMeta, provider_dirs, fetched provider dirs).""" + always_on_dirs = load_always_on_providers( + config_dir = args.configDir, + work_dir = workDir, + reference_sources = args.referenceSources, + fetch_repos = getattr(args, "fetchRepos", True), + bits_providers = getattr(args, "bits_providers", None), + taps = taps, + provider_policy = getattr(args, "provider_policy", {}), + force_tracked = getattr(args, "forceTracked", False), + overrides = overrides, + defaults = args.defaults, + default_vars = defaultsMeta.get("variables"), + ) + # Discovery <-> defaults resolution is a FIXED POINT, not one pass. The override + # that turns the `release` variable into an lcg.bits branch tag + # (`overrides: lcg.bits: tag: "%(release)s"`) can itself live INSIDE a provider + # (the shared stacks base), so it is invisible on the first, config-dir-only + # parse: a stacks-inheriting community (key4hep/ship/lhcb) would otherwise fetch + # lcg.bits at the bare `requires:` default (main) regardless of `release` or a + # `--set release=` on the CLI, because there is no override yet to consume the + # value. So fetch with the overrides/variables known so far, re-parse now that + # the providers (and their overrides) are on BITS_PATH, and repeat until the set + # of (provider, commit) pins stops moving. Re-pointing a provider changes its + # checkout dir, so BITS_PATH is rebuilt from a stable base each pass to stop a + # superseded tag from shadowing the new one. atlas declares the override in its + # own config dir, so it converges on the first fetch (the second pass just + # confirms the fixed point); an inheriting community converges once the stacks + # override/release become visible. + _base_bits_path = os.environ.get("BITS_PATH", "") + provider_dirs = {} + _prev_pin_sig = None + for _disc_pass in range(MAX_PROVIDER_ITERATIONS): + os.environ["BITS_PATH"] = _base_bits_path + defaults_provider_seed = ( + list(defaultsMeta.get("requires", [])) + + list(defaultsMeta.get("build_requires", [])) + + list(getattr(args, "_bootstrap_provider_requires", []) or []) + ) + provider_dirs = fetch_repo_providers_iteratively( + packages = packages + defaults_provider_seed, + config_dir = args.configDir, + work_dir = workDir, + reference_sources = args.referenceSources, + fetch_repos = getattr(args, "fetchRepos", True), + taps = taps, + provider_policy = getattr(args, "provider_policy", {}), + overrides = overrides, + defaults = args.defaults, + default_vars = defaultsMeta.get("variables"), + force_tracked = getattr(args, "forceTracked", False), + announce = False, # once, after the loop converges + ) + _fetched_dirs = dict(provider_dirs) + provider_dirs.update(always_on_dirs) + # (provider, commit) pin set: order-independent and re-point-sensitive. + _pin_sig = frozenset((_n, _h) for (_d, (_n, _h)) in provider_dirs.items()) + # Re-resolve the defaults chain now that the provider repos are on BITS_PATH, + # so the full stacks base + compiler axis contribute their env / package_family + # / overrides / append_arch / system. Reset disable to the original CLI set so + # a provider-supplied disable is picked up without double-counting config-dir + # ones. + args.disable = list(_cli_disable) + (err, overrides, taps, defaultsMeta) = parse(args.disable, + defaultsReader, debug, args.architecture, args.configDir) + dieOnError(err, err) + if _pin_sig == _prev_pin_sig: + break + _prev_pin_sig = _pin_sig + else: + warning("Provider discovery did not reach a fixed point after %d passes; " + "using the last resolved provider set (some provider pins may still " + "be moving).", MAX_PROVIDER_ITERATIONS) + return overrides, taps, defaultsMeta, provider_dirs, _fetched_dirs + + +def resolve_defaults_variables(meta, args, branch_basename): + """The defaults' variables as a build resolves them: gated entries, the --set + flavours, the release (bits status uses this too, for the same hashes).""" + flavours = getattr(args, "flavours", None) or {} + exported = exported_flavours(meta.get("variables"), flavours) + meta["variables"] = resolve_variables( + meta.get("variables"), flavours, args.architecture, args.defaults) + # The release: an explicit non-trunk release:, else the recipe branch, else + # main; %(release)s then picks the lcg.bits branch, as the CVMFS {release} does. + from bits_helpers.cvmfs_layout import ( + resolve_release as _resolve_release, _declared_release as _declared) + if isinstance(meta.get("variables"), dict) and _declared(meta): + meta["variables"]["release"] = _resolve_release(meta, branch_basename) + # Flavours also go into the env (so the hash), not those of `export: false` variables. + if exported: + from collections import OrderedDict as _OD + # An empty `env:` block parses to None, so setdefault would keep it None. + if not isinstance(meta.get("env"), dict): + meta["env"] = _OD() + for _k, _v in exported.items(): + meta["env"][_k] = _v def doBuild(args, parser): @@ -2973,32 +3373,7 @@ def defaultsReader(): # --flavour values are folded in as inputs. The resulting map feeds the # (?NAME) conditional matchers that gate package requires and %(NAME)s recipe # templating. CLI flavours override a defaults entry of the same name. - flavours = getattr(args, "flavours", None) or {} - meta["variables"] = resolve_variables( - meta.get("variables"), flavours, args.architecture, args.defaults) - # Resolve the release LABEL and write it back into `release`, so that - # %(release)s (which the defaults feed to the lcg.bits branch override) - # follows the same rule as the CVMFS {release} slot: an explicit non-trunk - # release: wins, else the working-dir branch, else "main". This is what makes - # a build on a recipe branch fetch the matching lcg.bits branch, while the - # default/main line keeps building lcg.bits `main` exactly as before. - # Only groups that OPT IN by declaring a `release` are touched — a group using - # a different convention (e.g. %(lcgversion)s) is left untouched, so no extra - # variable is introduced and its package hashes are unchanged. - from bits_helpers.cvmfs_layout import ( - resolve_release as _resolve_release, _declared_release as _declared) - if isinstance(meta.get("variables"), dict) and _declared(meta): - meta["variables"]["release"] = _resolve_release(meta, branch_basename) - # Flavours are ALSO exported into the build environment + package hash (via - # the `env:` map, which becomes the defaults-release env every package - # depends on), exactly as before. - if flavours: - from collections import OrderedDict as _OD - # An empty `env:` block parses to None, so setdefault would keep it None. - if not isinstance(meta.get("env"), dict): - meta["env"] = _OD() - for _k, _v in flavours.items(): - meta["env"][_k] = _v + resolve_defaults_variables(meta, args, branch_basename) # init.sh-from-modules (the default) publishes a build-mode marker through the # defaults-release env. Routing it here is deliberate: the defaults env is # (a) folded into every package's hash, so the mode yields a distinct, @@ -3039,78 +3414,8 @@ def defaultsReader(): # Discovery is transitive (fetch_repo_providers_iteratively re-reads each # cloned provider's own requires), so the config-dir seed only needs the # first hop and a single re-parse then covers the whole provider graph. - always_on_dirs = load_always_on_providers( - config_dir = args.configDir, - work_dir = workDir, - reference_sources = args.referenceSources, - fetch_repos = args.fetchRepos, - bits_providers = getattr(args, "bits_providers", None), - taps = taps, - provider_policy = getattr(args, "provider_policy", {}), - force_tracked = getattr(args, "forceTracked", False), - overrides = overrides, - defaults = args.defaults, - default_vars = defaultsMeta.get("variables"), - ) - # Discovery <-> defaults resolution is a FIXED POINT, not one pass. The override - # that turns the `release` variable into an lcg.bits branch tag - # (`overrides: lcg.bits: tag: "%(release)s"`) can itself live INSIDE a provider - # (the shared stacks base), so it is invisible on the first, config-dir-only - # parse: a stacks-inheriting community (key4hep/ship/lhcb) would otherwise fetch - # lcg.bits at the bare `requires:` default (main) regardless of `release` or a - # `--set release=` on the CLI, because there is no override yet to consume the - # value. So fetch with the overrides/variables known so far, re-parse now that - # the providers (and their overrides) are on BITS_PATH, and repeat until the set - # of (provider, commit) pins stops moving. Re-pointing a provider changes its - # checkout dir, so BITS_PATH is rebuilt from a stable base each pass to stop a - # superseded tag from shadowing the new one. atlas declares the override in its - # own config dir, so it converges on the first fetch (the second pass just - # confirms the fixed point); an inheriting community converges once the stacks - # override/release become visible. - _base_bits_path = os.environ.get("BITS_PATH", "") - provider_dirs = {} - _prev_pin_sig = None - for _disc_pass in range(MAX_PROVIDER_ITERATIONS): - os.environ["BITS_PATH"] = _base_bits_path - defaults_provider_seed = ( - list(defaultsMeta.get("requires", [])) - + list(defaultsMeta.get("build_requires", [])) - + list(getattr(args, "_bootstrap_provider_requires", []) or []) - ) - provider_dirs = fetch_repo_providers_iteratively( - packages = packages + defaults_provider_seed, - config_dir = args.configDir, - work_dir = workDir, - reference_sources = args.referenceSources, - fetch_repos = args.fetchRepos, - taps = taps, - provider_policy = getattr(args, "provider_policy", {}), - overrides = overrides, - defaults = args.defaults, - default_vars = defaultsMeta.get("variables"), - force_tracked = getattr(args, "forceTracked", False), - announce = False, # once, after the loop converges - ) - _fetched_dirs = dict(provider_dirs) - provider_dirs.update(always_on_dirs) - # (provider, commit) pin set: order-independent and re-point-sensitive. - _pin_sig = frozenset((_n, _h) for (_d, (_n, _h)) in provider_dirs.items()) - # Re-resolve the defaults chain now that the provider repos are on BITS_PATH, - # so the full stacks base + compiler axis contribute their env / package_family - # / overrides / append_arch / system. Reset disable to the original CLI set so - # a provider-supplied disable is picked up without double-counting config-dir - # ones. - args.disable = list(_cli_disable) - (err, overrides, taps, defaultsMeta) = parseDefaults(args.disable, - defaultsReader, debug, args.architecture, args.configDir) - dieOnError(err, err) - if _pin_sig == _prev_pin_sig: - break - _prev_pin_sig = _pin_sig - else: - warning("Provider discovery did not reach a fixed point after %d passes; " - "using the last resolved provider set (some provider pins may still " - "be moving).", MAX_PROVIDER_ITERATIONS) + overrides, taps, defaultsMeta, provider_dirs, _fetched_dirs = providers_and_defaults( + args, packages, workDir, defaultsReader, _cli_disable, overrides, taps, defaultsMeta) announce_providers(_fetched_dirs) # A defaults name with no defaults-.sh anywhere on the (now complete) # search path is a typo or a missing repository, not a flavour: say so, rather @@ -3222,6 +3527,13 @@ def defaultsReader(): args.reuseOverlay = None args.reuseBuildId = None args.reuseCvmfsBase = None + if getattr(args, "symlinkPackagesFrom", None): + args.symlinkPackagesFrom = os.path.abspath(os.path.expanduser(args.symlinkPackagesFrom)) + dieOnError(not os.path.isdir(args.symlinkPackagesFrom), + "--symlink-packages-from is not a directory: %s" % + args.symlinkPackagesFrom) + info("Symlink packages: %s (%s policy)", args.symlinkPackagesFrom, + getattr(args, "symlinkPolicy", "strict")) if args.reuseFrom: info("Reuse-from modules: %s", args.reuseFrom) _install_base = _cvmfs.get("install_path") if _cvmfs else None @@ -3307,6 +3619,14 @@ def _system_opt(key, top_default): # A recipe's own sandbox_network field still overrides this per package # (handled in sandbox.wrap_build_command). YAML parses bare on/off as bools, # so normalise to the "on"/"off" strings the sandbox layer expects. + # The sandbox, when --sandbox is not given: $BITS_SANDBOX, else the defaults' + # `sandbox:` (on/off), turns on sandbox-exec for a macOS build outside Docker. + if getattr(args, "sandbox", None) is None: + from bits_helpers.sandbox import default_mode + args.sandbox = default_mode(os.environ.get("BITS_SANDBOX"), _system_opt("sandbox", None), + bool(getattr(args, "docker", False))) + # The cache of sources builds download themselves (issue #129), if any. + args.fetchCache = fetch_cache.cache_name(_system_opt("fetch_cache", None)) if getattr(args, "sandboxNetwork", None) is None: _dn = _system_opt("sandbox_network", "on") if isinstance(_dn, bool): @@ -3405,17 +3725,20 @@ def _truthy(v): or os.environ.get("BITS_S3_ENDPOINT_URL") or os.environ.get("S3_ENDPOINT_URL") or os.environ.get("AWS_ENDPOINT_URL_S3") or os.environ.get("AWS_ENDPOINT_URL")) _prefix = _system_opt("trust_manifest_prefix", "MANIFESTS/common-manifest") - _store = getattr(args, "remoteStore", "") - # Prefer the manifests actually in the store (arch-independent hashes) over - # two names guessed from our own arch; fall back to name construction. - _srcs = _list_store_manifest_srcs(_store, _prefix, _ep, - getattr(args, "workDir", None)) - if not _srcs: - _srcs = derive_trust_manifest_srcs( - _store, _prefix, str(getattr(args, "architecture", "") or ""), _ep) - args._guessedTrustManifests = set(_srcs) # names guessed, may not exist + # The signed manifests of every read store (a b3:// one at its own endpoint). + from bits_helpers.sync import store_urls, bucket_endpoint + _srcs, _guessed = [], set() + for _store in store_urls(getattr(args, "remoteStore", "")): + _sep = (bucket_endpoint(_store[5:].split("/", 1)[0]) + if _store.startswith("b3://") else None) or _ep + _s, _g = trust_manifest_srcs(_store, _prefix, _sep, getattr(args, "workDir", None), + str(getattr(args, "architecture", "") or "")) + _srcs += [x for x in _s if x not in _srcs] + _guessed |= _g + if _guessed: + args._guessedTrustManifests = _guessed # names guessed, may not exist if _srcs: - args.trustManifest = ",".join(_srcs) + args.trustManifest, args._trustFromStore = ",".join(_srcs), True banner("Signed reuse: %d trust manifest(s) from\n %s/", len(_srcs), os.path.dirname(os.path.commonprefix(_srcs))) debug("--require-signed-reuse: trust manifests -> %s", args.trustManifest) @@ -3462,6 +3785,12 @@ def _truthy(v): dieOnError(True, "--reuse-policy relaxed produces loose-provenance artifacts that cannot be " "published. Drop --write-store, or rebuild with --reuse-policy strict.") + if (getattr(args, "symlinkPackagesFrom", None) + and getattr(args, "symlinkPolicy", "strict") == "relaxed" + and getattr(args, "writeStore", "")): + dieOnError(True, + "--symlink-policy relaxed produces loose-provenance artifacts that cannot be " + "published. Drop --write-store, or use --symlink-policy strict.") # Snapshot the resolved build knobs now that the resolution phase above has # settled them, so downstream reads use typed cfg. instead of repeating @@ -3477,6 +3806,17 @@ def _truthy(v): s3_secret_key=getattr(args, "s3SecretKey", None), s3_region=getattr(args, "s3Region", None), s3_addressing_style=getattr(args, "s3AddressingStyle", None)) + # Read-only stores that cannot be reached (an offline laptop): build without them + # rather than retry every request. With a write store nothing changes. + from bits_helpers.sync import drop_unreachable, NoRemoteSync + syncHelper, _down = drop_unreachable(syncHelper, getattr(args, "insecure", False)) + for _url, _why in _down: + warning("The store %s cannot be reached (%s): building without it " + "(--no-remote-store does so without trying).", _url, _why) + if _down and isinstance(syncHelper, NoRemoteSync): + args.remoteStore = "" + if getattr(args, "_trustFromStore", False): + args.trustManifest = None # derived from that store: nothing to verify # If the bits workdir contains a .sl directory (or .git/sl for git repos # with Sapling enabled), we use Sapling as SCM. Otherwise, we default to git @@ -3560,6 +3900,7 @@ def performPreferCheckWithTempDir(pkg, cmd): return getstatusoutput_docker(cmd, cwd=temp_dir) _satisfied_reqs = set() # system_requirement packages found present (SBOM) + _system_info = {} # their versions and references, as their checks said systemPackages, ownPackages, failed, validDefaults = \ getPackageList(packages = packages, specs = specs, @@ -3578,7 +3919,8 @@ def performPreferCheckWithTempDir(pkg, cmd): log = debug, provider_dirs = provider_dirs, defaults_meta = defaultsMeta, - satisfied_requirements = _satisfied_reqs) + satisfied_requirements = _satisfied_reqs, + system_info = _system_info) # Read the container fingerprint (only present in a bits-containers image), so # own_hash packages can fold the build environment — bison/flex/glibc/binutils @@ -3642,7 +3984,8 @@ def performPreferCheckWithTempDir(pkg, cmd): _u = x.get("unfiltered_requires") or {} _edges[n] = ([r for r in _u.get("runtime") or [] if r in _sys], [r for r in _u.get("build") or [] if r in _sys]) - args.manifest.set_system_packages(_sys, _edges) + args.manifest.set_system_packages(_sys, _edges, + {n: i for n, i in _system_info.items() if n in _sys}) for x in specs.values(): x["requires"] = [r for r in x["requires"] if r not in args.disable] x["build_requires"] = [r for r in x["build_requires"] if r not in args.disable] @@ -3705,7 +4048,8 @@ def _display_ref(pkg): spec = specs[pkg] return resolve_spec_data(spec, str(spec.get("tag", spec.get("version", "?"))), args.defaults, strict=False) - if len(builtPackages) > 1: + if args.plugin != "legacy" and len(builtPackages) > 1: + # The legacy build prints its plan instead (further below). # One row per package as (name/ref, source); source is the recipe origin # "repo@commit" (already includes its own '@'), or the devel marker. Align # the source column so a long build order is easy to scan, e.g.: @@ -3725,19 +4069,9 @@ def _build_row(pkg): banner("Packages will be built in the following order:\n - %s", "\n - ".join("{} [{}]".format(label.ljust(_w), source) for label, source in _rows)) - else: + elif len(builtPackages) <= 1: banner("No dependencies of package %s to build.", buildOrder[-1]) - # Tell the progress reporter how many packages will be built, so it can post - # a percentage to the GitLab commit status as each one starts (no-op unless - # running under gitlab-runner). defaults-release is excluded to match the - # plan shown above. - try: - from bits_helpers import progress as _progress - _progress.set_total(len([x for x in builtPackages if x != "defaults-release"])) - except Exception: - pass - if develPkgs: banner("You have packages in development mode (%s).\n" @@ -3918,12 +4252,46 @@ def _devel(spec): debug("We will build packages in the following order: %s", " ".join(buildOrder)) + # The plan: where each package comes from, decided as the loop will decide it + # (read-only; the store is only listed). Build-only dependencies of reused + # packages are not needed, so not fetched nor built (issue #141). + _plan_args = dict( + targets=[x for x in packages if x in specs] or buildOrder[-1:], + build_targets={buildOrder[-1]} if args.onlyDeps else set(), + source=lambda x: specs[x].get("recipe_source", "?")) if args.dryRun: - from bits_helpers.plan import plan_build - plan_build(buildOrder, specs, args, workDir, syncHelper, raw_architecture, - cfg, lambda: trusted_reuse_index(args, workDir)) + _plan.plan_build(buildOrder, specs, args, workDir, syncHelper, raw_architecture, + cfg, lambda: trusted_reuse_index(args, workDir), **_plan_args) info("--dry-run / -n specified. Not building.") return + _full_order, _skipped, _states = list(buildOrder), set(), {} + try: + import copy + _planned = copy.deepcopy(specs) + _, _plan_states, _need = _plan.decide( + buildOrder, _planned, args, workDir, syncHelper, raw_architecture, cfg, + lambda: trusted_reuse_index(args, workDir), title="Build plan", **_plan_args) + _plan.adopt(specs, _planned, set(buildOrder) - _need) + _skipped, _states = set(buildOrder) - _need, _plan_states + buildOrder = [x for x in buildOrder if x in _need] + except Exception as exc: # pylint: disable=broad-except + warning("Could not plan the build (%s): every package is fetched or built.", exc) + try: + from bits_helpers.cleanup import touch_sentinel as _touch + for d in _skipped: # kept by bits cleanup: an older init.sh may still source them + if _states.get(d) == _plan.INSTALLED: + _touch(workDir, args.architecture, d, ver_rev(specs[d])) + except OSError as exc: + debug("Could not mark the installed packages not needed as used: %s", exc) + # Tell the progress reporter how many packages will be built, so it can post + # a percentage to the GitLab commit status as each one starts (no-op unless + # running under gitlab-runner). defaults-release is excluded, as in the plan. + try: + from bits_helpers import progress as _progress + _progress.set_total(len([x for x in buildOrder if x != "defaults-release"]) + - (1 if args.onlyDeps and len(buildOrder) > 1 else 0)) + except Exception: + pass # We now iterate on all the packages, making sure we build correctly every # single one of them. This is done this way so that the second time we run we @@ -4077,22 +4445,19 @@ def _devel(spec): _prefetch_workers = cfg.prefetch_workers if _prefetch_workers < 0: _prefetch_workers = min(max(int(getattr(args, "builders", 1)), 1), 4) - _prefetch_executor = None - if _prefetch_workers > 0 and buildOrder and not isinstance(syncHelper, + _prefetch_executor, _prefetch_futures = None, [] + _prefetch_order = prefetch_wanted(buildOrder, _states) + if _prefetch_workers > 0 and _prefetch_order and not isinstance(syncHelper, __import__("bits_helpers.sync", fromlist=["NoRemoteSync"]).NoRemoteSync): debug("Starting %d prefetch worker(s)", _prefetch_workers) _prefetch_executor = concurrent.futures.ThreadPoolExecutor( max_workers=_prefetch_workers, thread_name_prefix="bits-prefetch", ) - for _pkg in buildOrder: - _pspec = specs[_pkg] - _prefetch_executor.submit(_prefetch_package, _pspec, syncHelper, workDir, args.architecture, - raw_architecture) - # Do NOT call executor.shutdown() here — we let it run in the background - # and join lazily via a daemon-thread finaliser registered below. - import atexit - atexit.register(lambda ex=_prefetch_executor: ex.shutdown(wait=False, cancel_futures=True)) + _prefetch_futures = [ + _prefetch_executor.submit(_prefetch_package, specs[_pkg], syncHelper, workDir, + args.architecture, raw_architecture, sources=_src) + for _pkg, _src in _prefetch_order] # Default build_family for the final banner, matching the in-loop formula # (build.py sets it per-package below). Needed so a reused/skipped MAIN package @@ -4107,55 +4472,54 @@ def _devel(spec): develPackageBranch=develPackageBranch, defaultsMeta=defaultsMeta, buildTargets=buildTargets, packages=packages, prefetch_executor=_prefetch_executor, cmake_prefix_env=_cmake_prefix_env, - specs_for_checksum_phase=specs_for_checksum_phase, monitoredDirs=monitoredDirs) - while buildOrder: - build_one_package(buildOrder.pop(0), ctx) - buildWorkDir = ctx.build_work_dir + specs_for_checksum_phase=specs_for_checksum_phase, monitoredDirs=monitoredDirs, + planned=_states, skipped=_skipped, build_order=buildOrder, full_order=_full_order) + set_package_progress(specs, buildOrder) + try: + while buildOrder: + build_one_package(buildOrder.pop(0), ctx) + buildWorkDir = ctx.build_work_dir - if (args.builders > 1) and buildTargets: - _run_t0 = time.monotonic() - try: - scheduler.run() - finally: - # Always stop the straggler-renice watchdog, even if run() raised. - if getattr(scheduler, "renice_watchdog", None) is not None: - scheduler.renice_watchdog.stop() - _run_wall = time.monotonic() - _run_t0 - # Refresh the self-tuning resource-stats file from this run's monitor traces - # so the next --builders invocation can schedule with up-to-date estimates (P3). - # Also estimate CPU utilisation and, when there is headroom, record/print a - # suggestion for --builders / --oversubscribe. - _tuning = None - if args.resourceMonitoring and monitoredDirs: + if (args.builders > 1) and buildTargets: + _run_t0 = time.monotonic() try: - from bits_helpers.build_stats import aggregate_and_write, tuning_report, default_stats_path - _tuning = tuning_report(monitoredDirs, _run_wall, args.builders, args.jobs, - cfg.oversubscribe) - aggregate_and_write(workDir, monitoredDirs, tuning=_tuning, arch=args.architecture) - except Exception as exc: # pylint: disable=broad-except - warning("Could not update build resource stats: %s", exc) - for (action, error) in scheduler.errors.items(): - info("* The action \"{}\" was not completed successfully because {}".format(action, error)) - # Write a concise failure summary plus a combined full error log, and tell - # the user where to find them and the individual per-package logs. - _summary_path, _full_path = write_failure_summary(workDir, scheduler, args.architecture) - if _summary_path or _full_path: - info("=" * 70) - info("Build finished with errors. Where to look:") - if _summary_path: - info(" Summary (start here): %s", _summary_path) - if _full_path: - info(" Full error log: %s", _full_path) - info(" Per-package build logs: %s/BUILD/-latest/log", workDir) - info("=" * 70) - # End-of-run resource-tuning hint. Only on a clean build — the utilisation - # numbers are meaningless for a partial/failed run. The full report is in - # bits_build_stats.json under "tuning". - if _tuning and _tuning.get("headroom") and not scheduler.brokenJobs: - banner("Resource tuning (recorded in %s):\n %s", - default_stats_path(workDir, args.architecture), _tuning["recommendation"]) - if scheduler.brokenJobs: - dieOnError(True, "Please fix the above errors.") + scheduler.run() + finally: + # Always stop the straggler-renice watchdog, even if run() raised. + if getattr(scheduler, "renice_watchdog", None) is not None: + scheduler.renice_watchdog.stop() + _run_wall = time.monotonic() - _run_t0 + # Refresh the self-tuning resource-stats file from this run's monitor traces + # so the next --builders invocation can schedule with up-to-date estimates (P3). + # Also estimate CPU utilisation and, when there is headroom, record/print a + # suggestion for --builders / --oversubscribe. + _tuning = None + if args.resourceMonitoring and monitoredDirs: + try: + from bits_helpers.build_stats import aggregate_and_write, tuning_report, default_stats_path + _tuning = tuning_report(monitoredDirs, _run_wall, args.builders, args.jobs, + cfg.oversubscribe) + aggregate_and_write(workDir, monitoredDirs, tuning=_tuning, arch=args.architecture) + except Exception as exc: # pylint: disable=broad-except + warning("Could not update build resource stats: %s", exc) + for (action, error) in scheduler.errors.items(): + info("* The action \"{}\" was not completed successfully because {}".format(action, error)) + # Write a concise failure summary plus a combined full error log, and tell + # the user where to find them and the individual per-package logs. + report_failure_summary(workDir, scheduler, args.architecture) + # End-of-run resource-tuning hint. Only on a clean build — the utilisation + # numbers are meaningless for a partial/failed run. The full report is in + # bits_build_stats.json under "tuning". + if _tuning and _tuning.get("headroom") and not scheduler.brokenJobs: + banner("Resource tuning (recorded in %s):\n %s", + default_stats_path(workDir, args.architecture), _tuning["recommendation"]) + if scheduler.brokenJobs: + dieOnError(True, "Please fix the above errors.") + finally: + # The queued prefetches nobody needs any more: at exit Python would wait + # for every one (a pool worker drains its queue before it stops). + for _f in _prefetch_futures: + _f.cancel() # ── Post-build checksum phase ────────────────────────────────────────────── # Runs after all packages have been built (or confirmed up-to-date) so that @@ -4215,7 +4579,8 @@ def _devel(spec): f"\n\n {abspath(join(args.workDir, args.architecture))}\n" f"{_install_root_line}\n" "You can use this package by loading the environment:" - f"\n\n bits {_arch_flag}enter {mainPackage}/latest-{mainBuildFamily}" + f"\n\n bits {_arch_flag}enter {enter_target(mainPackage, _main_spec, mainBuildFamily)}" + f"{latest_alias_note(mainPackage, _main_spec, mainBuildFamily)}" f"{_arch_note}{_container_note}", ) else: @@ -4236,6 +4601,8 @@ def _devel(spec): if getattr(args, "manifest", None) is not None: args.manifest.complete() banner("Build manifest written to:\n %s", args.manifest.path) + _report_reuse(args, mainPackage, _mon_url if _mon_on else None, + lambda: cfg.monitor_instance or _system_opt("monitor_instance", None)) if getattr(args, "signManifest", None): from bits_helpers import trust sig = trust.sign_manifest(args.manifest.path, args.signManifest) @@ -4260,16 +4627,14 @@ def _devel(spec): # a conditional branch and may never have been assigned on this path. _store_mon_url = (cfg.monitor_url or os.environ.get("METRICS_URL") or "").strip().rstrip("/") - if _store_mon_url and getattr(syncHelper, "writeStore", "") and getattr(syncHelper, "s3", None): + # The first b3:// write store's (the gauges carry no bucket label). + _w = next((w for w in (getattr(syncHelper, "writers", None) or [syncHelper]) + if getattr(w, "writeStore", "") and getattr(w, "s3", None)), None) + if _store_mon_url and _w is not None: from bits_helpers import store_stats as _ss - _bucket = getattr(syncHelper, "remoteStore", "") or getattr(syncHelper, "writeStore", "") - # Endpoint precedence matches the S3 client / the require-signed-reuse derive. - _store_ep = (getattr(args, "s3Endpoint", None) - or os.environ.get("BITS_S3_ENDPOINT_URL") or os.environ.get("S3_ENDPOINT_URL") - or os.environ.get("AWS_ENDPOINT_URL_S3") or os.environ.get("AWS_ENDPOINT_URL")) - _ss.push_store_gauges(syncHelper.s3, _bucket, _store_mon_url, - work_dir=args.workDir, - store=getattr(args, "remoteStore", "") or _bucket, - endpoint=_store_ep) + from bits_helpers.sync import s3_settings + _bucket = getattr(_w, "remoteStore", "") or getattr(_w, "writeStore", "") + _ss.push_store_gauges(_w.s3, _bucket, _store_mon_url, work_dir=args.workDir, + store="b3://" + _bucket, endpoint=s3_settings(_bucket)["endpoint"]) debug("Everything done") diff --git a/bits_helpers/build_config.py b/bits_helpers/build_config.py index 8ed065ba..22b18859 100644 --- a/bits_helpers/build_config.py +++ b/bits_helpers/build_config.py @@ -34,10 +34,13 @@ class BuildConfig: parallel_downloads: int # raw --parallel-downloads (site clamps to >= 1) parallel_sources: int # concurrent source fetches prefetch_workers: int # tarball prefetch workers, -1 = auto + fetch_build_requires: bool # also fetch/build build_requires of reused packages # Reuse / store consumers (resolved during import; read later in the loop). reuse_overlay: Optional[str] # host path of the imported reuse overlay reuse_cvmfs_base: Optional[str] # CVMFS Packages base for the reuse overlay + symlink_packages_from: Optional[str] # install root supplying symlinked payloads + symlink_policy: str # independent strict/relaxed package selection reuse_beacon: Optional[str] # reuse-beacon URL (falls back to env at the site) store_integrity: bool # verify store-integrity ledger on cached reuse @@ -74,8 +77,11 @@ def from_args(cls, args): parallel_downloads=getattr(args, "parallelDownloads", 2), parallel_sources=getattr(args, "parallelSources", 1), prefetch_workers=getattr(args, "prefetchWorkers", -1), + fetch_build_requires=getattr(args, "fetchBuildRequires", False), reuse_overlay=getattr(args, "reuseOverlay", None), reuse_cvmfs_base=getattr(args, "reuseCvmfsBase", None), + symlink_packages_from=getattr(args, "symlinkPackagesFrom", None), + symlink_policy=getattr(args, "symlinkPolicy", "strict"), reuse_beacon=getattr(args, "reuseBeacon", None), store_integrity=getattr(args, "storeIntegrity", False), monitor=getattr(args, "monitor", None), diff --git a/bits_helpers/build_epoch.py b/bits_helpers/build_epoch.py index 592087b9..b3e9aa26 100644 --- a/bits_helpers/build_epoch.py +++ b/bits_helpers/build_epoch.py @@ -27,6 +27,8 @@ "bits_helpers/build_template.sh", "bits_helpers/initdotsh.py", "bits_helpers/relativize-configs.sh", + "bits_helpers/relocate-references.sh", + "bits_helpers/reuse_symlinks.py", "bits_helpers/relocate-me.sh", "bits_helpers/view-list.sh", ) @@ -35,7 +37,12 @@ # the container settings. A change there that alters builds bumps BUILD_EPOCH # by review, not by this check. -COVERED_SHA256 = "44af202488de34288811754d2eefc5beb5c1cd805b6afb4dccd79386846a5f3c" +# Progress markers only change console output; package contents and BUILD_EPOCH are unchanged. +# --symlink-packages-from is opt-in; leave the epoch unchanged to preserve strict reuse of +# existing deployments. The staging and post-install helpers are covered here. +# grep -r: symlinks in a package are no longer listed for relocation; existing +# tarballs stay valid, so BUILD_EPOCH is unchanged. +COVERED_SHA256 = "9e75269258e86a7a0cff92018adebc9daef8946b85ce849c43e7c71ab7196d00" def covered_sha256(root=None): diff --git a/bits_helpers/build_progress.py b/bits_helpers/build_progress.py new file mode 100644 index 00000000..c3858a3c --- /dev/null +++ b/bits_helpers/build_progress.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Human-readable package and stage progress, independent of CI reporting. + +Packages are numbered along the selected plan (reused ones included, +defaults-release not), after selections such as --only-deps; parallel builds +report out of that order. A serial build on a terminal shows a spinner with the +compiler's percentages; parallel builds print labeled lines, no spinners. + +The build script delimits its phases with `bits:stage:` lines (STAGES). +Within a recipe, stages come from Bash's trace of the commands it runs +(_command_stage), never from text that mentions make or install, nor from the +traced echo of a marker. A custom tool can print bits:stage:configure, compile or +install itself. Post-install hooks, packaging and relocation share "Installing". +""" + +import logging +import os.path +import re +import shlex +import sys + +from bits_helpers.log import ProgressPrint, debug, info, logger + + +# Explicit markers delimit recipe execution. Within a recipe, Bash's existing +# command trace identifies common build tools without changing recipe execution. +STAGE_PREFIX = "bits:stage:" +STAGES = { + "prepare": "Preparing build environment", + "recipe": "Running build recipe", + "incremental": "Running incremental build recipe", + "configure": "Configuring", + "compile": "Compiling", + "install": "Installing", + "unpack": "Unpacking cached archive", + "post-install": "Installing", + "relocate-config": "Installing", + "package": "Installing", + "relocate": "Installing", +} + + +def _command_stage(line): + """Identify an executed build command, never text echoed by a recipe.""" + if not re.match(r"^\++ ", line): + return None + try: + words = shlex.split(line.lstrip("+")) + except ValueError: + return None # A multiline traced argument is not a complete command. + while words and (re.match(r"^[A-Za-z_][A-Za-z_0-9]*=", words[0]) + or words[0] in ("command", "exec", "env", "ccache", "sccache")): + words.pop(0) + if not words: + return None + command, args = os.path.basename(words[0]), words[1:] + if command in ("configure", "Configure"): + return "configure" + if command == "Make": + return "compile" + if command in ("MakeInstall", "install"): + return "install" + if command == "cmake": + if "--build" in args: + if "--target" in args and "install" in args[args.index("--target") + 1:]: + return "install" + return "compile" + if "--install" in args: + return "install" + if any(a in ("-E", "-P", "--version", "--help", "--find-package") for a in args): + return None + return "configure" + if command == "meson" and args: + return {"setup": "configure", "configure": "configure", + "compile": "compile", "install": "install"}.get(args[0]) + if command in ("make", "gmake", "ninja", "ninja-build"): + if any(a in ("--version", "--help", "-t", "-n", "--just-print", "--dry-run") for a in args): + return None + targets = [] + skip = False + for index, arg in enumerate(args): + if skip: + skip = False + elif arg in ("-C", "-f", "--directory", "--file", "--makefile"): + skip = True + elif arg in ("-j", "-l"): + skip = index + 1 < len(args) and bool(re.fullmatch(r"\d+(?:\.\d+)?", args[index + 1])) + elif not arg.startswith("-") and "=" not in arg: + targets.append(arg) + if any(t == "install" or t.startswith(("install-", "install/", "install_")) for t in targets): + return "install" + if targets and all(t in ("clean", "distclean", "maintainer-clean", "help") for t in targets): + return None + return "compile" + if command in ("sh", "bash", "perl") and args and os.path.basename(args[0]) in ("configure", "Configure"): + return "configure" + if re.fullmatch(r"python(?:[23](?:\.\d+)*)?", command): + if args[:2] == ["-m", "pip"]: + args = args[2:] + command = "pip" + elif args[:2] == ["-m", "build"]: + return "compile" + elif args and os.path.basename(args[0]) == "setup.py": + if "install" in args: + return "install" + if any(a in ("build", "build_ext", "bdist_wheel") for a in args): + return "compile" + if re.fullmatch(r"pip(?:[23](?:\.\d+)*)?", command) and args: + return {"install": "install", "wheel": "compile"}.get(args[0]) + return None + + +def set_package_progress(specs, packages): + """Number the selected plan, including reused packages, excluding defaults.""" + planned = [p for p in packages if p != "defaults-release"] + for index, package in enumerate(planned, 1): + specs[package]["_build_progress"] = (index, len(planned)) + + +STEP = "==> " # every package line starts so, with a spinner or without + + +def package_info(fmt, *args): + """A package's progress line, aligned with the stage lines.""" + info(STEP + fmt, *args) + + +def package_label(spec, version=None): + label = "{}@{}".format(spec["package"], spec["version"] if version is None else version) + if "_build_progress" in spec: + label = "[{}/{}] {}".format(*spec["_build_progress"], label) + return label + + +class BuildProgress: + """Show stage transitions in logs and retain compiler percentages on a TTY.""" + + def __init__(self, label, stage, emit=None, interactive=True): + self.label = label + self.emit = emit or info + self.interactive = interactive + self.display = None + self.stage = None + self.in_recipe = False + self.update(stage) + + def update(self, stage, interactive=True): + if stage == self.stage: + return + if self.display is not None: + self.display.end("done") + self.display = None + self.stage = stage + message = "{}: {}".format(self.label, stage) + if interactive and self.interactive and sys.stdout.isatty() and not logger.isEnabledFor(logging.DEBUG): + self.display = ProgressPrint(message) + self.display("%s", "") + else: + self.emit(STEP + message) + + def __call__(self, text, *args): + line = text % args if args else text + if line.startswith(STAGE_PREFIX) and line[len(STAGE_PREFIX):] in STAGES: + marker = line[len(STAGE_PREFIX):] + self.in_recipe = marker in ("recipe", "incremental", "configure", "compile", "install") + self.update(STAGES[marker]) + return + stage = _command_stage(line) if self.in_recipe else None + if stage: + self.update(STAGES[stage]) + if self.display is not None: + self.display("%s", line) + else: + debug("%s", line) + + def end(self, error=False): + if self.display is not None: + self.display.end("failed" if error else "done", error) + self.display = None + else: + self.emit("{}{}: {}".format(STEP, self.label, "failed" if error else "done")) diff --git a/bits_helpers/build_template.sh b/bits_helpers/build_template.sh index 6156f56f..4621756b 100644 --- a/bits_helpers/build_template.sh +++ b/bits_helpers/build_template.sh @@ -3,6 +3,7 @@ BITS_START_TIMESTAMP=$(date +%%s) # Automatically generated build script unset DYLD_LIBRARY_PATH echo "bits: start building $PKGNAME-$PKGVERSION-$PKGREVISION at $BITS_START_TIMESTAMP" +echo "bits:stage:prepare" get_file_from_configDir() { # B2 FIX: quote $BITS_CONFIG_DIR in dirname/basename and quote ${repo_dir}/${d} # in the test so that paths with spaces are not word-split. @@ -50,9 +51,14 @@ cleanup() { trap cleanup EXIT -# Cleanup variables which should not be exposed to user code +# Cleanup variables which should not be exposed to user code: every S3 secret, +# the run's and each bucket's (BITS_S3__*). unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY +for _bits_v in $(compgen -e | grep -E '^(AWS_SESSION_TOKEN|BITS_AWS_(ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN)|BITS_S3_.+_(ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN))$'); do + unset "$_bits_v" +done +unset _bits_v set -e set +h @@ -155,6 +161,8 @@ fi export BUILDROOT="$BITS_BUILD_WORK_DIR/BUILD/$PKGHASH" export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/$COMMIT_HASH" export BUILDDIR="$BUILDROOT/$PKGNAME" +# fetch_cache: true: the directory bits restored the recipe's downloads into. +if [ -n "${BITS_FETCH_CACHE:-}" ]; then export BITS_FETCH_CACHE="$WORK_DIR/$BITS_FETCH_CACHE"; fi # All caching for RECC should happen relative to $WORK_DIR export RECC_PROJECT_ROOT=$WORK_DIR @@ -244,9 +252,11 @@ source_up EOF cd "$BUILDROOT" -ln -snf "$PKGHASH" "$BITS_BUILD_WORK_DIR/BUILD/$PKGNAME-latest" +# The "latest" links with /bin/ln: GNU ln -f replaces a link through a temporary +# name beside it, which the macOS sandbox (sandbox.py) does not allow. +/bin/ln -snf "$PKGHASH" "$BITS_BUILD_WORK_DIR/BUILD/$PKGNAME-latest" if [[ $DEVEL_PREFIX ]]; then - ln -snf "$PKGHASH" "$BITS_BUILD_WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX" + /bin/ln -snf "$PKGHASH" "$BITS_BUILD_WORK_DIR/BUILD/$PKGNAME-latest-$DEVEL_PREFIX" fi cd "$BUILDDIR" @@ -271,6 +281,7 @@ function Run() { # dummy function } if [[ "$CACHED_TARBALL" == "" && ! -f $BUILDROOT/log ]]; then + echo "bits:stage:recipe" set -o pipefail; # Keep DYLD_LIBRARY_PATH (set from dependencies by init.sh above) so build-time # tools on macOS find their dependencies' dylibs, mirroring LD_LIBRARY_PATH on @@ -282,10 +293,12 @@ if [[ "$CACHED_TARBALL" == "" && ! -f $BUILDROOT/log ]]; then } 2>&1 | tee "$BUILDROOT/log" rc=${PIPESTATUS[0]}; [ "$rc" -eq 0 ] || exit "$rc" # read PIPESTATUS[0] BEFORE any command clobbers it elif [[ "$CACHED_TARBALL" == "" && $INCREMENTAL_BUILD_HASH != "0" && -f "$BUILDDIR/.build_succeeded" ]]; then + echo "bits:stage:incremental" set -o pipefail (%(incremental_recipe)s) 2>&1 | tee "$BUILDROOT/log" rc=${PIPESTATUS[0]}; [ "$rc" -eq 0 ] || exit "$rc" # propagate the real recipe exit code (was masked to 1) elif [[ "$CACHED_TARBALL" == "" ]]; then + echo "bits:stage:recipe" set -o pipefail; # Keep DYLD_LIBRARY_PATH (from dependencies via init.sh) — see note above. { set -e @@ -295,6 +308,7 @@ elif [[ "$CACHED_TARBALL" == "" ]]; then } 2>&1 | tee "$BUILDROOT/log" rc=${PIPESTATUS[0]}; [ "$rc" -eq 0 ] || exit "$rc" # read PIPESTATUS[0] BEFORE any command clobbers it else + echo "bits:stage:unpack" # Unpack the cached tarball in the $INSTALLROOT and remove the unrelocated # files. rm -rf "$BUILDROOT/log" @@ -316,6 +330,15 @@ else mv "$WORK_DIR/TMP/$PKGHASH/$PKGPATH" "$INSTALLROOT" pushd "$WORK_DIR/INSTALLROOT/$PKGHASH" if [ -w "$INSTALLROOT" ]; then + # Relocate with this bits' relocate-me.sh, not the one in the tarball: that + # is from the bits that packed it, and an older one doubles INSTALLROOT/ + # in the paths when, as here, the target lies under the work dir. The + # package's own lines (modulefile, macOS) are written again below and run + # when the installed copy is relocated. (A tarball too old to have + # .bits-pkginfo keeps its own script: this one needs that file.) + if [ -f "$INSTALLROOT/etc/profile.d/.bits-pkginfo" ]; then + install "${BITS_SCRIPT_DIR}/bits_helpers/relocate-me.sh" "$INSTALLROOT/" + fi WORK_DIR=$WORK_DIR /bin/bash -ex "$INSTALLROOT/relocate-me.sh" fi popd @@ -323,6 +346,15 @@ else rm -rf "$WORK_DIR/TMP/$PKGHASH" fi +# What the build left in its fetch cache, by sha256, for bits to save (a failure only skips that). +if [ -z "$CACHED_TARBALL" ] && [ -n "${BITS_FETCH_CACHE:-}" ] && [ -d "$BITS_FETCH_CACHE" ]; then + _bits_sha=sha256sum; command -v sha256sum >/dev/null || _bits_sha="shasum -a 256" + (cd "$BITS_FETCH_CACHE" && find . -type f -exec $_bits_sha {} + | LC_ALL=C sort -k2) \ + > "$BUILDROOT/fetched-sources.sha256" || rm -f "$BUILDROOT/fetched-sources.sha256" + unset _bits_sha +fi + +echo "bits:stage:relocate-config" # Regenerate init.sh, in case the package build clobbered it. This # particularly happens in the AliEn-Runtime package, since it copies other # packages into its installroot wholesale. @@ -428,9 +460,11 @@ cd "$WORK_DIR/INSTALLROOT/$PKGHASH" # Run post-install hooks if [[ $PKGNAME != defaults-* ]]; then + echo "bits:stage:post-install" run_hooks "POST_INSTALL" fi +echo "bits:stage:relocate-config" # Relativise in-tree absolute symlinks so the artefact is relocatable. # A build can install an absolute symlink into its own INSTALLROOT (e.g. bzip2's # bin/bzless -> $INSTALLROOT/bin/bzmore). That target is a build-time path: it @@ -464,12 +498,20 @@ unset _pack_root # ${pcfiledir}, .cmake on ${CMAKE_CURRENT_LIST_DIR}; the grep guard leaves # already-relative configs untouched. if [ -w "$INSTALLROOT" ]; then + # Like CMS relocateReference: linked dependencies can emit their original + # prefixes into a NEW package's text files. Rewrite those local files before + # collecting .bits-relocate, so subsequent relocation sees WORK_DIR paths. + # The helper uses find/xargs/grep/perl and never follows symlink directories. + if [ -n "${BITS_REUSE_RELOCATIONS_FILE:-}" ]; then + bash "${BITS_SCRIPT_DIR}/bits_helpers/relocate-references.sh" \ + "$INSTALLROOT" --file "$BITS_REUSE_RELOCATIONS_FILE" + fi bash "${BITS_SCRIPT_DIR}/bits_helpers/relativize-configs.sh" "$INSTALLROOT" # Find which files need relocation — only now, after every tree mutation, so # configs made relative above drop out and files written by POST_INSTALL hooks # are included. ( cd "$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH" && \ - { grep -I -H -l -R "\($WORK_DIR\|[@][@]PKGREVISION[@]$PKGHASH[@][@]\)" . || true; } \ + { grep -I -H -l -r "\($WORK_DIR\|[@][@]PKGREVISION[@]$PKGHASH[@][@]\)" . || true; } \ | sed -e 's|^\./||' > "$INSTALLROOT/etc/profile.d/.bits-relocate" ) # The package's file list for release merged views (.bits-view.json). # Not fatal: without it a view lists the package from its tarball. @@ -477,6 +519,7 @@ if [ -w "$INSTALLROOT" ]; then || echo "bits: WARNING: could not write $PKGNAME's .bits-view.json" >&2 fi +echo "bits:stage:package" # Archive creation # B7 FIX: replace backtick with $(...) and quote $PKGHASH; use -c (chars) consistently. HASHPREFIX=$(echo "$PKGHASH" | cut -c1,2) @@ -551,6 +594,7 @@ elif [ -z "$CACHED_TARBALL" ]; then fi wait "$rsync_pid" +echo "bits:stage:relocate" # We've copied files into their final place; now relocate. # Use $PKGPATH (= $EFFECTIVE_ARCHITECTURE[/$PKGFAMILY]/$PKGNAME/$_VERREV) so # that PKGFAMILY packages (e.g. externals/foo, cms/bar) are found correctly. @@ -562,11 +606,11 @@ fi # Last package built gets a "latest" mark. # dirname of $PKGPATH = $EFFECTIVE_ARCHITECTURE[/$PKGFAMILY]/$PKGNAME # B4 FIX: quote ${_VERREV} and $(dirname ...) so spaces in PKGPATH don't split. -ln -snf "${_VERREV}" "$(dirname "$PKGPATH")/latest" +/bin/ln -snf "${_VERREV}" "$(dirname "$PKGPATH")/latest" # Latest package built for a given devel prefix gets latest-$BUILD_FAMILY if [[ $BUILD_FAMILY ]]; then - ln -snf "${_VERREV}" "$(dirname "$PKGPATH")/latest-${BUILD_FAMILY}" + /bin/ln -snf "${_VERREV}" "$(dirname "$PKGPATH")/latest-${BUILD_FAMILY}" fi # When the package is definitely fully installed, install the file that marks diff --git a/bits_helpers/certify.py b/bits_helpers/certify.py index 6621997a..ea7d6c11 100644 --- a/bits_helpers/certify.py +++ b/bits_helpers/certify.py @@ -70,11 +70,12 @@ class CertifyConflict(CertifyError): """Two builds disagree on the tarball_sha256 for one content hash.""" -def _now_iso() -> str: +def now_iso() -> str: + """Now, in UTC, as ISO 8601 to the second (…Z).""" return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def _norm_sha(value) -> str: +def norm_sha(value) -> str: """Normalise 'sha256:HEX' / 'HEX' to a bare lowercase hex string.""" s = str(value or "").strip().lower() return s.split(":", 1)[1] if ":" in s else s @@ -178,7 +179,7 @@ def _settle_conflict(key, cands, probe, stale): arch, h = key stored = [probe(arch, h, e.get("tarball")) for e, _b in cands] if probe is not None else [] confirmed = [c for c, actual in zip(cands, stored) - if actual is not None and _norm_sha(actual) == _norm_sha(c[0].get("tarball_sha256"))] + if actual is not None and norm_sha(actual) == norm_sha(c[0].get("tarball_sha256"))] if len(confirmed) == 1: keep = confirmed[0] stale.extend((c[0].get("package", "?"), h, c[1]) for c in cands if c is not keep) @@ -266,7 +267,12 @@ def merge_common_manifest(manifests, default_group=None, valid_days=None, for _fld in ("bits_version", "bits_dist_hash"): if man.get(_fld) and _fld not in entry: entry[_fld] = man[_fld] - by_key.setdefault((arch, h), {}).setdefault(_norm_sha(sha), (entry, bid)) + slot = by_key.setdefault((arch, h), {}) + have = slot.setdefault(norm_sha(sha), (entry, bid))[0] + if entry.get("built_by") and not have.get("built_by"): + # Who compiled it and when: the build that did, not one that reused it. + have.update({k: entry[k] for k in ("built_by", "completed_at", "bits_version", + "bits_dist_hash") if entry.get(k)}) stale = [] # (package, hash, build_id) claims the store refuted packages = [] for key in sorted(by_key): @@ -287,7 +293,7 @@ def merge_common_manifest(manifests, default_group=None, valid_days=None, common = { "schema_version": SCHEMA_VERSION, "kind": COMMON_MANIFEST_KIND, - "created_at": _now_iso(), + "created_at": now_iso(), "sources": sources, "packages": packages, } @@ -324,7 +330,7 @@ def validate_against_store(common, probe): for p in common.get("packages") or []: h = p.get("hash") arch = p.get("effective_architecture") or "" - claimed = _norm_sha(p.get("tarball_sha256")) + claimed = norm_sha(p.get("tarball_sha256")) actual = probe(arch, h, p.get("tarball")) if actual is None: # The store has no object for this hash. We cannot vouch for it, but @@ -333,13 +339,13 @@ def validate_against_store(common, probe): # upload, a layout migration). So this is DROPPABLE, not fatal: the # caller removes the entry and signs the rest. missing.append(p) - elif _norm_sha(actual) != claimed: + elif norm_sha(actual) != claimed: # The object exists but its bytes differ from what the manifest # claims. This IS the manifest lying about the store — corruption or # tampering — and must never be signed. Fatal. fatal.append( "sha256 mismatch for %s %s: manifest=%s store=%s" - % (p.get("package", "?"), h, claimed, _norm_sha(actual))) + % (p.get("package", "?"), h, claimed, norm_sha(actual))) return fatal, missing @@ -465,7 +471,7 @@ def certify(manifests, key_pem_path, out_path, probe=None, sig_path=None, identity that authorised the certification travels with the signature. """ signer = _make_signer(key_pem_path, sign_proxy, sign_service) - common = _prepare_common(manifests, signer, probe, default_group, + common = _prepare_common(manifests, signer.key_id, probe, default_group, valid_days, source_commit, approval_check) out_abs, sig_path = _write_signed(common, signer, out_path, sig_path) debug("certify: signed common manifest %s (%d pkgs) -> %s", @@ -514,11 +520,14 @@ def _local(p): return out -def _prepare_common(manifests, signer, probe, default_group, valid_days, +def _prepare_common(manifests, key_id, probe, default_group, valid_days, source_commit, approval_check) -> dict: """Merge → (approve) → (store-validate) → key-policy check. Returns the validated common-manifest dict (with certified_by/at stamped), ready to write. Raises :class:`CertifyConflict`/:class:`CertifyError` on any problem. + + *key_id* is a callable giving the signing key's id, for the key-policy + check; None skips that check (a computation that signs nothing). """ probe = _cached_probe(probe) manifests = _drop_local_revisions(load_build_manifests(manifests)) @@ -554,9 +563,9 @@ def _prepare_common(manifests, signer, probe, default_group, valid_days, raise CertifyError("store validation failed:\n " + "\n ".join(fatal)) # Producer-side per-key group binding: refuse to sign groups this key is not # authorised for, so an unauthorised signature is never even produced. - policy = trust.load_key_policy() + policy = trust.load_key_policy() if key_id is not None else None if policy is not None: - kid = signer.key_id() + kid = key_id() bad = sorted({(p.get("group") or "common") for p in common["packages"] if not trust.key_authorized(kid, p.get("group"), policy)}) if bad: @@ -565,7 +574,7 @@ def _prepare_common(manifests, signer, probe, default_group, valid_days, % (kid, ", ".join(bad))) if certified_by: common["certified_by"] = sorted(set(certified_by)) - common["certified_at"] = _now_iso() + common["certified_at"] = now_iso() return common @@ -628,7 +637,29 @@ def certify_by_arch(manifests, key_pem_path, out_path, probe=None, see ``_prepare_common``. Returns ``[(out_path, sig_path, arch), ...]`` sorted by arch. """ + signer = _make_signer(key_pem_path, sign_proxy, sign_service) + outputs = [] + for arch, sub in build_by_arch(manifests, probe=probe, default_group=default_group, + valid_days=valid_days, source_commit=source_commit, + approval_check=approval_check, only_archs=only_archs, + key_id=signer.key_id): + op, sp = _write_signed(sub, signer, _arch_stem(out_path, arch)) + debug("certify: signed %s manifest %s (%d pkgs)", arch, op, len(sub["packages"])) + outputs.append((op, sp, arch)) + return outputs + + +def build_by_arch(manifests, probe=None, default_group=None, valid_days=None, + source_commit=None, approval_check=None, only_archs=None, + key_id=None) -> list: + """The per-architecture common manifests :func:`certify_by_arch` signs, + computed without signing or writing anything: ``[(arch, manifest), ...]`` + sorted by arch. *key_id* (a callable giving the signer's key id) enables the + key-policy check; None skips it. The orchestrator uses this to compute what + a certification would sign and compare it with what was signed. + """ loaded = load_build_manifests(manifests) + only = None if only_archs is not None: only = {a.strip() for a in only_archs if a and a.strip()} kept = [] @@ -639,30 +670,27 @@ def certify_by_arch(manifests, key_pem_path, out_path, probe=None, debug("certify: scoped to %s — %d of %d manifest(s) kept", ", ".join(sorted(only)), len(kept), len(loaded)) loaded = kept - signer = _make_signer(key_pem_path, sign_proxy, sign_service) - common = _prepare_common(loaded, signer, probe, default_group, + common = _prepare_common(loaded, key_id, probe, default_group, valid_days, source_commit, approval_check) buckets = {} for p in common["packages"]: buckets.setdefault(p.get("effective_architecture") or "shared", []).append(p) # Every scoped arch gets a file even when its packages are gone: an empty # signed manifest is the revocation of that platform's entries. - if only_archs is not None: + if only is not None: for arch in only: buckets.setdefault(arch, []) # Always emit at least the shared file so consumers/GC have a well-formed, # signed artifact to read even when a certification is (transiently) empty. if not buckets: buckets = {"shared": []} - outputs = [] + out = [] for arch in sorted(buckets): sub = dict(common) sub["architecture"] = arch sub["packages"] = buckets[arch] - op, sp = _write_signed(sub, signer, _arch_stem(out_path, arch)) - debug("certify: signed %s manifest %s (%d pkgs)", arch, op, len(buckets[arch])) - outputs.append((op, sp, arch)) - return outputs + out.append((arch, sub)) + return out def make_s3_probe(store_url, work_dir, default_arch): @@ -672,35 +700,57 @@ def make_s3_probe(store_url, work_dir, default_arch): digest (computed from the *actual* bytes, so certification cannot be tricked by a manifest that lies about a tarball), or None when the object is absent. """ - import hashlib from bits_helpers.publish import _normalize_s3_store from bits_helpers.sync import remote_from_url - from bits_helpers.utilities import resolve_store_path store = _normalize_s3_store(store_url) writer = remote_from_url(store, store, default_arch, work_dir) bucket = getattr(writer, "remoteStore", None) or getattr(writer, "writeStore", None) - s3 = writer.s3 + return s3_probe(writer.s3, bucket, default_arch) + + +def _s3_absent(exc): + """Whether an S3 error says the object is not there (rather than failing).""" + resp = getattr(exc, "response", None) or {} + code = str((resp.get("Error") or {}).get("Code", "")) + status = (resp.get("ResponseMetadata") or {}).get("HTTPStatusCode") + return code in ("404", "NoSuchKey", "NotFound") or status == 404 + + +def s3_probe(s3, bucket, default_arch, cache=None): + """The probe of :func:`make_s3_probe` over an S3 client and bucket. + + *cache* (optional) remembers digests across runs: ``cache.get(key, etag, + size) -> digest | None`` and ``cache.put(key, etag, size, digest)``. Store + objects are immutable, so an object whose key, ETag and size are unchanged + has the bytes it had; only new objects are streamed and hashed. + """ + import hashlib + from bits_helpers.utilities import resolve_store_path def probe(arch, h, tarball=None): prefix = resolve_store_path(arch or default_arch, h) + "/" - key = None + key = etag = size = None # Prefer the exact object the manifest names, so we validate *those* bytes # and never hash a different tarball that happens to share the hash dir. if tarball: cand = prefix + tarball try: - s3.head_object(Bucket=bucket, Key=cand) - key = cand - except Exception: + head = s3.head_object(Bucket=bucket, Key=cand) + key, etag, size = cand, head.get("ETag"), head.get("ContentLength") + except Exception as exc: + # Only "not there" means absent. Any other error (a timeout, a + # 5xx, throttling) must not drop the package from the signed + # manifest as if it were gone: fail the run instead. + if not _s3_absent(exc): + raise CertifyError("store check failed for %s: %s" % (cand, exc)) key = None if key is None: try: listing = s3.list_objects_v2(Bucket=bucket, Prefix=prefix).get("Contents", []) except Exception as exc: - warning("certify: store list failed for %s (%s)", prefix, exc) - return None - tars = [o["Key"] for o in listing if o["Key"].endswith(".tar.gz")] + raise CertifyError("store list failed for %s: %s" % (prefix, exc)) + tars = [o for o in listing if o["Key"].endswith(".tar.gz")] # Only fall back when there is exactly one tarball: an ambiguous dir # must not be validated against an arbitrarily-chosen object. if len(tars) != 1: @@ -710,7 +760,11 @@ def probe(arch, h, tarball=None): debug("certify: named tarball %s absent under %s (found %d .tar.gz)", tarball, prefix, len(tars)) return None - key = tars[0] + key, etag, size = tars[0]["Key"], tars[0].get("ETag"), tars[0].get("Size") + if cache is not None and etag and size is not None: + known = cache.get(key, etag, size) + if known: + return known digest = hashlib.sha256() body = s3.get_object(Bucket=bucket, Key=key)["Body"] try: @@ -718,7 +772,10 @@ def probe(arch, h, tarball=None): digest.update(chunk) finally: body.close() # release the pooled connection on error too - return "sha256:" + digest.hexdigest() + out = "sha256:" + digest.hexdigest() + if cache is not None and etag and size is not None: + cache.put(key, etag, size, out) + return out return probe @@ -814,6 +871,22 @@ def _check(common): return _check +def _refuse_orchestrated(store_url, work_dir, arch, parser): + """`bits sign` signs what is then put in the store: refuse a store whose + manifests bits-orchestrator signs (see orchestrated.py), or that cannot say + whether it does (the manifests CI would overwrite numbered manifests).""" + from bits_helpers import orchestrated + try: + marker = orchestrated.for_store(store_url, work_dir, arch) + except orchestrated.Unknown as exc: + parser.error("sign: cannot tell whether bits-orchestrator signs this store's " + "manifests: %s (the store's readers need MANIFESTS/orchestrator.json)" % exc) + if marker: + parser.error("sign: %s signs the manifests of %s (since %s): its certified set " + "changes only through it (bits certify, the decertify action)" + % (marker["issuer"], store_url, marker["since"])) + + def doSign(args, parser): """CLI entrypoint for ``bits sign`` (forge-agnostic; the manifests CI wraps this).""" approval_check = None @@ -824,6 +897,7 @@ def doSign(args, parser): sources = [os.path.join(args.workDir, "MANIFESTS")] probe = None if not getattr(args, "noStoreCheck", False): + _refuse_orchestrated(args.certifyStore, args.workDir, args.architecture, parser) probe = make_s3_probe(args.certifyStore, args.workDir, args.architecture) valid_days = getattr(args, "validDays", None) if valid_days is not None and valid_days < 0: diff --git a/bits_helpers/cleanup.py b/bits_helpers/cleanup.py index c8d7d698..7a2d592c 100644 --- a/bits_helpers/cleanup.py +++ b/bits_helpers/cleanup.py @@ -319,7 +319,7 @@ def _try_evict(entry: _SentinelEntry, dry_run: bool) -> Optional[int]: # * sentinels whose install dir is gone. _NON_ARCH_DIRS = {"TARS", "BUILD", "TMP", "INSTALLROOT", "SOURCES", "SPECS", - "MIRROR", "REPOS", "MANIFESTS", "LOGS", "MODULES", + "MIRROR", "REPOS", "MANIFESTS", "LOGS", "MODULES", "FETCH_CACHE", "wrapper-scripts"} _HEX_HASH_RE = None # compiled lazily (re imported locally) @@ -425,11 +425,9 @@ def mark_published_from(work_dir, src): import json path = src if "://" in str(src): - from bits_helpers.download import downloadUrllib2 - dest = join(work_dir, "MANIFESTS", "retention") - os.makedirs(dest, exist_ok=True) - downloadUrllib2(src, dest, work_dir, dest_filename="cvmfs-status.json") - path = join(dest, "cvmfs-status.json") + from bits_helpers.download import fetch_fresh + path = fetch_fresh(src, join(work_dir, "MANIFESTS", "retention"), + "cvmfs-status.json", work_dir) with open(path) as fh: data = json.load(fh) n = 0 diff --git a/bits_helpers/cmd.py b/bits_helpers/cmd.py index 0ff4d4e1..10e9dd1d 100644 --- a/bits_helpers/cmd.py +++ b/bits_helpers/cmd.py @@ -5,6 +5,7 @@ import errno import os import os.path +import re import time from shlex import quote from subprocess import Popen, PIPE, STDOUT, TimeoutExpired @@ -42,9 +43,21 @@ def getoutput(command, timeout=None): return decode_with_fallback(stdout) -def getstatusoutput(command, timeout=None, cwd=None): +# S3 secrets, the run's and each bucket's: never handed to recipe code. +_SECRET = re.compile(r"^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|" + r"BITS_AWS_(ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN)|" + r"BITS_S3_.+_(ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN))$") + + +def secret_free_env(): + """os.environ without the S3 secrets (build_template.sh unsets the same).""" + return {k: v for k, v in os.environ.items() if not _SECRET.match(k)} + + +def getstatusoutput(command, timeout=None, cwd=None, env=None): """Run command and return its return code and output (stdout and stderr).""" - proc = Popen(command, shell=isinstance(command, str), stdout=PIPE, stderr=STDOUT, cwd=cwd) + proc = Popen(command, shell=isinstance(command, str), stdout=PIPE, stderr=STDOUT, cwd=cwd, + env=env) try: merged_output, _ = proc.communicate(timeout=timeout) except TimeoutExpired: @@ -113,8 +126,8 @@ def getstatusoutput_docker(cmd, cwd=None): command_prefix="" if self._extra_env: command_prefix="env " + " ".join(f"{k}={quote(v)}" for (k,v) in self._extra_env.items()) + " " - return getstatusoutput(f"{command_prefix}{BASH} -c {quote(cmd)}" - , cwd=cwd) + return getstatusoutput(f"{command_prefix}{BASH} -c {quote(cmd)}", + cwd=cwd, env=secret_free_env()) # recipe code envOpts = [] for env in self._extra_env.items(): envOpts.append("-e") diff --git a/bits_helpers/compliance.py b/bits_helpers/compliance.py index fe330060..495b190e 100644 --- a/bits_helpers/compliance.py +++ b/bits_helpers/compliance.py @@ -27,6 +27,7 @@ import urllib.request from bits_helpers.log import banner, debug, error, info, warning +from bits_helpers import site # Front-matter keys read from each recipe. Values may be quoted (the header # auto-quoting writes them back quoted); strip one level of quotes. @@ -219,7 +220,7 @@ def scan_specs(specs): def _anonymous_store_access(endpoint, bucket): """True when the bucket answers an UNAUTHENTICATED listing request.""" - url = "%s/%s/?max-keys=1" % (endpoint.rstrip("/"), bucket) + url = "%s/%s/?max-keys=1" % ((endpoint or "https://s3.amazonaws.com").rstrip("/"), bucket) # boto3's default try: with urllib.request.urlopen(url, timeout=15) as resp: return resp.status == 200 and b"ListBucketResult" in resp.read(4096) @@ -240,19 +241,25 @@ def _s3_client(store_url): from botocore import UNSIGNED from botocore.client import Config from bits_helpers.publish import _normalize_s3_store - from bits_helpers.sync import resolve_and_export_s3_config + from bits_helpers.sync import resolve_and_export_s3_config, s3_settings store = _normalize_s3_store(store_url) bucket = store.split("://", 1)[-1].strip("/") resolve_and_export_s3_config() - endpoint = os.environ.get("S3_ENDPOINT_URL") or os.environ.get( - "BITS_S3_ENDPOINT_URL") or "https://s3.cern.ch" - style = os.environ.get("S3_ADDRESSING_STYLE", "path") - if os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY"): + settings = s3_settings(bucket) # this bucket's, else the run's + endpoint = settings["endpoint"] or os.environ.get( + "BITS_S3_ENDPOINT_URL") or site.get("store.s3_endpoint") + style = settings["addressing_style"] or "path" + keys = {} + if settings["access_key"] and settings["secret_key"]: cfg = Config(s3={"addressing_style": style}) + keys = {"aws_access_key_id": settings["access_key"], + "aws_secret_access_key": settings["secret_key"]} + if settings["session_token"]: + keys["aws_session_token"] = settings["session_token"] else: info("no S3 credentials found — auditing anonymously") cfg = Config(signature_version=UNSIGNED, s3={"addressing_style": style}) - return boto3.client("s3", endpoint_url=endpoint, config=cfg), bucket, endpoint + return boto3.client("s3", endpoint_url=endpoint, config=cfg, **keys), bucket, endpoint def audit_store(store_url, restricted, work_dir): @@ -355,8 +362,36 @@ def _recipe_source_prefixes(recipes_dir, rec, restricted): return prefixes, unresolved +def _decertify_packages(console, groups, packages, dry_run): + """Ask bits-orchestrator to take *packages* (names) out of the certified BOMs + of *groups* (those the caller administers): each BOM is replaced by itself + without them (removal only), and the manifests are re-signed. Returns the + number of requests that did not succeed.""" + from bits_helpers import orchestrated + if dry_run: + for name in sorted(packages): + info(" [dry-run] would ask the orchestrator to decertify %s in %s", name, + ", ".join(groups) or "the groups you administer") + return 0 + failed = 0 + for name in sorted(packages): + for group in groups: + try: + action = console.settle(console.decertify_package( + group, name, "compliance: not redistributable")) + except orchestrated.RequestError as exc: + error(" decertify %s in %s: %s", name, group, exc) + failed += 1 + continue + ok, words = console.outcome(action) + (info if ok else error)(" decertify %s in %s: action %s %s", + name, group, action.get("id"), words) + failed += 0 if ok else 1 + return failed + + def enforce_store(store_url, rec, recipes_dir, key_pem=None, dry_run=False, - work_dir="sw"): + work_dir="sw", console=None): """Remove non-compliant packages from the store and its manifests. The admin action behind ``bits compliance --enforce``: @@ -373,16 +408,54 @@ def enforce_store(store_url, rec, recipes_dir, key_pem=None, dry_run=False, CI certification self-heals (missing objects are dropped), but the signed manifests are stale until then. + In a store whose certified set bits-orchestrator owns (orchestrated.py), 4 is + a decertify-package request instead, for each restricted package found in a + BOM or a signed manifest, in each group the caller administers: the set's + BOMs lose those entries only. It is made through *console* (an + ``orchestrated.Console``; not needed for a dry run), and *key_pem* is + refused: the orchestrator alone signs those manifests. + With *dry_run* every action is printed and nothing is touched. Returns the number of issues remaining (0 = store clean after enforcement). """ + from bits_helpers import orchestrated restricted = {p.lower() for p in rec["restricted"]} restricted_src = {p.lower() for p in rec["restricted_sources"]} s3, bucket, _endpoint = _s3_client(store_url) - if not dry_run and not (os.environ.get("AWS_ACCESS_KEY_ID") - and os.environ.get("AWS_SECRET_ACCESS_KEY")): + from bits_helpers.sync import s3_settings + if not dry_run and not (s3_settings(bucket)["access_key"] + and s3_settings(bucket)["secret_key"]): error("--enforce needs S3 write credentials (a dry run does not)") return 1 + # Before touching anything: who signs this store's manifests. + try: + marker = orchestrated.read(s3, bucket) + except orchestrated.Unknown as exc: + error("--enforce: %s (it says whether bits-orchestrator signs the manifests)", exc) + return 1 + if marker and key_pem: + error("--enforce: %s signs this store's manifests; drop --key, and the restricted " + "packages are decertified instead", orchestrated.describe(marker)) + return 1 + if marker and console is None and not dry_run: + error("--enforce: %s certifies this store: decertifying the restricted packages needs " + "the bits-console URL (--console, $BITS_CONSOLE_URL, the site's console.backend_url) " + "and a GitLab token", + orchestrated.describe(marker)) + return 1 + groups = [] + if marker and not dry_run: + try: + console.check() + groups = console.groups() + except orchestrated.RequestError as exc: + error("--enforce: %s", exc) + return 1 + if not groups: + error("--enforce: you administer no group of %s's certified set, so nothing " + "could be decertified: an admin of its groups must enforce", + marker["issuer"]) + return 1 act = "[dry-run] would" if dry_run else "will" def _delete_prefix(prefix, why): @@ -406,8 +479,19 @@ def _delete_prefix(prefix, why): Prefix="MANIFESTS/") for o in page.get("Contents", []) or []] affected_archs, remaining_boms, removed, repo_prune = set(), [], 0, [] + found = set() # restricted packages present for key in keys: leaf = key.split("/")[-1] + if marker and leaf.startswith("common-manifest") and leaf.endswith(".json"): + # What the set still certifies, even where the BOM copies are gone. + try: + signed = json.loads(s3.get_object(Bucket=bucket, Key=key)["Body"].read()) + found |= {str(e.get("package")) for e in signed.get("packages") or [] + if isinstance(e, dict) + and str(e.get("package", "")).lower() in restricted} + except Exception as exc: # pylint: disable=broad-except + warning("enforce: unreadable signed manifest %s (%s)", key, exc) + continue if (not leaf.endswith(".json") or "/rev-index/" in key or leaf.startswith("common-manifest")): continue @@ -425,6 +509,7 @@ def _delete_prefix(prefix, why): for e in offending: arch = e.get("effective_architecture") or "" affected_archs.add(arch) + found.add(str(e.get("package"))) removed += _delete_prefix( resolve_store_path(arch, e.get("hash", "")) + "/", "%s — redistributable: false" % e.get("package")) @@ -457,13 +542,21 @@ def _delete_prefix(prefix, why): info("Enforcement: %d object(s) %s removed, %d build manifest(s) affected, " "architectures: %s", removed, "would be" if dry_run else "", len(repo_prune), ", ".join(sorted(affected_archs)) or "none") - if repo_prune: + if repo_prune and not marker: warning("Certification re-derives from the bits-manifests git repo — " "prune the matching BOM file(s) there too: %s", ", ".join(sorted(repo_prune))) - # 4 — re-certify the affected platforms from the rewritten BOMs. - if affected_archs and key_pem and not dry_run: + # 4 — take the packages out of the orchestrator's set, or re-certify the + # affected platforms from the rewritten BOMs. + if marker: + # One spelling per package: the set matches names case-insensitively. + names = {n.lower(): n for n in sorted(found, reverse=True)} + if names and _decertify_packages(console, groups, set(names.values()), dry_run): + error("Restricted objects were removed, but not every package was " + "decertified; re-run this command (deletions are idempotent)") + return 1 + elif affected_archs and key_pem and not dry_run: import tempfile from bits_helpers import certify as _certify out = os.path.join(tempfile.mkdtemp(prefix="bits-enforce-"), @@ -502,6 +595,19 @@ def _delete_prefix(prefix, why): return 0 +def _console(args): + """The bits-console the decertify requests of --enforce go to, or None + without a URL or a GitLab token (needed only where the orchestrator owns + the set; enforce_store says so then).""" + from bits_helpers import forge, orchestrated + url = orchestrated.console_url(getattr(args, "console", None)) + token = forge.resolve_gitlab_token(getattr(args, "gitlabToken", None)) + if not (url and token): + return None + return orchestrated.Console(url, token, cafile=getattr(args, "consoleCafile", None), + insecure=getattr(args, "consoleInsecure", False)) + + def doCompliance(args, parser): """CLI entrypoint for ``bits compliance``. Returns the exit code.""" roots = list(getattr(args, "packages", []) or []) @@ -551,41 +657,47 @@ def doCompliance(args, parser): if getattr(args, "noStoreCheck", False): parser.error("--enforce audits and cleans the store; it cannot be " "combined with --no-store-check") - return enforce_store(args.complianceStore, rec, recipes_dir, - key_pem=getattr(args, "enforceKey", None), - dry_run=getattr(args, "dryRun", False), - work_dir=os.path.abspath(args.workDir)) or ( - 1 if issues else 0) + from bits_helpers.sync import store_urls + left = 0 + for url in store_urls(args.complianceStore): # each store in turn + left += enforce_store(url, rec, recipes_dir, + key_pem=getattr(args, "enforceKey", None), + dry_run=getattr(args, "dryRun", False), + work_dir=os.path.abspath(args.workDir), + console=_console(args)) or 0 + return left or (1 if issues else 0) if not getattr(args, "noStoreCheck", False): restricted = {p.lower() for p in rec["restricted"]} - try: - st = audit_store(args.complianceStore, restricted, - os.path.abspath(args.workDir)) - except Exception as exc: # pylint: disable=broad-except - error("store audit failed: %s (use --no-store-check to audit " - "recipes only)", exc) - return 1 - info("Store %s: %d build manifest(s), %d signed manifest(s)", - st["bucket"], st["boms"], st["signed"]) - info(" anonymous access ........ %s", - "WORLD-READABLE (unauthenticated listing succeeds)" - if st["public"] else "no (authenticated only)") - if st["stored"]: - sev = ("PUBLICLY EXPOSED (bucket is world-readable — this is " - "redistribution)" if st["public"] else - "present (private store: reuse only, do not publish)") - line = "%d restricted package build(s) in the store — %s:\n %s" % ( - len(st["stored"]), sev, "\n ".join(sorted(st["stored"]))) - if st["public"]: - issues.append(line) - else: - info(" %s", line) - if st["certified"]: - issues.append( - "%d restricted package(s) in SIGNED manifests (certified for " - "reuse — review): %s" - % (len(st["certified"]), "; ".join(sorted(st["certified"])))) + from bits_helpers.sync import store_urls + for store in store_urls(args.complianceStore): # each store in turn + try: + st = audit_store(store, restricted, + os.path.abspath(args.workDir)) + except Exception as exc: # pylint: disable=broad-except + issues.append("store audit of %s failed: %s (use --no-store-check to audit " + "recipes only)" % (store, exc)) + continue + info("Store %s: %d build manifest(s), %d signed manifest(s)", + st["bucket"], st["boms"], st["signed"]) + info(" anonymous access ........ %s", + "WORLD-READABLE (unauthenticated listing succeeds)" + if st["public"] else "no (authenticated only)") + if st["stored"]: + sev = ("PUBLICLY EXPOSED (bucket is world-readable — this is " + "redistribution)" if st["public"] else + "present (private store: reuse only, do not publish)") + line = "%d restricted package build(s) in store %s — %s:\n %s" % ( + len(st["stored"]), st["bucket"], sev, "\n ".join(sorted(st["stored"]))) + if st["public"]: + issues.append(line) + else: + info(" %s", line) + if st["certified"]: + issues.append( + "%d restricted package(s) in SIGNED manifests of %s (certified for " + "reuse — review): %s" + % (len(st["certified"]), st["bucket"], "; ".join(sorted(st["certified"])))) if issues: banner("Compliance issues: %d", len(issues)) diff --git a/bits_helpers/cvmfs_publish.py b/bits_helpers/cvmfs_publish.py index 9ad86e4b..c22fc6b0 100644 --- a/bits_helpers/cvmfs_publish.py +++ b/bits_helpers/cvmfs_publish.py @@ -359,10 +359,11 @@ def submit_ingest(prepub_url, token, repo, path, tar_file, build_id="", fields["prewarm"] = (None, "true") for k, v in identity.items(): fields[k] = (None, v) - with open(tar_file, "rb") as tfh: - fields["tar"] = ("pkg.tar", tfh, "application/octet-stream") + fields["tar"] = ("pkg.tar", tar_file, "application/octet-stream") + with _pp.MultipartFile(fields) as body: # streamed: the tar is not held in memory + headers["Content-Type"] = body.content_type try: - resp = session.post(url, files=fields, headers=headers, timeout=1800) + resp = session.post(url, data=body, headers=headers, timeout=1800) except requests.ConnectionError as exc: raise SystemExit("upload of %s cut off by prepub (%s); prepub refuses" " an upload that is too large or would fill its" @@ -450,6 +451,106 @@ def _writes_outside_pkgroot(before, after, pkgroot): if p != root and not p.startswith(root + os.sep)) +def _pkginfo(pkgroot): + """(PKG_DIR, PP) from etc/profile.d/.bits-pkginfo, or (None, None).""" + info = {} + try: + with open(os.path.join(pkgroot, "etc", "profile.d", ".bits-pkginfo")) as fh: + for line in fh: + k, _, v = line.strip().partition("=") + info[k] = v.strip('"') + except OSError: + return None, None + m = re.fullmatch(r"\$\{PKGPATH:-(.*)\}", info.get("PP", "")) + return info.get("PKG_DIR") or None, (m.group(1) if m else info.get("PP")) or info.get("OP") + + +def dependency_targets(pkgroot, ctx): + """{work-dir path: CVMFS path} for every dependency in the package's + .meta.json, and its own path to PKG_DIR (relocate-me.sh maps that). A value + is a ValueError (or None) when the dependency has no CVMFS path.""" + pkg_dir, pp = _pkginfo(pkgroot) + if not pkg_dir or not pp: + return {} + out = {"%s/%s" % (pkg_dir, pp): pkg_dir} + try: + with open(os.path.join(pkgroot, ".meta.json")) as fh: + rec = (json.load(fh).get("dependencies") or {}).get("recursive") or {} + except (OSError, ValueError): + rec = {} + for d in (rec.get("build") or []) + (rec.get("runtime") or []): + if not d.get("name"): + continue + ver, rev = str(d.get("version") or ""), str(d.get("revision") or "") + fam = d.get("pkg_family") or "" + vdir = ver + ("-" + rev if rev else "") + arch = d.get("effective_architecture") or ctx["arch"] + old = "/".join([pkg_dir, arch] + ([fam] if fam else []) + [d["name"], vdir]) + if old in out: + continue + try: # .meta.json has no dependency commits: {commit} stays unresolved + p = resolve_pkg_path( + pkgroot, ctx["repo"], d["name"], vdir, ver, rev, ctx.get("platform", ""), + ctx.get("install_dir", ""), "{commit}", ctx.get("user", ""), fam, + kind="path", tmpl_prefix=ctx["tmpl_prefix"], arch=arch, + prefix_fallback=ctx.get("prefix_fallback"), templates=ctx.get("templates")) + out[old] = "/cvmfs/%s/%s" % (ctx["repo"], p) if p else None + except ValueError as e: + out[old] = e + return out + + +def relocate_dependencies(pkgroot, targets): + """Rewrite the dependencies' work-dir paths in the files relocate-me.sh + relocates to their CVMFS paths, before it runs: it maps everything under + PKG_DIR to the package's own path. Returns the files still naming PKG_DIR.""" + listing = os.path.join(pkgroot, "etc", "profile.d", ".bits-relocate") + if not targets or not os.path.isfile(listing): + return [] + pkg_dir, _pp = _pkginfo(pkgroot) + olds = sorted(targets, key=len, reverse=True) + # A path ends where a version could not go on: 1.0-1 is not 1.0-10. + rx = re.compile(b"(?:" + b"|".join(re.escape(o.encode()) for o in olds) + + rb")(?![\w.+~-])") + # Left for relocate-me.sh: the package's own staging dir. Anything else + # (BUILD/, SOURCES/, a package not in the closure) would land in the package. + left = re.compile(re.escape(pkg_dir.encode()) + b"/(?!INSTALLROOT/)") + with open(listing) as fh: + rels = [l.rstrip("\n") for l in fh if l.strip()] + leftover = [] + for rel in rels: + if os.path.isabs(rel) or ".." in rel.split("/"): + continue # relocate-me.sh only edits files in the package + f = os.path.join(pkgroot, rel) + if os.path.islink(f) or not os.path.isfile(f): + continue + with open(f, "rb") as fh: + data = fh.read() + + def new(m): + old = m.group(0).decode() + t = targets[old] + if t == pkg_dir: + return t.encode() + if not isinstance(t, str): + raise SystemExit("%s refers to %s, which has no CVMFS path: %s" + % (rel, old, t or "no template")) + if pkg_dir in t: # relocate-me.sh would rewrite it again + raise SystemExit("cannot relocate %s to %s: it contains the build dir %s" + % (old, t, pkg_dir)) + return t.encode() + out = rx.sub(new, data) + if left.search(rx.sub(b"", data)): + leftover.append(rel) + if out != data: # replace, as sed -i does: keeps the mode, breaks hard links + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(f), prefix=".reloc-") + with os.fdopen(fd, "wb") as fh: + fh.write(out) + os.chmod(tmp, os.stat(f).st_mode & 0o7777) + os.replace(tmp, f) + return leftover + + def _publish_tar(ctx, path, tar, label, fp=None, identity="", identity_hash="", replace=False): """Publish ONE prepared tar via the configured path; return its job id and @@ -746,29 +847,70 @@ def _list_tree(entries): return tree -def _package_tree(spec, ctx): - """The package's view entries: its .bits-view.json (written by the build), - from the local install tree when that holds this very build, else from the - tarball; a tarball without one (built before) is listed in full.""" +def _vdir(spec): rev = spec.get("revision", "") - vdir = spec.get("version", "") + ("-" + rev if rev else "") + return spec.get("version", "") + ("-" + rev if rev else "") + + +def _local_root(spec, ctx): + """The package's local install tree when it holds this very build, else None.""" fam = spec.get("pkg_family", "") local = os.path.join(os.path.dirname(ctx["tars_root"].rstrip("/")), spec.get("effective_architecture") or ctx["arch"], - *([fam] if fam else []), spec["package"], vdir) + *([fam] if fam else []), spec["package"], _vdir(spec)) try: with open(os.path.join(local, ".meta.json")) as fh: same = (json.load(fh).get("package") or {}).get("hash") == spec.get("hash") - if same: + except (OSError, ValueError, AttributeError): + return None + return local if same else None + + +def _package_tarball(spec, ctx, what): + tar = tar_path(spec, ctx["tars_root"], ctx["arch"]) + if not os.path.isfile(tar): + raise SystemExit("%s@%s: no tarball to %s from (%s)" + % (spec["package"], _vdir(spec), what, tar)) + return tar + + +def _package_tree(spec, ctx): + """The package's view entries: its .bits-view.json (written by the build), + from the local install tree when that holds this very build, else from the + tarball; a tarball without one (built before) is listed in full.""" + local = _local_root(spec, ctx) + try: + if local: with open(os.path.join(local, ".bits-view.json")) as fh: return _list_tree(json.load(fh)["entries"]) except (OSError, ValueError, KeyError, TypeError): pass - tar = tar_path(spec, ctx["tars_root"], ctx["arch"]) - if not os.path.isfile(tar): - raise SystemExit("%s@%s: no tarball to list its files from (%s)" - % (spec["package"], vdir, tar)) - return _tar_tree(tar) + return _tar_tree(_package_tarball(spec, ctx, "list its files")) + + +def _package_file(spec, ctx, rel): + """The text of the package's *rel* (package-relative) as published, or None + when it has none: from the local install tree when that holds this very + build, else from the tarball (/[/]//...), its + revision placeholder filled as the install fills it.""" + import tarfile + local = _local_root(spec, ctx) + if local: + try: + with open(os.path.join(local, rel)) as fh: + return fh.read() + except OSError: + return None + fam = spec.get("pkg_family", "") + want = os.path.join(spec.get("effective_architecture") or ctx["arch"], + *([fam] if fam else []), spec["package"], _vdir(spec), rel) + with tarfile.open(_package_tarball(spec, ctx, "read %s" % rel), "r:*") as tf: + for m in tf: + if os.path.normpath(m.name) == want and m.isfile(): + text = tf.extractfile(m).read().decode("utf-8", "replace") + return text.replace("@@PKGREVISION@%s@@" % spec.get("hash", ""), + spec.get("revision", "")) + return None def _tar_tree(tar): @@ -818,6 +960,38 @@ def _tar_tree(tar): return tree +def _resolve(tree, path, depth=0): + """*path* of a package tree from _tar_tree with its symlinks followed + inside the package, or None when it leaves it or does not exist.""" + kind, target = tree.get(path, (None, "")) + if kind != "link" or depth > 16: + return path if kind else None + t = os.path.normpath(os.path.join(os.path.dirname(path), target)) + return None if target.startswith("/") or t.startswith("..") else _resolve(tree, t, depth + 1) + + +def _resolve_path(tree, rel, depth=0): + """*rel* of a package tree from _tar_tree with every symlink on the way + followed, as the file system would; None when it does not exist. Raises + ValueError when a link leads out of the package.""" + cur = "" + for part in rel.split("/"): + cur = cur + "/" + part if cur else part + kind, target = tree.get(cur, (None, "")) + if kind is None: + return None + if kind == "link": + t = os.path.normpath(os.path.join(os.path.dirname(cur), target)) + if depth > 16: + raise ValueError("%s: too many links" % cur) + if target.startswith("/") or t == ".." or t.startswith("../"): + raise ValueError("%s is a link out of the package" % cur) + cur = _resolve_path(tree, t, depth + 1) + if cur is None: + return None + return cur + + def _view_entries(tree, subdirs): """(view-relative path, package-relative path) for every file or symlink the view links, walking *subdirs* of a package tree from _tar_tree. A symlinked @@ -827,25 +1001,18 @@ def _view_entries(tree, subdirs): for path in tree: children.setdefault(os.path.dirname(path), []).append(path) - def resolve(path, depth=0): - kind, target = tree.get(path, (None, "")) - if kind != "link" or depth > 16: - return path if kind else None - t = os.path.normpath(os.path.join(os.path.dirname(path), target)) - return None if target.startswith("/") or t.startswith("..") else resolve(t, depth + 1) - out = [] def walk(logical, real, seen): for child in sorted(children.get(real, [])): name = os.path.join(logical, os.path.basename(child)) - real_child = resolve(child) + real_child = _resolve(tree, child) if real_child and tree[real_child][0] == "dir" and real_child not in seen: walk(name, real_child, seen | {real_child}) else: out.append((name, name)) for sub in subdirs: - real = resolve(sub) + real = _resolve(tree, sub) if real and tree[real][0] == "dir": walk(sub, real, {real}) return out @@ -865,6 +1032,17 @@ def _excluded(path, patterns): "share/man") +def _view_rules(spec, exclude): + """(directories the view walks, patterns it drops) for a package, or None + when the view leaves it out.""" + rules = spec.get("view", True) + if spec["package"] in exclude or rules is False: + return None + rules = rules if isinstance(rules, dict) else {} + return (list(VIEW_SUBDIRS) + [str(p).strip("/") for p in (rules.get("include") or [])], + [str(p).strip("/") for p in (rules.get("exclude") or [])]) + + def merged_view(ctx, specs, staging, view_path): """Build the release's merged view in *staging*: relative symlinks from *view_path* (repo-relative) into each package's published path, listed from @@ -877,12 +1055,10 @@ def merged_view(ctx, specs, staging, view_path): exclude = set(ctx["templates"].get("view_exclude") or []) plans, users = [], {} # users: view path -> packages with an entry at/below it for spec in reversed(specs): - rules = spec.get("view", True) - if spec["package"] in exclude or rules is False: + rules = _view_rules(spec, exclude) + if rules is None: continue - rules = rules if isinstance(rules, dict) else {} - drop = [str(p).strip("/") for p in (rules.get("exclude") or [])] - subdirs = list(VIEW_SUBDIRS) + [str(p).strip("/") for p in (rules.get("include") or [])] + subdirs, drop = rules entries = [] for vrel, prel in _view_entries(_package_tree(spec, ctx), subdirs): dropped = _excluded(prel, drop) @@ -939,20 +1115,97 @@ def fold(vrel, pkg, roots): return res -def write_view_setup(staging, published_at): +def view_environment(ctx, specs, view_path, top_level): + """What loading the release's modules sets, for its view's setup.sh: the + modulefiles of *top_level* (all *specs* without it) and of the modules they + load, read without Tcl (bits_helpers.modulefile_env). Returns (env, problems), + env {"paths": {var: [entries]}, "sets": {var: value}, "base": (relative to + the view, absolute)} with modulefile_env.VIEW / BASE in the values, or None. + A problem means setup.sh does not set everything the modules do.""" + from bits_helpers import modulefile_env as mfe + pkgs_root = (fixed_dir(ctx, "packages") + if (ctx.get("templates") or {}).get("packages") else None) + if not pkgs_root: + return None, ["the group has no packages directory"] + by = {sp["package"]: sp for sp in specs} + members = {n: _vdir(sp) for n, sp in by.items()} + trees = {} + + def unsupported(what, call): + try: + return call() + except (SystemExit, OSError, ValueError, EOFError) as exc: + raise mfe.Unsupported("cannot %s: %s" % (what, exc)) + + def tree(pkg): + if pkg not in trees: + trees[pkg] = unsupported("list it", lambda: _package_tree(by[pkg], ctx)) + return trees[pkg] + + def is_dir(pkg, rel): + try: + real = _resolve_path(tree(pkg), rel) + except ValueError as exc: # modulecmd follows it out of the package; we cannot + raise mfe.Unsupported(str(exc)) + return real is not None and tree(pkg)[real][0] == "dir" + + def read(pkg): + return unsupported("read its modulefile", lambda: _package_file( + by[pkg], ctx, "etc/modulefiles/%s" % pkg)) + + exclude = set(ctx["templates"].get("view_exclude") or []) + # Beside what a view links, a package root holds only its own metadata. + own = ("etc", "relocate-me.sh") + + def in_view(pkg, rel): + rules = _view_rules(by[pkg], exclude) + if not rules: + return False + if not rel: # the root (CMAKE_PREFIX_PATH): only when the view has all of it + try: + names = [p for p in tree(pkg) if "/" not in p] + except mfe.Unsupported: + return False + return not rules[1] and all( + p.startswith(".") or p in own or p in rules[0] for p in names) + return not _excluded(rel, rules[1]) and any( + rel == d or rel.startswith(d + "/") for d in rules[0]) + + top = list(top_level or []) or [sp["package"] for sp in specs] + problems = ["the release's %s is not in it" % t for t in top if t not in by] + ops, more = mfe.release_ops([t for t in top if t in by], members, read, is_dir) + paths, sets, rest = mfe.setup_env(ops, members, in_view) + base = (os.path.relpath(pkgs_root, view_path), "/cvmfs/%s/%s" % (ctx["repo"], pkgs_root)) + return {"paths": paths, "sets": sets, "base": base}, problems + more + rest + + +def write_view_setup(staging, published_at, env=None): """setup.sh / setup.csh for a merged view; one entry per variable, prepended. setup.sh locates itself under bash and zsh (so the view works wherever the repository is mounted); other shells, and csh which cannot, use the path it - is published at (*published_at*, absolute).""" + is published at (*published_at*, absolute). With *env* (view_environment), + also what the release's modules set: their path entries beside the view's, + and their other variables (each package's _ROOT, ...), below the + packages directory found from the view.""" import glob from bits_helpers.view import view_env + from bits_helpers.modulefile_env import BASE, INHERITED, VIEW def _ver(site): mm = os.path.basename(os.path.dirname(site))[len("python"):] return tuple(int(x) for x in mm.split(".") if x.isdigit()), mm sites = glob.glob(os.path.join(staging, "lib*", "python*", "site-packages")) python_mm = max(map(_ver, sites))[1] if sites else None - env = view_env(staging, python_mm=python_mm) - env = {k: v.replace(staging, "@V@") for k, v in sorted(env.items())} + # Each variable's entries in order, INHERITED where its value from before goes: + # the modules' as they set them, then the view's own that they did not add. + paths = {var: list(entries) for var, entries in (env or {}).get("paths", {}).items()} + for var, value in view_env(staging, python_mm=python_mm).items(): + have = paths.setdefault(var, [INHERITED]) + for e in value.replace(staging, VIEW).split(":"): + if e not in have: + have.insert(have.index(INHERITED), e) + sets = (env or {}).get("sets", {}) + uses_base = any(BASE in v for v in list(sets.values()) + + [e for es in paths.values() for e in es if e is not INHERITED]) sh = ["# Source this file (bash/zsh/sh) to use this release view.", 'if [ -n "${BASH_SOURCE:-}" ]; then _s="${BASH_SOURCE[0]}"', "elif [ -n \"${ZSH_VERSION:-}\" ]; then eval '_s=${(%):-%x}'", @@ -962,10 +1215,28 @@ def _ver(site): csh = ["# Source this file (csh/tcsh) to use this release view.", 'set _v = "%s"' % published_at, "setenv BITS_VIEW $_v"] - for var, val in env.items(): - sh.append('export %s="%s${%s:+:$%s}"' % (var, val.replace("@V@", "$_v"), var, var)) - csh.append('if ($?%s) then\n setenv %s "%s:${%s}"\nelse\n setenv %s "%s"\nendif' - % (var, var, val.replace("@V@", "$_v"), var, var, val.replace("@V@", "$_v"))) + if uses_base: + rel, absolute = env["base"] + sh.append('_b="$(cd "$_v/%s" 2>/dev/null && pwd)" || _b="$_v/%s"' % (rel, rel)) + csh.append('set _b = "%s"' % absolute) + + def shell(v): # the same in both: $_v and $_b + return v.replace(VIEW, "$_v").replace(BASE, "$_b") + for var in sorted(paths): + at = paths[var].index(INHERITED) + before = shell(":".join(paths[var][:at])) + after = shell(":".join(paths[var][at + 1:])) + if before: + value = before + "${%s:+:$%s}" % (var, var) + (":" + after if after else "") + else: + value = "${%s:+$%s:}" % (var, var) + after + sh.append('export %s="%s"' % (var, value)) + csh.append('if ($?%s) then\n setenv %s "%s"\nelse\n setenv %s "%s"\nendif' + % (var, var, ":".join(p for p in (before, "${%s}" % var, after) if p), + var, ":".join(p for p in (before, after) if p))) + for var in sorted(sets): + sh.append('export %s="%s"' % (var, shell(sets[var]))) + csh.append('setenv %s "%s"' % (var, shell(sets[var]))) # Man pages only while MANPATH is set: unset, man derives its search path # from PATH (the view's bin -> share/man), and setting it would hide the # system's own pages. @@ -973,8 +1244,8 @@ def _ver(site): sh.append('if [ -n "${MANPATH:-}" ]; then export MANPATH="$_v/share/man:$MANPATH"; fi') # Multi-line: csh expands ${MANPATH} on a one-line if even when unset. csh.append('if ($?MANPATH) then\n setenv MANPATH "$_v/share/man:${MANPATH}"\nendif') - sh.append("unset _s _v") - csh.append("unset _v") + sh.append("unset _s _v" + (" _b" if uses_base else "")) + csh.append("unset _v" + (" _b" if uses_base else "")) for name, body in (("setup.sh", sh), ("setup.csh", csh)): with open(os.path.join(staging, name), "w") as fh: fh.write("\n".join(body) + "\n") @@ -1111,8 +1382,13 @@ def publish_one(spec, ctx): reloc = os.path.join(pkgroot, "relocate-me.sh") if os.path.isfile(reloc): - env = dict(os.environ, - INSTALL_BASE="/cvmfs/%s/%s" % (ctx["repo"], path), + own = "/cvmfs/%s/%s" % (ctx["repo"], path) + stray = relocate_dependencies(pkgroot, dependency_targets(pkgroot, ctx)) + if stray: + print("WARNING: %s@%s: %d file(s) name the build area outside the package " + "and its dependencies: %s" % (pkg, vdir, len(stray), " ".join(stray[:5]) + + (" ..." if len(stray) > 5 else "")), file=sys.stderr) + env = dict(os.environ, INSTALL_BASE=own, WORK_DIR=work_dir, BITS_RELOCATE_STRIP_PP="1") # Views (lcg-view) place the packages they reference with these. if ctx.get("templates"): @@ -1466,8 +1742,16 @@ def _publishable(s): rel_links = rel_view = False if a.release_view: tm = templates or {} + # A build of the toolchain (own_hash) alone is no release of this arch: + # its top-level packages live under another one (noarch ones may). + tops = [s for s in pkgs if s.get("package") in (_man.get("requested_packages") or [])] + outside = bool(tops and a.arch) and all( + (s.get("effective_architecture") or a.arch) not in (a.arch, SHARED_ARCH, "shared") + for s in tops) why = ("the group has no cvmfs_packages_template" if not tm.get("packages") else "the build has no release (the main line)" if not tm.get("release") + else "its top-level packages (%s) live outside %s" + % (", ".join(sorted(s["package"] for s in tops)), a.arch) if outside else "") rel_links = not why and bool(tm.get("path")) and tm["path"] != tm["packages"] rel_view = not why and bool(tm.get("views")) @@ -1678,7 +1962,20 @@ def _publish_merged_view(ctx, specs, emit, top_level=None, force=False): for path, winner, loser in res["conflicts"]: sys.stderr.write("[publish] merged view: %s from %s, not %s\n" % (path, winner, loser)) - write_view_setup(staging, "/cvmfs/%s/%s" % (ctx["repo"], view_path)) + env, problems = view_environment(ctx, specs, view_path, top_level) + for why in problems: + sys.stderr.write("[publish] merged view: setup.sh does not set what the " + "modules do: %s\n" % why) + write_view_setup(staging, "/cvmfs/%s/%s" % (ctx["repo"], view_path), env) + # The one top-level module setup.sh stands in for, when it does. + top = next((sp for sp in specs if sp["package"] in top_level), None) \ + if not problems and len(top_level) == 1 else None + if top: + marker = os.path.join(staging, setup_marker(top)) + with open(marker, "w") as fh: + fh.write("# setup.sh sets what loading %s/%s sets\n" + % (top["package"], _vdir(top))) + os.chmod(marker, 0o644) # The view's identity: what prepub compares to decide to replace it. # Modes are hashed, so they must not depend on the publisher's umask. for dp, dns, _fns in os.walk(staging): @@ -1692,7 +1989,7 @@ def _publish_merged_view(ctx, specs, emit, top_level=None, force=False): if exists: if state.get("hash") == fp: emit(["SKIPPED merged view: %s is unchanged" % view_path]) - return 0 + return _publish_view_pointer(ctx, top, view_path, emit) if top else 0 owner = _view_owner(ctx, view_path) if not force else None if owner and top_level and owner != top_level: emit(["FAILED merged view: %s was made from %s, not %s; " @@ -1707,7 +2004,9 @@ def _publish_merged_view(ctx, specs, emit, top_level=None, force=False): with open(os.path.join(staging, ".meta.json"), "w") as f: json.dump({"package": {"package": "merged-view", "hash": fp}, "top_level": top_level, "build_id": ctx.get("build_id", ""), - "members": recorded}, f, indent=1) + "members": recorded, + "setup": {"modules": not problems, "problems": problems[:50]}}, + f, indent=1) os.chmod(os.path.join(staging, ".meta.json"), 0o644) # No .cvmfscatalog here: both publish paths ingest with create-catalog-on- # root (-c / -C true), which adds the marker itself; a second one fails. @@ -1720,7 +2019,7 @@ def _publish_merged_view(ctx, specs, emit, top_level=None, force=False): jid = _publish_tar(ctx, view_path, tar, label, identity=view_path, identity_hash=fp, replace=replace) emit(["PUBLISHED %s %s" % (jid, label)]) - return 0 + return _publish_view_pointer(ctx, top, view_path, emit) if top else 0 except (SystemExit, Exception) as exc: emit(["FAILED merged view: %s" % exc]) return 1 @@ -1728,6 +2027,41 @@ def _publish_merged_view(ctx, specs, emit, top_level=None, force=False): shutil.rmtree(staging, ignore_errors=True) +def setup_marker(spec): + """The file in a release's view that says its setup.sh sets what loading + the release's top-level module *spec* sets: there only while the view is + one made for that very module.""" + return ".setup-complete.%s.%s" % (spec["package"], _vdir(spec)) + + +def view_pointer(spec, ctx): + """Where the modules directory points from a release's top-level module to + its view: //..view, a link to the view's + setup_marker (a file, not a modulefile, so modulecmd passes it over). A view + replaced by another release's has no such marker: the link then dangles.""" + mod = _spec_path(spec, ctx, "modules") + return mod and "%s/.%s.view" % (mod, _vdir(spec)) + + +def _publish_view_pointer(ctx, spec, view_path, emit): + """The link from the release's top-level module *spec* to its view + (view_pointer), unless it is published already. Returns the rc contribution.""" + root = (fixed_dir(ctx, "modules") + if (ctx.get("templates") or {}).get("modules") else None) + link = view_pointer(spec, ctx) if root else None + if not link: + return 0 + st = published_state(ctx, link) if ctx["submit"] else None + if st and st.get("exists"): + return 0 + label = "view-pointer@%s" % root + jid, err = publish_links(ctx, root, [(link, "%s/%s" % (view_path, setup_marker(spec)))], + label) + emit(["PUBLISHED %s %s" % (jid, label)] if err is None else + ["FAILED view pointer %s: %s" % (link, err)]) + return 0 if err is None else 1 + + def _publish_release_links(ctx, specs, emit): """The release links (cvmfs_releases_template): one symlink per package at its releases path. The release root is shared (earlier builds, other diff --git a/bits_helpers/cvmfs_release.py b/bits_helpers/cvmfs_release.py index 0af92955..8c6c8b95 100644 --- a/bits_helpers/cvmfs_release.py +++ b/bits_helpers/cvmfs_release.py @@ -27,12 +27,14 @@ class ReleaseError(Exception): pass -def _get_json(s3, bucket, key): +def get_json(s3, bucket, key): + """The JSON object stored at *key* (boto3 client *s3*).""" body = s3.get_object(Bucket=bucket, Key=key)["Body"].read() return json.loads(body) -def _list_keys(s3, bucket, prefix): +def list_keys(s3, bucket, prefix): + """All keys under *prefix*, across listing pages.""" out, token = [], None while True: kw = {"Bucket": bucket, "Prefix": prefix} @@ -45,7 +47,8 @@ def _list_keys(s3, bucket, prefix): token = page.get("NextContinuationToken") -def _is_bom(doc): +def is_bom(doc): + """Whether *doc* has the shape of a build manifest (BOM).""" return (isinstance(doc, dict) and doc.get("build_id") and doc.get("effective_architecture") and isinstance(doc.get("packages"), list) and all(isinstance(p, dict) and p.get("package") for p in doc["packages"])) @@ -58,9 +61,9 @@ def find_boms(s3, bucket, pipeline=None, build_id=None): keys, indexed = None, False if pipeline: builds = {} - for key in _list_keys(s3, bucket, "BUILDS/by-pipeline/%s/" % pipeline): + for key in list_keys(s3, bucket, "BUILDS/by-pipeline/%s/" % pipeline): try: - index = _get_json(s3, bucket, key) + index = get_json(s3, bucket, key) except Exception as exc: raise ReleaseError("unreadable %s: %s" % (key, exc)) builds[index.get("build_id", "")] = index.get("keys") or [] @@ -77,18 +80,18 @@ def find_boms(s3, bucket, pipeline=None, build_id=None): if keys is None: if not build_id: raise ReleaseError("name the build with --pipeline or --build-id") - keys = [k for k in _list_keys(s3, bucket, "MANIFESTS/%s/" % build_id) + keys = [k for k in list_keys(s3, bucket, "MANIFESTS/%s/" % build_id) if k.endswith(".json")] newest = {} for key in keys: try: - bom = _get_json(s3, bucket, key) + bom = get_json(s3, bucket, key) except Exception as exc: if indexed: # a BOM of this build: without it the release is incomplete raise ReleaseError("cannot read %s: %s" % (key, exc)) warning("skipping unreadable %s: %s", key, exc) continue - if not _is_bom(bom): + if not is_bom(bom): continue # NOTICE, SBOMs and other files beside the BOMs if pipeline and str(bom.get("source_pipeline_id", "")) != str(pipeline): continue @@ -100,12 +103,23 @@ def find_boms(s3, bucket, pipeline=None, build_id=None): return [newest[a] for a in sorted(newest)] +def build_architecture(boms): + """The build's architecture as its BOMs record it, or None. Trusted only when it + is a package's effective architecture, or one qualified further (the toolchain's + with the build type): older BOMs recorded the publishing host's instead.""" + rec = next((str(b["architecture"]) for b in boms if b.get("architecture")), "") + effs = {str(p.get("effective_architecture") or b.get("effective_architecture") or "") + for b in boms for p in b.get("packages") or []} + return rec if rec and any(e and (rec == e or rec.startswith(e + "-")) for e in effs) else None + + def merge_boms(boms, arch=None): """One build manifest for `bits cvmfs publish` from a build's BOMs (one per effective architecture: the build's, its own_hash toolchain's, noarch), in dependency order, with the build's templates. The build's architecture - is its top-level package's, or for a noarch top level the one most of its - packages have. Returns (manifest, architecture).""" + is the one its BOMs record (build_architecture), else its top-level package's, + or for a noarch top level the one most of its packages have. Returns + (manifest, architecture).""" from bits_helpers.arch import SHARED_ARCH from bits_helpers.cvmfs_publish import dependency_order if not boms: @@ -123,7 +137,7 @@ def merge_boms(boms, arch=None): packages.append(dict(p, effective_architecture=p.get("effective_architecture") or b["effective_architecture"])) top = [p["effective_architecture"] for p in packages if p["package"] in requested] - arch = arch or (top[0] if top and top[0] != SHARED_ARCH else None) + arch = arch or build_architecture(boms) or (top[0] if top and top[0] != SHARED_ARCH else None) if not arch: counts = {} for p in packages: @@ -142,11 +156,14 @@ def merge_boms(boms, arch=None): def check_signed(packages, index): - """Refuse a package the signed manifests do not attest with this sha256.""" - from bits_helpers.certify import _norm_sha + """Refuse a package the signed manifests do not attest with this sha256 + (*index*: {(effective_architecture, hash): sha256}).""" + from bits_helpers.certify import norm_sha + def certified(p): + return index.get((p.get("effective_architecture"), p.get("hash"))) bad = [p["package"] for p in packages - if _norm_sha(index.get(p.get("hash"), "")) != _norm_sha(p.get("tarball_sha256", "")) - or not index.get(p.get("hash"))] + if not certified(p) + or norm_sha(certified(p)) != norm_sha(p.get("tarball_sha256", ""))] if bad: raise ReleaseError("not in the signed manifests (or another sha256): %s" % ", ".join(bad)) @@ -158,7 +175,7 @@ def fetch_missing(reader, packages, tars_root, published): BOM's sha256, and place it where `bits cvmfs publish` looks for it. Returns how many were fetched.""" from bits_helpers.checksum import checksum_file - from bits_helpers.certify import _norm_sha + from bits_helpers.certify import norm_sha from bits_helpers.cvmfs_publish import tar_path from bits_helpers.utilities import resolve_store_path store = os.path.join(os.path.dirname(tars_root.rstrip("/")), "store") @@ -177,7 +194,7 @@ def fetch_missing(reader, packages, tars_root, published): except Exception as exc: raise ReleaseError("%s@%s: cannot fetch %s from the store: %s" % (p["package"], p["hash"], key, exc)) - if _norm_sha(checksum_file(got)) != _norm_sha(p.get("tarball_sha256", "")): + if norm_sha(checksum_file(got)) != norm_sha(p.get("tarball_sha256", "")): raise ReleaseError("%s@%s: the store's tarball does not match the BOM's sha256" % (p["package"], p["hash"])) dest = tar_path(p, tars_root, arch) @@ -236,6 +253,9 @@ def main(argv=None): from bits_helpers.sync import remote_from_url work = a.work_dir or tempfile.mkdtemp(prefix="bits-release-") try: + if "," in a.remote_store: + raise ReleaseError("--remote-store takes one S3 store here, not a list: %r" + % a.remote_store) reader = remote_from_url(a.remote_store, "", "", work) if not hasattr(reader, "s3"): raise ReleaseError("--remote-store must be an S3 store (b3://): %r" diff --git a/bits_helpers/defaults.py b/bits_helpers/defaults.py index f2a0d86e..f5b92736 100644 --- a/bits_helpers/defaults.py +++ b/bits_helpers/defaults.py @@ -13,7 +13,7 @@ from collections import OrderedDict from os.path import exists -from bits_helpers.log import banner, debug, dieOnError +from bits_helpers.log import banner, debug, dieOnError, info from bits_helpers.recipe import getRecipeReader, parseRecipe from bits_helpers.paths import getConfigPaths, resolveDefaultsFilename from bits_helpers.utilities import asList @@ -22,6 +22,13 @@ SOURCE_KEYS = frozenset({"version", "tag", "source", "sources", "patches"}) +def override_key(key): + """An override's key as the overrides are kept: the package regex lowercased + (anything from an @ on dropped), the :matcher as written.""" + pkg, sep, matcher = str(key).split("@", 1)[0].partition(":") + return pkg.lower() + sep + matcher + + def validateDefaults(finalPkgSpec, defaults): if "valid_defaults" not in finalPkgSpec: return (True, "", []) @@ -157,7 +164,43 @@ def resolve_pkg_family(defaults_meta: dict, package_name: str) -> str: return default_family +# The build type a chain gets when it qualifies the architecture (a compiler +# profile's append_arch) but names none: every build of the same binaries then +# lands under one qualified architecture (…-gcc15-opt), never also …-gcc15. +DEFAULT_BUILD_TYPE = "opt" +_BUILD_TYPES = ("opt", "dbg") + + def readDefaults(configDir, defaults, error, architecture): + """The merged defaults of the chain *defaults*, and their bodies. + + A chain that qualifies the architecture but names no build type (no profile + marked ``own_hash_neutral``, such as defaults-opt / defaults-dbg) gets + DEFAULT_BUILD_TYPE, when that profile can be found: it is inserted into + *defaults* (in place, so every later step sees the chain actually built) + before any trailing structural overlay, and exempt from recipes' + valid_defaults (it was implied, not asked for). + """ + meta, body = _read_defaults_chain(configDir, defaults, error, architecture) + if (meta.get("_append_arch_qualifiers") and not meta.get("_own_hash_drop_qualifiers") + and not any(d in _BUILD_TYPES for d in defaults) + and isinstance(defaults, list) + and resolveDefaultsFilename(DEFAULT_BUILD_TYPE, configDir, failOnError=False)): + exempt = set(meta.get("_valid_defaults_exempt") or ()) - {"release"} + at = len(defaults) + while at > 0 and defaults[at - 1] in exempt: + at -= 1 # overlays stay last + before = "::".join(defaults) + defaults.insert(at, DEFAULT_BUILD_TYPE) + info("--defaults %s names no build type: building %s", before, "::".join(defaults)) + meta, body = _read_defaults_chain(configDir, defaults, error, architecture) + # Implied, not asked for: a recipe's valid_defaults gate is not held against it. + meta["_valid_defaults_exempt"] = list(meta.get("_valid_defaults_exempt") or []) + [ + DEFAULT_BUILD_TYPE] + return meta, body + + +def _read_defaults_chain(configDir, defaults, error, architecture): defaultsMeta = {} defaultsBody = "" append_arch_qualifiers = [] # per-default append_arch values, in chain order @@ -166,6 +209,7 @@ def readDefaults(configDir, defaults, error, architecture): missing_defaults = [] # names with no defaults-.sh on the search path defaults_dirs = {} # name -> recipe dir its defaults file came from override_dirs = {} # override key -> dir of the (last) profile setting it + version_profiles = {} # override key -> the (last) profile setting its version layout_dirs = set() # repositories whose cvmfs.yaml is merged already # A repository's cvmfs.yaml never overrides a key that a more specific # repository's file set, whatever the chain order: first the group's (the @@ -244,7 +288,9 @@ def _layout(directory, meta): # changes what the package fetches claims it (the blocks deep-merge). for key, block in xMeta["overrides"].items(): if SOURCE_KEYS & set(block or {}): - override_dirs[str(key).split("@", 1)[0].lower()] = defaults_dirs.get(xdefaults) + override_dirs[override_key(key)] = defaults_dirs.get(xdefaults) + if "version" in (block or {}): + version_profiles[override_key(key)] = xdefaults # A repository's cvmfs.yaml (its CVMFS layout) counts as part of the # first profile taken from it, beneath that profile's own keys. d = defaults_dirs.get(xdefaults) @@ -277,6 +323,7 @@ def _layout(directory, meta): defaultsMeta["_missing_defaults"] = missing_defaults defaultsMeta["_defaults_dirs"] = defaults_dirs defaultsMeta["_override_dirs"] = override_dirs + defaultsMeta["_version_profiles"] = version_profiles debug("Merged Defaults: %s ",json.dumps(defaultsMeta,indent = 4)) @@ -379,7 +426,7 @@ def parseDefaults(disable, defaultsGetter, log, architecture=None, configDir=Non commonEnv = {"env": defaultsMeta["env"]} if "env" in defaultsMeta else {} overrides["defaults-release"] = commonEnv for k, v in defaultsMeta.get("overrides", {}).items(): - f = k.split("@", 1)[0].lower() + f = override_key(k) if "@" in k: taps[f] = "dist:"+k overrides[f] = dict(**(v or {})) diff --git a/bits_helpers/doctor.py b/bits_helpers/doctor.py index 8c09f6cc..3c7ae062 100644 --- a/bits_helpers/doctor.py +++ b/bits_helpers/doctor.py @@ -349,6 +349,15 @@ def _check_disk_space(work_dir: str, min_free_gib: float = 10.0) -> Tuple[str, s return WARN, "cannot determine disk usage for %s: %s" % (work_dir, exc) +def check_read_store(url: str, args) -> Tuple[str, str]: + """_check_store of a read store; a site default that cannot be reached is a + WARN: a build without --remote-store then goes on without it.""" + status, detail = _check_store(url, getattr(args, "insecure", False)) + if status == FAIL and url and not getattr(args, "remoteStoreExplicit", True): + return WARN, detail + " (the site's default read store; builds go on without it)" + return status, detail + + def _check_store(url: str, insecure: bool = False) -> Tuple[str, str]: """Check remote-store reachability. @@ -390,10 +399,10 @@ def _check_store(url: str, insecure: bool = False) -> Tuple[str, str]: return WARN, "s3:// store configured but ~/.s3cfg not found: %s" % url if url.startswith("b3://"): - key = os.environ.get("AWS_ACCESS_KEY_ID", "") - if key: - return PASS, "AWS_ACCESS_KEY_ID set for b3:// store %s" % url - return WARN, "b3:// store configured but AWS_ACCESS_KEY_ID not set: %s" % url + from bits_helpers.sync import s3_settings + if s3_settings(url[5:].split("/", 1)[0])["access_key"]: + return PASS, "S3 credentials set for b3:// store %s" % url + return WARN, "b3:// store configured but no S3 credentials for it: %s" % url if url.startswith("rsync://"): return SKIP, "rsync:// store connectivity check not supported; assumed reachable: %s" % url @@ -403,6 +412,19 @@ def _check_store(url: str, insecure: bool = False) -> Tuple[str, str]: # ── Store tarball probe (--check-store mode) ─────────────────────────────────── +def _probe_in_stores(spec: dict, arch: str, store_urls: str, + insecure: bool = False) -> Tuple[str, str]: + """_probe_tarball_in_store over every store of a comma-separated list, in + order: the first that has the tarball, else the first store's answer.""" + from bits_helpers.sync import store_urls as _split + urls = _split(store_urls) or [""] + results = [(url, _probe_tarball_in_store(spec, arch, url, insecure)) for url in urls] + for url, (status, detail) in results: + if status == PASS: + return status, detail + (" in %s" % url if len(urls) > 1 else "") + return results[0][1] + + def _probe_tarball_in_store(spec: dict, arch: str, store_url: str, insecure: bool = False) -> Tuple[str, str]: """Probe whether a pre-built tarball for *spec* is available in the remote store. @@ -506,7 +528,7 @@ def _run_check_store_checks(args, specs: dict, own: set, if pkg not in own and pkg not in always_built: continue spec = specs[pkg] - status, detail = _probe_tarball_in_store(spec, arch, store_url, insecure) + status, detail = _probe_in_stores(spec, arch, store_url, insecure) targets.append((pkg, status, detail)) if not targets: @@ -669,10 +691,16 @@ def _run_runner_checks(args) -> List[CheckResult]: checks.append(("disk space (%s)" % work_dir, disk_status, disk_detail)) # ── Remote store ───────────────────────────────────────────────────────── - store_url = (getattr(args, "remoteStore", "") or "").rstrip("/") + from bits_helpers.sync import store_urls insecure = getattr(args, "insecure", False) - store_status, store_detail = _check_store(store_url, insecure) - checks.append(("remote store", store_status, store_detail)) + reads = store_urls(getattr(args, "remoteStore", "")) + writes = [w for w in store_urls(getattr(args, "writeStore", "")) if w not in reads] + for url, label in [(u, "remote store") for u in reads or [""]] + \ + [(u, "write store") for u in writes]: + store_status, store_detail = (check_read_store(url.rstrip("/"), args) if label == "remote store" + else _check_store(url.rstrip("/"), insecure)) + checks.append((label + (" " + url if len(reads) + len(writes) > 1 else ""), + store_status, store_detail)) # ── cvmfs-prepub service (optional) ────────────────────────────────────── prepub_url = (getattr(args, "prepubUrl", "") or "").rstrip("/") diff --git a/bits_helpers/doctor_setup.py b/bits_helpers/doctor_setup.py index 39515ac0..76fe7bb5 100644 --- a/bits_helpers/doctor_setup.py +++ b/bits_helpers/doctor_setup.py @@ -11,11 +11,12 @@ import getpass import importlib.util import os +import re import shutil import subprocess import sys -from bits_helpers.doctor import (PASS, FAIL, WARN, SKIP, _check_compiler, +from bits_helpers.doctor import (PASS, FAIL, WARN, SKIP, _check_compiler, check_read_store, _check_disk_space, _check_host_tool, _check_store) # (import name, pip name). yaml/requests/jinja2 are imported at startup, so bits @@ -59,6 +60,19 @@ def check_modules(modules, missing_status): sys.executable, ", ".join(missing), _pip_hint(missing)) +def check_site(): + """The site configuration files in effect, and where the main values come from.""" + from bits_helpers import site + found = site.files() + if not found: + return WARN, "no site configuration files (bits' own site.d is missing)" + keys = ("store.read", "store.operations", "store.s3_endpoint", "containers.registry", + "containers.tag", "providers.url") + values = ["%s=%s [%s]" % (k, site.get(k), os.path.basename(site.origin(k) or "?")) + for k in keys if site.get(k) is not None] + return PASS, "%s; %s" % (", ".join(found), "; ".join(values) or "no values") + + def check_root(): if os.geteuid() != 0: return PASS, "running as %s" % getpass.getuser() @@ -157,51 +171,60 @@ def check_s3cmd_store(url): return PASS, "%s: s3cmd and ~/.s3cfg present" % url -def _s3_client(): - """boto3 client configured as Boto3RemoteSync does, with short timeouts.""" +def _s3_client(settings): + """boto3 client for one bucket's *settings* (sync.s3_settings), as + Boto3RemoteSync makes it, with short timeouts.""" import boto3 from botocore.config import Config - style = os.environ.get("S3_ADDRESSING_STYLE") + style = settings["addressing_style"] config = Config(connect_timeout=10, read_timeout=10, retries={"max_attempts": 1}, **({"s3": {"addressing_style": style}} if style else {})) - kwargs = {"endpoint_url": os.environ.get("S3_ENDPOINT_URL"), "config": config} - region = os.environ.get("AWS_DEFAULT_REGION") or os.environ.get("AWS_REGION") - if region: - kwargs["region_name"] = region + kwargs = {"endpoint_url": settings["endpoint"], "config": config, + "aws_access_key_id": settings["access_key"], + "aws_secret_access_key": settings["secret_key"]} + if settings["session_token"]: + kwargs["aws_session_token"] = settings["session_token"] + if settings["region"]: + kwargs["region_name"] = settings["region"] return boto3.client("s3", **kwargs) def check_s3_store(url, args, label): """Credentials and bucket access for a b3:// (boto3) store.""" - from bits_helpers.sync import resolve_and_export_s3_config + from bits_helpers.sync import resolve_and_export_s3_config, s3_settings resolve_and_export_s3_config(getattr(args, "s3Endpoint", None), getattr(args, "s3AccessKey", None), getattr(args, "s3SecretKey", None), getattr(args, "s3Region", None), getattr(args, "s3AddressingStyle", None)) - if not (os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY")): - return FAIL, ("%s: no S3 credentials. Set AWS_ACCESS_KEY_ID and " - "AWS_SECRET_ACCESS_KEY, pass --s3-access-key/--s3-secret-key, or " - "put them in ~/.bits/s3keys" % url) + bucket = url[len("b3://"):].split("/", 1)[0] + settings = s3_settings(bucket) + if not (settings["access_key"] and settings["secret_key"]): + key = "BITS_S3_%s_" % re.sub(r"[^A-Za-z0-9]", "_", bucket).upper() + return FAIL, ("%s: no S3 credentials. Set %sACCESS_KEY_ID and %sSECRET_ACCESS_KEY " + "(or a [%s] section in ~/.bits/s3keys) for this bucket, or " + "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, --s3-access-key/" + "--s3-secret-key or ~/.bits/s3keys for all" % (url, key, key, bucket)) if importlib.util.find_spec("boto3") is None: return FAIL, "%s: boto3 missing; %s" % (url, _pip_hint(["boto3"])) - bucket = url[len("b3://"):].split("/", 1)[0] try: - _s3_client().head_bucket(Bucket=bucket) + _s3_client(settings).head_bucket(Bucket=bucket) except Exception as exc: # botocore ClientError, endpoint or network errors return FAIL, "%s: bucket not accessible with these credentials (%s)" % (url, exc) return PASS, "%s: credentials found, bucket accessible (%s)" % (url, label) def run_setup_checks(args): - read = (getattr(args, "remoteStore", "") or "").rstrip("/") - write = (getattr(args, "writeStore", "") or "").rstrip("/") - b3 = any(u.startswith("b3://") for u in (read, write)) + from bits_helpers.sync import store_urls + reads = [u.rstrip("/") for u in store_urls(getattr(args, "remoteStore", ""))] + writes = [u.rstrip("/") for u in store_urls(getattr(args, "writeStore", ""))] + b3 = any(u.startswith("b3://") for u in reads + writes) checks = [ ("python", *check_python()), ("python modules", *check_modules(_MODULES, FAIL)), ("python modules (signing)", *check_modules(_SIGNING_MODULES, WARN)), ("python modules (b3:// store)", *check_modules(_B3_MODULES, FAIL if b3 else WARN)), + ("site configuration", *check_site()), ("user", *check_root()), ("git", *_check_host_tool("git")), ("C++ compiler", *_check_compiler()), @@ -210,15 +233,18 @@ def run_setup_checks(args): work_dir = getattr(args, "workDir", "sw") or "sw" checks.append(("disk space (%s)" % work_dir, *_check_disk_space(work_dir, float(getattr(args, "minDisk", None) or 10.0)))) - for url, label in ((read, "read store"), (write, "write store")): - if not url or (label == "write store" and url == read): - continue + stores = [(u, "read store") for u in reads] + \ + [(u, "write store") for u in writes if u not in reads] + for url, label in stores: + label = label + (" " + url if len(stores) > 1 else "") if url.startswith("b3://"): checks.append((label, *check_s3_store(url, args, label))) elif url.startswith("s3://"): checks.append((label, *check_s3cmd_store(url))) + elif label.startswith("read store"): + checks.append((label, *check_read_store(url, args))) else: checks.append((label, *_check_store(url, getattr(args, "insecure", False)))) - if not read and not write: + if not stores: checks.append(("stores", SKIP, "no --remote-store / --write-store configured")) return checks diff --git a/bits_helpers/download.py b/bits_helpers/download.py index 0550925d..73946b02 100644 --- a/bits_helpers/download.py +++ b/bits_helpers/download.py @@ -6,6 +6,7 @@ except ImportError: from hashlib import md5 as md5adder from os.path import abspath, join, exists, dirname, basename +from glob import glob from os import rename, unlink import os import re @@ -14,6 +15,7 @@ from urllib.request import urlopen, Request from urllib.error import URLError import base64 +import hashlib from time import time, sleep from types import SimpleNamespace from bits_helpers.log import error, warning, debug, info @@ -180,6 +182,128 @@ def downloadUrllib2(source, destDir, work_dir, dest_filename=None): return False return True +def fetch_fresh(url, dest_dir, name, work_dir=None): + """Download *url* to /, replacing any copy already there + (downloadUrllib2 alone keeps an existing file). Raises OSError when the + download fails; an older copy is then left alone, never returned.""" + os.makedirs(dest_dir, exist_ok=True) + part = "%s.%d.%s.part" % (name, os.getpid(), os.urandom(4).hex()) + path = join(dest_dir, part) + try: + downloadUrllib2(url, dest_dir, work_dir, dest_filename=part) + if not os.path.isfile(path): + raise OSError("could not download %s" % url) + os.replace(path, join(dest_dir, name)) + finally: + for left in [path] + glob(path + ".*.tmp"): # (downloadUrllib2's own) + if os.path.exists(left): + unlink(left) + return join(dest_dir, name) + + +def _http_get(url, headers, timeout=60): + """``(status, body, etag)`` for a GET of *url*; 304 gives no body. Raises + OSError on any other failure.""" + from http.client import HTTPException + from urllib.error import HTTPError + try: + resp = urlopen(Request(url, headers=headers), timeout=timeout) + with resp: + return resp.status, resp.read(), resp.headers.get("ETag") + except HTTPError as e: + if e.code == 304: + return 304, None, headers.get("If-None-Match") + raise OSError("HTTP %s for %s" % (e.code, url)) + except (URLError, ValueError, HTTPException) as e: # (a truncated body too) + raise OSError("could not fetch %s: %s" % (url, e)) + + +def fetch_current(url, dest_dir, name, work_dir=None): + """/ as current as *url*, downloading only what changed. + + The ETag of the last download is kept beside the file with the file's + sha256, and sent as If-None-Match: the server answers 304 when its copy is + the same. A local copy that no longer matches the sha256 it was kept with, + or a URL that is not plain http(s) (credentials in it, another scheme), is + fetched in full (fetch_fresh). Raises OSError when the fetch fails; an older + copy is then never returned. + """ + if not url.startswith(("http://", "https://")) or urlAuthRe.match(url): + return fetch_fresh(url, dest_dir, name, work_dir) + os.makedirs(dest_dir, exist_ok=True) + path, kept = join(dest_dir, name), join(dest_dir, name + ".etag") + etag = None + try: + with open(kept) as fh: + tag, digest = fh.read().split("\n")[:2] + with open(path, "rb") as fh: + if hashlib.sha256(fh.read()).hexdigest() == digest: + etag = tag or None + except (OSError, ValueError): + etag = None + headers = {"Cache-Control": "no-cache"} + if etag: + headers["If-None-Match"] = etag + status, body, new_tag = _http_get(url, headers) + if status == 304: + debug("%s unchanged (ETag %s)", url, etag) + return path + if status != 200 or body is None: + raise OSError("HTTP %s for %s" % (status, url)) + part = "%s.%d.%s.part" % (path, os.getpid(), os.urandom(4).hex()) + try: + with open(part, "wb") as fh: + fh.write(body) + os.replace(part, path) + finally: + if os.path.exists(part): + unlink(part) + try: + # A weak tag (W/"…") may be matched by a copy whose bytes differ. + if new_tag and not new_tag.startswith("W/"): + with open(kept + ".part", "w") as fh: + fh.write("%s\n%s\n" % (new_tag, hashlib.sha256(body).hexdigest())) + os.replace(kept + ".part", kept) + elif os.path.exists(kept): + unlink(kept) + except OSError: + pass # only the next fetch gets slower + return path + + +_SIGNED = {} # (url, dest dir) -> local path, or the error + + +def fetch_signed_manifest(src, dest_dir, work_dir=None): + """A local path for a signed manifest *src* with its detached .sig beside it. + + A local path is returned as it is. A URL is brought up to date, manifest and + signature both (fetch_current: a 304 when unchanged), once per process + (every caller in one run verifies the same copy). A failed fetch raises OSError, also for later callers in the run: an + older copy is never used instead, since it may still vouch for a hash that + has been revoked since, or miss the ones certified since. + """ + if "://" not in str(src): + return src + key = (src, os.path.abspath(dest_dir)) + got = _SIGNED.get(key) + if got is None: + # The URL's own name, so two stores' manifests of one name never share + # a local copy. + name = "%s.%s" % (hashlib.sha256(src.encode()).hexdigest()[:12], + basename(src.split("?")[0]) or "trust-manifest.json") + try: + # (Re-signed in between, the pair does not match: it fails closed.) + got = fetch_current(src, dest_dir, name, work_dir) + fetch_current(src + ".sig", dest_dir, name + ".sig", work_dir) + except OSError as exc: + got = exc + _SIGNED[key] = got + if isinstance(got, Exception): + raise got + return got + + # Download a files from a git url. We do not clone the remote reposiotory, but # we simply pull the branch we are interested in and then we drop all the git # information while creating a tarball. The syntax to define a repository is @@ -391,7 +515,7 @@ def downloadFile(source, dest, work_dir): def download(source, dest, work_dir, checksum=None, enforce_mode="off", - sync_helper=None): + sync_helper=None, progress=None): """Download *source* into *dest*, optionally verifying its checksum. Parameters @@ -423,6 +547,9 @@ def download(source, dest, work_dir, checksum=None, enforce_mode="off", with the same remote store) can skip the upstream download. Pass ``None`` (the default) to preserve the previous behaviour. + progress: + Optional callback receiving the source location before retrieval, + including local-cache and remote-store attempts. """ noCmssdtCache = True if 'no-cmssdt-cache=1' in source else False isCmsdistGenerated = True if 'cmdist-generated=1' in source else False @@ -504,16 +631,23 @@ def download(source, dest, work_dir, checksum=None, enforce_mode="off", break fetched_from_upstream = False try: - if not exists(realFile): + if exists(realFile): + if progress: + progress("local cache {} (source: {})".format(realFile, source)) + else: # Before hitting the upstream URL, check whether the remote store # already has an archived copy of this source. This makes rebuilds # resilient to upstream URL disappearance. if sync_helper is not None: debug("Trying remote store for source file: %s", filename) + if progress and getattr(sync_helper, "remoteStore", ""): + progress("{} (source: {})".format(sync_helper.remoteStore, source)) sync_helper.fetch_source(url_checksum, filename, downloadDir) if not exists(realFile): debug("Trying to fetch source file: %s", source) + if progress: + progress(source) downloadHandler(source, downloadDir, work_dir) fetched_from_upstream = True finally: diff --git a/bits_helpers/fetch_cache.py b/bits_helpers/fetch_cache.py new file mode 100644 index 00000000..5917a9d4 --- /dev/null +++ b/bits_helpers/fetch_cache.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Cache of the sources a build downloads itself (issue #129). + +A recipe with ``fetch_cache: true``, built with defaults that name a cache +(``system: fetch_cache: ``), gets ``$BITS_FETCH_CACHE``: a directory filled +from the store before its build and saved to it after a successful one. The +recipe points its tool there (Bazel ``--repository_cache``, a process directory, +...). The cache is not part of the package hash. + +In the store each file is kept once, by its sha256, with the source mirror's +layout and upload rules (``SOURCES/cache///fetched``), so versions +share it. The list of files of a package version on an architecture is written +once, by the first build that saves it, and never changed: later builds restore +exactly those files. build_template.sh lists what the build left in the +directory (``$BUILDROOT/fetched-sources.sha256``); that is what is saved. +""" + +import hashlib +import json +import os +import re +import shutil +import tempfile + +from bits_helpers.log import debug, info, warning +from bits_helpers.sync import source_sync_for, sources_redistributable + +LISTING = "fetched-sources.sha256" +BLOB = "fetched" +INDEX = "index.json" +_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SHA = re.compile(r"^[0-9a-f]{64}$") + + +def cache_name(value): + """The cache's name from the defaults' ``system: fetch_cache:``, or "" (off).""" + if value is None or value == "": + return "" + name = str(value).strip() + if isinstance(value, bool) or not _NAME.match(name): + warning("Ignoring system: fetch_cache: %r: it takes the cache's name ([A-Za-z0-9._-])", + value) + return "" + return name + + +def enabled(spec, name): + """Does *spec* use the cache *name*? Only if the recipe asks and the defaults allow.""" + if not spec.get("fetch_cache"): + return False + if not name: + debug("%s asks for fetch_cache, but the defaults name no cache (system: fetch_cache:)", + spec["package"]) + return bool(name) + + +def relative_dir(spec, arch): + """$BITS_FETCH_CACHE, relative to the work directory.""" + return os.path.join("FETCH_CACHE", arch, spec["package"], spec["version"]) + + +def _index_key(name, spec, arch): + return hashlib.sha256(("bits-fetch-cache\0%s\0%s\0%s\0%s" % ( + name, arch, spec["package"], spec["version"])).encode()).hexdigest() + + +def _sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _safe_path(rel): + """*rel* if it is a plain relative path inside the cache, else None.""" + parts = rel.replace("\\", "/").split("/") + if rel.startswith("/") or any(p in ("", ".", "..") for p in parts): + return None + return os.path.join(*parts) + + +def _inside(directory, path): + """Is *path* inside *directory* once symlinks are resolved?""" + top = os.path.realpath(directory) + return os.path.realpath(path).startswith(top + os.sep) + + +def _scratch(work_dir): + """A temporary directory next to the caches (same filesystem, not listed).""" + top = os.path.join(work_dir, "FETCH_CACHE") + os.makedirs(top, exist_ok=True) + return tempfile.TemporaryDirectory(prefix=".tmp-", dir=top) + + +class State: + """What restore() found: the stored list ({path: sha256}, None if there is + none or it could not be read) and the files it got from the store.""" + + def __init__(self, files=None, confirmed=()): + self.files, self.confirmed = files, set(confirmed) + + +def _fetch(sync, key, filename, tmp): + """The stored file *key*/*filename* in *tmp*, or None.""" + path = os.path.join(tmp, filename) + if os.path.exists(path): + os.unlink(path) # some backends refuse to overwrite + return path if sync.fetch_source(key, filename, tmp) and os.path.isfile(path) else None + + +def restore(spec, sync_helper, work_dir, name, arch): + """Fill the package's cache directory with its stored files. Never raises: on a + problem the build downloads what it misses itself.""" + directory = os.path.join(work_dir, relative_dir(spec, arch)) + sync = source_sync_for(spec, sync_helper) + state = State() + try: + os.makedirs(directory, exist_ok=True) + with _scratch(work_dir) as tmp: + index = _fetch(sync, _index_key(name, spec, arch), INDEX, tmp) + if index is None: + return state + with open(index) as f: + entries = {str(e["path"]): (str(e["sha256"]), int(e["size"])) for e in json.load(f)["files"]} + state.files = {} + for path, (sha, size) in sorted(entries.items()): + rel = _safe_path(path) + dest = os.path.join(directory, rel or "") + if rel is None or not _SHA.match(sha) or not _inside(directory, dest): + warning("Fetch cache of %s: ignoring the entry %r", spec["package"], path) + continue + state.files[rel] = sha + if os.path.isfile(dest) and os.path.getsize(dest) == size: + continue # left by an earlier build in this (trusted) work dir + try: + blob = _fetch(sync, sha, BLOB, tmp) + if blob is None or _sha256(blob) != sha: + warning("Fetch cache of %s: %s is %s", spec["package"], rel, + "not in the store" if blob is None else "corrupt in the store; not used") + continue + os.makedirs(os.path.dirname(dest), exist_ok=True) + os.replace(blob, dest) + state.confirmed.add(sha) + except Exception as exc: # pylint: disable=broad-except + warning("Fetch cache of %s: could not restore %s (%s)", spec["package"], rel, exc) + info("Fetch cache of %s %s: %d files (%d from the store)", spec["package"], + spec["version"], len(state.files), len(state.confirmed)) + except Exception as exc: # pylint: disable=broad-except + warning("Fetch cache of %s: could not restore it (%s); the build downloads what it needs", + spec["package"], exc) + return state + + +def _listing(path): + """{path: sha256} from build_template.sh's sha256sum listing.""" + files = {} + with open(path) as f: + for line in f: + sha, _, rel = line.rstrip("\n").partition(" ") + rel = _safe_path(rel[2:] if rel.startswith("./") else rel) + if rel and _SHA.match(sha): + files[rel] = sha + return files + + +def _upload(sync, path, key, filename, tmp): + """Upload *path* as *key*/*filename* (rsync keeps the local name: link it so).""" + staged = os.path.join(tmp, filename) + if os.path.lexists(staged): + os.unlink(staged) + try: + os.link(path, staged) + except OSError: + shutil.copyfile(path, staged) + sync.upload_source(staged, key, filename) + + +def save(spec, sync_helper, work_dir, name, arch, state, listing): + """Store the files the build left in its cache directory (*listing*): the + list, if none is stored yet, and the files it names that the store may lack. + Never raises.""" + sync = source_sync_for(spec, sync_helper) + if not getattr(sync, "writeStore", "") or not os.path.isfile(listing): + return + if not sources_redistributable(spec): + debug("Fetch cache of %s not saved: redistributable forbids its sources", spec["package"]) + return + directory = os.path.join(work_dir, relative_dir(spec, arch)) + try: + current = _listing(listing) + files = current if state.files is None else state.files + extra = sorted(p for p, s in current.items() if state.files is not None + and state.files.get(p) != s) + if extra: + warning("Fetch cache of %s %s: %d files the build downloaded are not in it (its list " + "is written once per version and architecture): %s", spec["package"], + spec["version"], len(extra), ", ".join(extra[:5]) + (", ..." if len(extra) > 5 else "")) + with _scratch(work_dir) as tmp: + done = set(state.confirmed) + for rel, sha in sorted(files.items()): + path = os.path.join(directory, rel) + if sha in done or current.get(rel) != sha: + continue # in the store, or not here to upload + _upload(sync, path, sha, BLOB, tmp) + done.add(sha) + if state.files is not None or not files: + return # stored once, and never empty + index = os.path.join(tmp, "new-" + INDEX) + with open(index, "w") as f: + json.dump({"cache": name, "architecture": arch, "package": spec["package"], + "version": spec["version"], "hash": spec.get("hash", ""), + "files": [{"path": p.replace(os.sep, "/"), "sha256": s, + "size": os.path.getsize(os.path.join(directory, p))} + for p, s in sorted(files.items())]}, f, indent=1) + _upload(sync, index, _index_key(name, spec, arch), INDEX, tmp) + info("Fetch cache of %s %s: saved %d files", spec["package"], spec["version"], len(files)) + except (Exception, SystemExit) as exc: # upload_source may dieOnError + warning("Fetch cache of %s: could not save it (%s)", spec["package"], exc) diff --git a/bits_helpers/forge.py b/bits_helpers/forge.py index bee0d0f4..7e8a9cda 100644 --- a/bits_helpers/forge.py +++ b/bits_helpers/forge.py @@ -16,6 +16,7 @@ import os from bits_helpers.log import warning +from bits_helpers import site def load_admin_policy(source) -> dict: @@ -241,7 +242,7 @@ def gitlab_identify(api_url, token, timeout=15): return None -DEFAULT_GITLAB_TOKEN_FILE = "~/.bits/gitlab-token" +DEFAULT_GITLAB_TOKEN_FILE = site.get("paths.gitlab_token_file") or "~/.bits/gitlab-token" def resolve_gitlab_token(explicit=None): @@ -384,13 +385,63 @@ def gitlab_create_merge_request(api_url, token, project, source_branch, target_b return {"iid": data.get("iid"), "web_url": data.get("web_url")} -def gitlab_merge_mr(api_url, token, project, iid, timeout=30) -> dict: - """Merge an MR (PUT …/merge). Raises on failure (e.g. not mergeable).""" +# Merge statuses that pass by themselves: GitLab is still working out whether +# the MR can merge (it rechecks every open MR when the target branch moves, e.g. +# after another certification merged). detailed_merge_status, and the older +# merge_status values. +_MERGE_PENDING = frozenset({"unchecked", "checking", "preparing", "approvals_syncing", + "mergeable", "can_be_merged", "cannot_be_merged_recheck"}) + + +def gitlab_merge_mr(api_url, token, project, iid, timeout=30, sha=None, wait=300, + sleep=None, clock=None) -> dict: + """Merge an MR (PUT …/merge). Raises on failure (e.g. not mergeable). + + A refusal while GitLab is still checking the MR's mergeability (or still + merging it: state locked) is retried for up to *wait* seconds; any other + refusal (conflict, draft, closed, pipeline required…) raises at once, naming + the status. With *sha*, only that head commit is merged (GitLab refuses with + 409 if the MR moved on since it was validated), and an MR found merged + already counts only if *sha* is what was merged. + """ + import time import requests - resp = requests.put("%s/merge_requests/%s/merge" % (_gl_project_url(api_url, project), iid), - headers={"PRIVATE-TOKEN": token}, timeout=timeout) - _gl_ok(resp) - return resp.json() or {} + sleep = sleep or time.sleep + clock = clock or time.monotonic + url = "%s/merge_requests/%s" % (_gl_project_url(api_url, project), iid) + headers = {"PRIVATE-TOKEN": token} + deadline = clock() + wait + pause = 3 + while True: + try: + resp = requests.put(url + "/merge", headers=headers, + json={"sha": sha} if sha else None, timeout=timeout) + except requests.exceptions.Timeout: + resp = None # it may have merged: look at the MR + if resp is not None and resp.status_code not in (405, 406, 422): + _gl_ok(resp) + return resp.json() or {} + info = requests.get(url, headers=headers, timeout=timeout) + _gl_ok(info) + mr = info.json() or {} + status = mr.get("detailed_merge_status") or mr.get("merge_status") or "?" + code = resp.status_code if resp is not None else "timeout" + if mr.get("state") == "merged": + if sha and mr.get("sha") == sha: + return mr + raise RuntimeError("MR !%s is merged already, %s" % ( + iid, "at %s, not the validated %s" % (mr.get("sha"), sha) if sha + else "and which commit was validated is not known")) + if not (mr.get("state") == "locked" + or (mr.get("state") == "opened" and status in _MERGE_PENDING)): + raise RuntimeError("GitLab refused to merge MR !%s: %s (state %s, %s)" + % (iid, status, mr.get("state"), code)) + if clock() + pause > deadline: + raise RuntimeError("GitLab refused to merge MR !%s for %ds: still %s" + % (iid, wait, status)) + warning("MR !%s not mergeable yet (%s); trying again in %ds", iid, status, pause) + sleep(pause) + pause = min(pause * 2, 30) def gitlab_mr_author(api_url, token, project, iid, timeout=15): diff --git a/bits_helpers/gc.py b/bits_helpers/gc.py index 5d1ce20f..a2e4df48 100644 --- a/bits_helpers/gc.py +++ b/bits_helpers/gc.py @@ -189,46 +189,47 @@ def delete_fn(keys): def _localize_manifest(src, work_dir): - """Return a local path to the signed manifest, downloading it (and its + """Return a local path to the signed manifest, fetching it afresh (and its detached ``.sig``) when *src* is a URL. The signature is verified from the - ``.sig`` sitting next to the manifest, so both must land together. Raising on - a failed fetch keeps gc fail-closed (the caller refuses to sweep). + ``.sig`` sitting next to the manifest, so both must land together. A failed + fetch raises, never falls back to an older copy (whose roots would miss + what was certified since): the caller refuses to sweep. """ - if "://" not in str(src): - return src - from bits_helpers.download import downloadUrllib2 - dest = os.path.join(work_dir, "MANIFESTS", "gc") - os.makedirs(dest, exist_ok=True) - name = os.path.basename(src.split("?")[0]) or "trust-manifest.json" - downloadUrllib2(src, dest, work_dir, dest_filename=name) - downloadUrllib2(src + ".sig", dest, work_dir, dest_filename=name + ".sig") - return os.path.join(dest, name) + from bits_helpers.download import fetch_signed_manifest + return fetch_signed_manifest(src, os.path.join(work_dir, "MANIFESTS", "gc"), work_dir) def doGc(args, parser): - """CLI entrypoint for ``bits gc``.""" + """CLI entrypoint for ``bits gc``: each store of the (comma-separated) list is + swept against its own signed manifest (--trust-manifest, one per store).""" from bits_helpers.publish import _normalize_s3_store - from bits_helpers.sync import remote_from_url + from bits_helpers.sync import remote_from_url, store_urls from bits_helpers.log import banner - store = _normalize_s3_store(args.gcStore) - writer = remote_from_url(store, store, args.architecture, args.workDir) - bucket = getattr(writer, "remoteStore", None) or getattr(writer, "writeStore", None) - s3 = writer.s3 - objects = gather_s3_store_objects(s3, bucket, args.architecture) - try: - manifest_path = _localize_manifest(args.trustManifest, args.workDir) - except Exception as exc: - parser.error("could not fetch trust manifest %s: %s" - % (args.trustManifest, exc)) - plan = collect_garbage( - objects, manifest_path, - grace_seconds=int(args.graceDays * 86400), - delete_fn=make_s3_deleter(s3, bucket, args.architecture), - dry_run=args.dryRun, allow_empty=args.allowEmpty, - architecture=args.architecture) - if not plan["verified"]: - parser.error("common manifest %s did not verify; nothing swept" % args.trustManifest) - verb = "would sweep" if args.dryRun else "swept" - banner("gc: %s %d object(s); kept %d, %d too young, %d unsafe/non-store", - verb, len(plan["delete"]), plan["kept"], plan["young"], plan["unsafe"]) + # Each store's own certified set: deleting must never use another store's roots. + stores = store_urls(args.gcStore) + roots = [m.strip() for m in str(args.trustManifest or "").split(",") if m.strip()] + if len(stores) != len(roots): + parser.error("gc needs one --trust-manifest per store, in the same order " + "(%d store(s), %d manifest(s))" % (len(stores), len(roots))) + for url, root in zip(stores, roots): + try: + manifest_path = _localize_manifest(root, args.workDir) + except Exception as exc: + parser.error("could not fetch trust manifest %s: %s" % (root, exc)) + store = _normalize_s3_store(url) + writer = remote_from_url(store, store, args.architecture, args.workDir) + bucket = getattr(writer, "remoteStore", None) or getattr(writer, "writeStore", None) + s3 = writer.s3 + objects = gather_s3_store_objects(s3, bucket, args.architecture) + plan = collect_garbage( + objects, manifest_path, + grace_seconds=int(args.graceDays * 86400), + delete_fn=make_s3_deleter(s3, bucket, args.architecture), + dry_run=args.dryRun, allow_empty=args.allowEmpty, + architecture=args.architecture) + if not plan["verified"]: + parser.error("common manifest %s did not verify; nothing swept in %s" % (root, store)) + verb = "would sweep" if args.dryRun else "swept" + banner("gc %s: %s %d object(s); kept %d, %d too young, %d unsafe/non-store", + store, verb, len(plan["delete"]), plan["kept"], plan["young"], plan["unsafe"]) diff --git a/bits_helpers/hashing.py b/bits_helpers/hashing.py index c5cdbf04..f853d6d0 100644 --- a/bits_helpers/hashing.py +++ b/bits_helpers/hashing.py @@ -33,6 +33,11 @@ # view: what the package exports to a release's merged view — presentation # only, applied at publish time from the build manifest. "view", + # references: CPEs and purls naming the package for vulnerability scanners + # (written into the SBOM); they never affect the build. + "references", + # sbom: tool marks a host tool, run but never shipped: scans leave it out. + "sbom", }) # Source-selection keys are ALSO dropped from the recipe TEXT hash — not because @@ -285,6 +290,8 @@ def h_all(data): # pylint: disable=function-redefined if not package.startswith("defaults-"): for hook_name in sorted(spec.get("hook", {})): + if hook_name == "SYMLINK_STAGE": # runs on reuse only: no part of the build + continue h_all("hook:" + hook_name + "=" + str(spec["hook"][hook_name])) for hook_name in sorted(spec.get("hook_params", {})): h_all("hook_params:" + hook_name + "=" + str(spec["hook_params"][hook_name])) diff --git a/bits_helpers/initdotsh.py b/bits_helpers/initdotsh.py index fe75ad61..b8267b36 100644 --- a/bits_helpers/initdotsh.py +++ b/bits_helpers/initdotsh.py @@ -99,9 +99,11 @@ def _dep_init_path(dep): # sourcing (and restore after) so its own and its transitive deps' paths land # on CVMFS. Per-DEPENDENCY, so a legacy-built package can consume a reused dep. # Needs /cvmfs mounted in the build container (no modulecmd required). - _reqs = list(spec.get("requires", ())) + # A not-needed dependency (build-only, of a reused package) is not installed. + _reqs = [d for d in spec.get("requires", ()) if not specs[d].get("not_needed")] _reused_set = {d for d in _reqs - if reuse_cvmfs_base and specs[d].get("reuse_module_id")} + if reuse_cvmfs_base and specs[d].get("reuse_module_id") + and not specs[d].get("reuse_symlinked")} def _reused_dep_lines(d): # Point the deployed init.sh's "$WORK_DIR/$BITS_ARCH_PREFIX" at the CVMFS @@ -120,7 +122,7 @@ def _reused_dep_lines(d): 'unset _bits_swd _bits_sap', ] - if _reused_set: + if _reused_set or any(specs[d].get("reuse_symlinked") for d in _reqs): # Emit deps in topological order (prerequisites first) so a dep set up # before a reused dep whose deployed init.sh transitively references it — # e.g. a locally-built bits-recipe-tools before a reused CMake — sets its @@ -160,7 +162,7 @@ def _reused_dep_lines(d): ' *) export PKG_CONFIG_PATH="$_bits_rpc${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" ;;', 'esac', 'unset _bits_rpc _bits_root _bits_pcd _bits_pc', - ]) + ] if _reused_roots else []) else: lines.extend(_dep_init_path(dep) for dep in _reqs) diff --git a/bits_helpers/log.py b/bits_helpers/log.py index 946c3333..5ad042e8 100644 --- a/bits_helpers/log.py +++ b/bits_helpers/log.py @@ -116,7 +116,7 @@ def end(self, msg="", error=False): self.erase() if msg: sys.stderr.write(": {}{}\033[m".format("\033[31m" if error else "\033[32m", msg)) - sys.stderr.write("\n") + sys.stderr.write("\033[K\n") # no spaces left from the spinner sys.stderr.flush() diff --git a/bits_helpers/manifest.py b/bits_helpers/manifest.py index bc064601..aa7f6865 100644 --- a/bits_helpers/manifest.py +++ b/bits_helpers/manifest.py @@ -60,6 +60,8 @@ "config_dir": str, # absolute path to the .bits checkout "config_commit": str, # BITS_DIST_HASH of the config repo "system_packages": [str], # taken from the system, not built (v4+) + "system_package_info": {name: {"version": str, "references": [str], "sbom": "tool"?}}, + # their versions, when their checks said (v4+) "cvmfs_templates": {str: str}, # this build's CVMFS layout (optional); # bits cvmfs publish uses it for all packages "providers": [ProviderEntry], @@ -84,6 +86,8 @@ "hash": str, # content-addressable build hash "commit_hash": str, # source commit hash (or "0") "outcome": "already_installed" | "from_store" | "built_from_source", + "rebuilt_because": str, # built_from_source only, when known: + # why no tarball was reused "tarball": str | null, # tarball filename "tarball_sha256": str | null, # sha256: of the tarball, if present "source_checksums": [SourceEntry], # per-source archive integrity anchors @@ -93,7 +97,13 @@ "build_requires": [str], # direct build-only dependencies (v4+) "source": str, # git repository URL, "" if none (v4+) "tag": str, # git tag/branch built, "" if none (v4+) + "references": [str], # CPEs / purls naming it for scanners; only when any + "pinned_by": str, # "defaults:" or "requires:" that + # set its version; only when pinned + "sbom": "tool", # a host tool, never shipped; only when so "built_by": str | None, # user@host that compiled this hash; null unless built_from_source + "symlinked_from": str, # already_installed by --symlink-packages-from: the + "symlink_policy": str, # deployed package linked, and the policy; only then "completed_at": ISO-8601 } @@ -276,6 +286,57 @@ def _dep_names(names) -> list: return out +_PYPI_NAME = re.compile(r"""^\s*(?:export\s+)?PYPI_NAME=["']?([A-Za-z0-9._-]+)["']?\s*(?:#.*)?$""", re.M) +_PIP_RECIPE = re.compile(r"^[^#\n]*bits-include\s+PythonPipRecipe\b", re.M) +_CPE_FIELDS = re.compile(r"(? list: + """The recipe's `references:` (``cpe:2.3:…`` and ``pkg:…`` strings, with + %(version)s, %(name)s and %(tag)s filled in). A PythonPipRecipe that names + no purl gets ``pkg:pypi/@``.""" + from urllib.parse import quote + name, version = str(spec.get("package") or ""), str(spec.get("version") or "") + fill = {"%(version)s": version, "%(name)s": name, "%(package)s": name, + "%(tag)s": str(spec.get("tag") or "")} + raw = spec.get("references") or [] + out = [] + for r in [raw] if isinstance(raw, str) else raw: + if not isinstance(r, str) or not r.strip(): + continue + r = r.strip() + if r.startswith("cpe:2.3:"): + ref = _cpe(r, fill) + elif r.startswith("pkg:"): + ref = r + for k, v in fill.items(): + ref = ref.replace(k, quote(v, safe=".-_~")) + else: + ref = None + if ref is None: + warning("%s: ignoring reference %r: not a cpe:2.3: CPE nor a pkg: purl", name, r) + elif ref not in out: + out.append(ref) + body = str(spec.get("recipe") or "") + if version and not any(r.startswith("pkg:") for r in out) and _PIP_RECIPE.search(body): + m = _PYPI_NAME.search(body) + pypi = (m.group(1) if m else name).lower().replace("_", "-") + out.append("pkg:pypi/%s@%s" % (quote(pypi, safe=".-"), quote(version, safe=".-_~"))) + return out + + def _patch_entries(spec: dict) -> list: """Return ``[{name, checksum}]`` for each patch the recipe applied. @@ -295,6 +356,11 @@ def _patch_entries(spec: dict) -> list: return result +def sbom_tool(spec): + """Is *spec* a host tool (recipe ``sbom: tool``): run, never shipped with the stack?""" + return str(spec.get("sbom") or "").strip().lower() == "tool" + + # ── BuildManifest ───────────────────────────────────────────────────────────── class BuildManifest: @@ -412,12 +478,16 @@ def add_providers(self, provider_dirs: dict) -> None: # ── Package recording ───────────────────────────────────────────────────── - def set_system_packages(self, names, edges=None) -> None: + def set_system_packages(self, names, edges=None, info=None) -> None: """Record the packages taken from the system (not built) and, per built package, its (runtime, build) dependencies among them: the build drops - those from the specs' requires, so the SBOM graph needs them here.""" + those from the specs' requires, so the SBOM graph needs them here. + *info* gives the versions (and CPEs/purls) known for them.""" with self._lock: self._data["system_packages"] = sorted(names or ()) + known = {n: i for n, i in (info or {}).items() if i.get("version") or i.get("sbom")} + if known: + self._data["system_package_info"] = known self._sys_edges = dict(edges or {}) def add_package( @@ -448,6 +518,7 @@ def add_package( """ from bits_helpers.sync import redistributable_forms _forms = redistributable_forms(spec.get("redistributable")) + _refs = _references(spec) entry = { "package": spec.get("package", ""), "version": spec.get("version", ""), @@ -515,6 +586,10 @@ def add_package( "build_requires": _dep_names( list(spec.get("build_requires") or []) + list(self._sys_edges.get(spec.get("package"), ((), ()))[1])), + **({"references": _refs} if _refs else {}), + **({"pinned_by": spec["pinned_by"]} if spec.get("pinned_by") else {}), + # A host tool (recipe `sbom: tool`): run, never shipped; scans leave it out. + **({"sbom": "tool"} if sbom_tool(spec) else {}), "source": _no_userinfo(spec.get("source") or ""), "tag": str(spec.get("tag") or ""), # built_by identifies the host that actually compiled this hash. @@ -523,6 +598,13 @@ def add_package( # build's manifest, so leave it null rather than claim it. "built_by": _builder_id() if outcome == "built_from_source" else None, "completed_at": _now_iso(), + # Why a package was built here rather than reused (additive). + **({"rebuilt_because": spec["reuse_note"]} + if outcome == "built_from_source" and spec.get("reuse_note") else {}), + # Linked from a deployment (--symlink-packages-from), not installed here. + **({"symlinked_from": spec.get("symlink_source") or "", + "symlink_policy": spec.get("symlink_policy") or "strict"} + if spec.get("reuse_symlinked") else {}), } with self._lock: self._data["packages"].append(entry) @@ -533,6 +615,11 @@ def add_package( # ── Lifecycle ───────────────────────────────────────────────────────────── + def packages(self) -> list: + """The package entries recorded so far (copies).""" + with self._lock: + return [dict(e) for e in self._data["packages"]] + def complete(self) -> None: """Mark the manifest as successfully completed and write a final save.""" with self._lock: diff --git a/bits_helpers/matchers.py b/bits_helpers/matchers.py index 05ed4355..94cbaa7a 100644 --- a/bits_helpers/matchers.py +++ b/bits_helpers/matchers.py @@ -199,6 +199,10 @@ def resolve_variables(variables, flavours, architecture, defaults): Precedence (low -> high): predefined arch vars < CLI flavours < defaults-file entries, except that a CLI flavour always wins over a defaults entry of the same name (an explicit override) while remaining visible to every gate. + + ``name: {value: V}`` is a plain entry too; ``export: false`` beside the value + only keeps a CLI flavour of that name out of the build environment (see + exported_flavours). """ flavours = flavours or {} resolved = OrderedDict() @@ -211,11 +215,31 @@ def resolve_variables(variables, flavours, architecture, defaults): if _matcher_active(str(entry["when"]), architecture, defaults, resolved): resolved[name] = entry.get("value", True) # inactive -> leave undefined (falsy) + elif isinstance(entry, dict) and "value" in entry: + resolved[name] = entry["value"] else: resolved[name] = entry return resolved +def exported_flavours(variables, flavours): + """The CLI flavours (``--set``) that go into the build environment, and so + into every package's hash: all but those of a defaults ``variables:`` entry + declared ``export: false``. Those set only the variable, whose value enters + the hash of the recipes whose text uses ``%(NAME)s``, as any defaults + variable does (e.g. an Athena ref, used by two recipes alone).""" + unexported = set() + for name, entry in (variables or {}).items(): + if isinstance(entry, dict) and "export" in entry: + dieOnError(not isinstance(entry["export"], bool), + "variables: %s: export must be true or false, not %r" % (name, entry["export"])) + dieOnError("value" not in entry and "when" not in entry, + "variables: %s: export needs a value" % name) + if entry["export"] is False: + unexported.add(name) + return OrderedDict((k, v) for k, v in (flavours or {}).items() if k not in unexported) + + def filterByArchitectureDefaults(arch, defaults, requires, default_vars=None, version=None): """Yield requirements from *requires* that are satisfied by *arch*/*defaults*. diff --git a/bits_helpers/modulefile_env.py b/bits_helpers/modulefile_env.py new file mode 100644 index 00000000..eca0f5f1 --- /dev/null +++ b/bits_helpers/modulefile_env.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""The environment a release's bits modulefiles set, read without Tcl. + +A release's merged view carries a setup.sh (cvmfs_publish.write_view_setup). +With what this module reads from the release's modulefiles in it as well, that +setup.sh gives what loading the release's modules does (the path entries the +view collapses being the view's), and `bits setenv` can source it instead of +running modulecmd over every modulefile. + +Only the forms bits writes are understood: bits-recipe-tools' ModuleRecipe and +the plain setenv / prepend-path / append-path lines recipes add. Anything else +is reported as a problem: the result is then incomplete, and the release keeps +the modulecmd path. +""" + +import posixpath +import re + +from bits_helpers.view_cmd import COLLAPSE_VARS + +BASE = "@B@" # $::env(BASEDIR): the packages directory of the tree +VIEW = "@V@" # the merged view +INHERITED = None # where a path variable's value from before goes + +# Set by modulecmd or bits for themselves, not by the software. +_IGNORED_SETENV = ("BITS_VIEW",) +# What setup.sh / setup.csh can write without quoting. +_SAFE_VALUE = re.compile(r"^[\w@./:+,=%-]*$") + +_WORD = r'(?:"[^"]*"|\{[^{}]*\}|[^\s{}"\[\]]+)' +_SKIP = re.compile(r"^(?:#|module-whatis\b|conflict\b|module-version\b|module-alias\b)") +_SET = re.compile(r"^set\s+(\w+)\s+(%s)$" % _WORD) +_SETENV = re.compile(r"^setenv\s+(\w+)\s+(%s)$" % _WORD) +_PATH = re.compile(r"^(prepend-path|append-path)\s+(\w+)\s+(%s)$" % _WORD) +_LOAD = re.compile(r"^(module\s+(?:load|add)|prereq|depends-on)\s+(.+)$") +_IF_LOADED = re.compile( + r"^if\s+\{?\s*!\s*\[\s*is-loaded\s+['\"]?([^'\"\]\s]+)['\"]?\s*\]\s*\}?\s*" + r"\{\s*module\s+load\s+(\S+)\s*\}$") +_IF_DIR = re.compile( + r"^if\s+\{\s*\[\s*file\s+isdirectory\s+(%s)\s*\]\s*\}\s*" + r"\{\s*(prepend-path|append-path)\s+(\w+)\s+(%s)\s*\}$" % (_WORD, _WORD)) +# ModuleRecipe's man pages: setup.sh adds the view's itself, on the same condition. +_IF_MANPATH = re.compile(r"^if\s+\{\[info exists ::env\(MANPATH\)\].*prepend-path MANPATH \S+\s*\}$") +_REF = re.compile(r"\$::env\((\w+)\)|\$env\((\w+)\)|\$\{(\w+)\}|\$(\w+)") + + +class Unsupported(Exception): + """A modulefile form this module does not read.""" + + +def commands(text): + """The modulefile's commands, one string each (a braced block joined to + one line), without the ModulesHelp proc.""" + out, cur, depth = [], [], 0 + for line in text.splitlines(): + s = line.strip() + if not cur and (not s or s.startswith("#")): + continue + cur.append(s) + depth += s.count("{") - s.count("}") + if depth <= 0: + cmd = " ".join(p for p in cur if p) + cur, depth = [], 0 + if not cmd.startswith("proc "): + out.append(cmd) + if cur: + raise Unsupported("unbalanced braces: %s" % " ".join(cur)[:80]) + return out + + +def subst(word, tcl_vars, env): + """*word* with its variables substituted, as Tcl would: braces quote + literally; $::env(BASEDIR) is BASE; other env variables must have been set + by a modulefile loaded before.""" + if word.startswith("{") and word.endswith("}"): + return word[1:-1] + if word.startswith('"') and word.endswith('"'): + word = word[1:-1] + if "[" in word or "\\" in word: + raise Unsupported("command or escape in %s" % word) + + def one(m): + name = m.group(1) or m.group(2) + if name: + if name == "BASEDIR": + return BASE + if name in env: + return env[name] + raise Unsupported("$env(%s) is not set by the release" % name) + name = m.group(3) or m.group(4) + if name not in tcl_vars: + raise Unsupported("$%s is not set" % name) + return tcl_vars[name] + return _REF.sub(one, word) + + +def read_modulefile(text): + """The statements of one modulefile, in order: ("set", name, word), + ("load", module) and ("op", command, var, word, guard), guard a directory + that must exist (a word), or None. Words are substituted when the module + is loaded (see release_ops).""" + out = [] + for cmd in commands(text): + if _SKIP.match(cmd) or _IF_MANPATH.match(cmd): + continue + m = _SET.match(cmd) + if m: + out.append(("set", m.group(1), m.group(2))) + continue + m = _IF_LOADED.match(cmd) + if m: + out.append(("load", m.group(2))) + continue + m = _LOAD.match(cmd) + if m: + names = m.group(2).split() + if m.group(1) == "prereq" and len(names) > 1: # any one of them + raise Unsupported("prereq of several modules: %s" % cmd[:80]) + out.extend(("load", n) for n in names) + continue + m = _SETENV.match(cmd) + if m: + out.append(("op", "setenv", m.group(1), m.group(2), None)) + continue + m = _PATH.match(cmd) + if m: + out.append(("op", m.group(1), m.group(2), m.group(3), None)) + continue + m = _IF_DIR.match(cmd) + if m: + out.append(("op", m.group(2), m.group(3), m.group(4), m.group(1))) + continue + raise Unsupported("unknown command: %s" % cmd[:80]) + return out + + +def release_ops(top, members, read, is_dir): + """The environment operations of loading the release's *top* modules, in + the order modulecmd applies them (a module it loads, where it loads it), + and the problems that make the result incomplete. *members* maps each + package of the release to its version directory (-); + read(pkg) gives its modulefile text, or None; is_dir(pkg, rel) whether its + package-relative *rel* is a directory. Both may raise Unsupported. + Returns ([(pkg, command, var, value)], [problem, ...]).""" + out, problems, env, done = [], [], {}, set() + + def load(name, via): + pkg, _, ver = name.partition("/") + if name == "BASE/1.0": + return + if not ver: # modulecmd takes the tree's default, maybe of another release + problems.append("%s loads %s without a version" % (via, name)) + return + if members.get(pkg) != ver: + problems.append("%s loads %s, not a module of this release" % (via, name)) + return + if pkg in done: + return + done.add(pkg) + try: + text = read(pkg) + if text is None: + raise Unsupported("no modulefile") + statements = read_modulefile(text) + except Unsupported as exc: + problems.append("%s: %s" % (pkg, exc)) + return + root = "%s/%s/%s" % (BASE, pkg, members[pkg]) + tcl_vars = {} + for st in statements: + try: + if st[0] == "set": + tcl_vars[st[1]] = subst(st[2], tcl_vars, env) + continue + if st[0] == "load": + load(st[1], pkg) + continue + _, cmd, var, word, guard = st + value = subst(word, tcl_vars, env) + if guard is not None: + g = posixpath.normpath(subst(guard, tcl_vars, env)) + if g != root and not g.startswith(root + "/"): + raise Unsupported("tests a directory outside the package: %s" % g) + if not is_dir(pkg, g[len(root) + 1:]): + continue + except Unsupported as exc: + problems.append("%s: %s" % (pkg, exc)) + continue + if cmd == "setenv": + env[var] = value + out.append((pkg, cmd, var, value)) + + for name in top: + load("%s/%s" % (name, members.get(name, "")), "the release") + return out, problems + + +def setup_env(ops, members, in_view): + """What setup.sh sets for *ops* (from release_ops): ({var: [entries]} for + the path variables, in order, INHERITED where the value from before goes; + {var: value} for the others; problems). The path variables are built as + modulecmd builds them; then, as the view helper collapses a loaded + environment, a collapsed variable's entry for a package below what the view + links of it (in_view(pkg, rel), rel "" for the package root) becomes the + view's, keeping the first of equal entries.""" + raw, sets, problems = {}, {}, [] + for pkg, cmd, var, value in ops: + if not _SAFE_VALUE.match(value): + problems.append("%s: %s's value needs quoting: %s" % (pkg, var, value)) + elif any(v.startswith("/") for v in value.split(":")): # e.g. the build's own path + problems.append("%s: %s is an absolute path: %s" % (pkg, var, value)) + elif cmd == "setenv": + if var not in _IGNORED_SETENV: + sets[var] = value + elif ":" in value: + problems.append("%s: %s gets several entries at once: %s" % (pkg, var, value)) + else: + entries = raw.setdefault(var, [INHERITED]) + if any(e and e[1] == value for e in entries): + continue # modulecmd does not add an entry twice + if cmd == "prepend-path": + entries.insert(0, (pkg, value)) + else: + entries.append((pkg, value)) + paths = {} + for var, entries in raw.items(): + out = [] + for e in entries: + if e is not INHERITED and var in COLLAPSE_VARS: + pkg, value = e + root = "%s/%s/%s" % (BASE, pkg, members[pkg]) + rel = value[len(root) + 1:] if value.startswith(root + "/") else ( + "" if value == root else None) + if rel is not None and in_view(pkg, rel): + value = VIEW + ("/" + rel if rel else "") + e = value + elif e is not INHERITED: + e = e[1] + if e not in out: + out.append(e) + paths[var] = out + for var in sorted(set(paths) & set(sets)): + problems.append("%s is both set and extended" % var) + return paths, sets, problems diff --git a/bits_helpers/orchestrated.py b/bits_helpers/orchestrated.py new file mode 100644 index 00000000..af1f3a8d --- /dev/null +++ b/bits_helpers/orchestrated.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Whether bits-orchestrator owns a store's certified set, and asking it for changes. + +When the orchestrator takes certification over, it puts ``MANIFESTS/orchestrator.json`` +in the store (``{"issuer", "manifests", "since"}``). From then on it alone writes and +signs the manifests, and the certified set changes only through it: + +* ``bits certify`` uploads the build and asks for its pre-approval, and stops there: + the orchestrator certifies the build from that pre-approval (no merge request); +* a build leaves the set by a ``decertify`` action requested from bits-console + (``bits store rm --stale-boms --manifests-dir``, ``bits compliance --enforce``); +* nothing in bits signs or writes the manifests (``bits sign`` refuses). + +The marker is not signed. It can only make bits do less (open no merge request, +sign nothing), and only its presence is used: the console a token is sent to never +comes from it. +""" + +import json +import os +import time + +MARKER = "MANIFESTS/orchestrator.json" + + +class Unknown(Exception): + """The store could not say whether it has the marker.""" + + +class RequestError(Exception): + """bits-console did not take a request.""" + + +def _absent(exc): + resp = getattr(exc, "response", None) or {} + code = str((resp.get("Error") or {}).get("Code", "")) + status = (resp.get("ResponseMetadata") or {}).get("HTTPStatusCode") + return code in ("404", "NoSuchKey", "NotFound") or status == 404 + + +def _not_listed(s3, bucket): + """Whether a listing shows the marker absent: S3 answers 403, not 404, for a + missing object to a client that may not list the bucket.""" + try: + page = s3.list_objects_v2(Bucket=bucket, Prefix=MARKER, MaxKeys=1) + except Exception: # pylint: disable=broad-except + return False + return not any(o.get("Key") == MARKER for o in page.get("Contents") or []) + + +def read(s3, bucket): + """The marker of the store *bucket*, or None when it has none. A marker that + is there but unreadable still says the orchestrator owns the set. Raises + Unknown when the store cannot be asked.""" + try: + body = s3.get_object(Bucket=bucket, Key=MARKER)["Body"].read() + except Exception as exc: # pylint: disable=broad-except + if _absent(exc) or _not_listed(s3, bucket): + return None + raise Unknown("cannot read %s from the store: %s" % (MARKER, exc)) + try: + doc = json.loads(body) + except ValueError: + doc = None + doc = doc if isinstance(doc, dict) else {} + return {"issuer": str(doc.get("issuer") or "bits-orchestrator"), + "manifests": str(doc.get("manifests") or "MANIFESTS/common-manifest"), + "since": str(doc.get("since") or "?")} + + +def s3_of(store_url, work_dir="sw", arch="x86_64"): + """(s3 client, bucket) of *store_url*, or None for a store that is not S3.""" + from bits_helpers.publish import _normalize_s3_store + from bits_helpers.sync import remote_from_url + store = _normalize_s3_store(store_url) + if not store.startswith(("b3://", "s3://")): + return None + try: + writer = remote_from_url(store, store, arch, work_dir) + except BaseException as exc: # remote_from_url may exit on a bad store + raise Unknown("cannot open the store %s: %s" % (store_url, exc)) + s3 = getattr(writer, "s3", None) + if s3 is None: + return None + return s3, getattr(writer, "remoteStore", None) or getattr(writer, "writeStore", None) + + +def for_store(store_url, work_dir="sw", arch="x86_64"): + """The marker of the store at *store_url* (see :func:`read`); None for a store + that is not S3, which no orchestrator writes.""" + client = s3_of(store_url, work_dir, arch) + return read(*client) if client else None + + +def describe(marker): + return "%s, since %s" % (marker["issuer"], marker["since"]) + + +def console_url(explicit=None, system=None): + """bits-console's URL: *explicit* (--console), $BITS_CONSOLE_URL, the + defaults' ``system: console_url:``, then the site's ``console.backend_url``.""" + from bits_helpers import site + return (explicit or os.environ.get("BITS_CONSOLE_URL") or (system or {}).get("console_url") + or site.get("console.backend_url")) + + +class Console: + """Requests to bits-console as the holder of *token* (a GitLab token, until + bits-console signs people in itself).""" + + def __init__(self, url, token, cafile=None, insecure=False): + self.base = url.rstrip("/") + self.token = token + self._tls = (cafile, insecure) + self._call = None + + def check(self): + """Raise RequestError for a URL a token must not be sent to (not https).""" + from bits_helpers import sign_console + try: + sign_console._check_console_url(self.base, self._tls[1]) + except SystemExit as exc: + raise RequestError(str(exc)) + + def _json(self, path, obj=None): + if self._call is None: + from bits_helpers import sign_console + self.check() # a token is sent + sign_console._set_tls(*self._tls) + self._call = sign_console._json_call + code, body = self._call(self.base + path, obj, bearer=self.token) + if code not in (200, 201): + raise RequestError("%s: HTTP %s%s" % (path, code or "unreachable", + ": %s" % body["detail"] if body.get("detail") else "")) + return body + + def groups(self): + """The groups of the certified set the caller administers.""" + return sorted((self._json("/ops/certified?limit=1").get("groups") or {}).keys()) + + def decertify_package(self, group, package, reason="", arch=None): + """Request the decertify-package action: *group*'s BOMs (of *arch*, or every + architecture) are replaced by themselves without *package*'s entries; its + manifests are then re-signed without them. Returns the action.""" + params = {"group": group, "package": package} + if arch: + params["arch"] = arch + reason = " ".join(str(reason or "").split())[:500] + if reason: + params["reason"] = reason + return self._json("/ops/actions", {"kind": "decertify-package", "params": params}) + + def decertify(self, group, build_id, arch=None, reason=""): + """Request the decertify action: *group*'s BOMs of *build_id* (of *arch*, + or every architecture) leave the set; its manifests are then re-signed + without them. Returns the action.""" + params = {"group": group, "build_id": build_id} + if arch: + params["arch"] = arch + reason = " ".join(str(reason or "").split())[:500] + if reason: + params["reason"] = reason + return self._json("/ops/actions", {"kind": "decertify", "params": params}) + + @staticmethod + def outcome(action): + """(ok, words) for a decertify *action* as last seen: ok when done, or + approved and waiting for the worker; words name what is left over.""" + state = action.get("state") + if state not in ("done", "approved", "running"): + return False, "%s %s" % (state, action.get("error") or "(see bits-console)") + result = action.get("result") + if isinstance(result, str): + try: + result = json.loads(result) + except ValueError: + result = None + result = result if isinstance(result, dict) else {} + others = result.get("still_certified_by") + words = state if state == "done" else "%s (the worker runs it next)" % state + if state == "done" and "boms_replaced" in result: + words += " (%d BOM(s) without it, %d taken out)" % ( + result["boms_replaced"], result.get("boms_removed", 0)) + if others: + words += "; still certified by %s, until they decertify it too" % ", ".join(others) + return True, words + + def settle(self, action, timeout=60): + """Wait up to *timeout* seconds for *action* to finish; returns it as last seen.""" + deadline = time.monotonic() + timeout + while action.get("state") not in ("done", "failed", "rejected") \ + and time.monotonic() < deadline: + time.sleep(2) + try: + action = self._json("/ops/actions/%s" % int(action["id"])) + except RequestError: + break + return action diff --git a/bits_helpers/overlay/__init__.py b/bits_helpers/overlay/__init__.py index b7bddf1f..ca0c3912 100644 --- a/bits_helpers/overlay/__init__.py +++ b/bits_helpers/overlay/__init__.py @@ -58,7 +58,7 @@ def _usage(out=sys.stderr): def main(argv=None): argv = list(sys.argv[1:] if argv is None else argv) if not argv or argv[0] in ("-h", "--help"): - _usage() + _usage(sys.stdout if argv else sys.stderr) # asked for: stdout return 0 if argv else 2 fmt, rest = argv[0], argv[1:] mod = _load_format(fmt) diff --git a/bits_helpers/overlay/lcg.py b/bits_helpers/overlay/lcg.py index b548766c..67c42284 100644 --- a/bits_helpers/overlay/lcg.py +++ b/bits_helpers/overlay/lcg.py @@ -52,9 +52,17 @@ def _load_meta(meta_path): return json.load(handle) -def collect(work_dir, arch): +def closure_of(meta_path): + """{name: (version, revision)} of the dependency closure in a .meta.json.""" + deps = (_load_meta(meta_path).get("dependencies") or {}).get("recursive") or {} + return {d["name"]: (str(d.get("version", "")), str(d.get("revision") or "")) + for d in (deps.get("build") or []) + (deps.get("runtime") or []) if d.get("name")} + + +def collect(work_dir, arch, closure=None): """Scan ``/`` for installed packages (``/`` or, - with a family, ``//``). + with a family, ``//``). With *closure* (closure_of), only + those packages, at those versions: the work dir may hold others. Returns ``(records, warnings, errors)``: * records — dict name -> (install_dir, meta); newest ``.meta.json`` wins @@ -91,6 +99,9 @@ def collect(work_dir, arch): errors.append("%s: .meta.json has no package name/version" % meta_path) continue install_dir = os.path.dirname(meta_path) + if closure is not None and closure.get(name) != ( + str(version), str(pkg.get("revision") or "")): + continue prev = chosen.get(name) if prev is None or mtime > prev[2]: if prev is not None: @@ -100,6 +111,10 @@ def collect(work_dir, arch): else: warnings.append("duplicate package %r: using %s, ignoring %s" % (name, prev[0], install_dir)) + # Not under this arch (own_hash, share, reused from CVMFS) or not built here. + for name in sorted(set(closure or ()) - set(chosen)): + warnings.append("%s %s-%s of the closure is not under %s" % ( + (name,) + closure[name] + (root,))) return {n: (v[0], v[1]) for n, v in chosen.items()}, warnings, errors @@ -159,12 +174,17 @@ def main(argv=None): "scanned closure into this directory, with a setup.sh that " "collapses PATH/LD_LIBRARY_PATH/... onto it (native " "bits_helpers.view — no lcgcmake create_lcg_view needed).") + parser.add_argument("--closure", default="", + help="A .meta.json (e.g. the view package's own): list only " + "the packages of its dependency closure.") parser.add_argument("--lib-path-var", dest="lib_path_var", default="LD_LIBRARY_PATH", help="Loader path variable for the view setup.sh " "(DYLD_LIBRARY_PATH on macOS). Default: %(default)s.") args = parser.parse_args(argv) - records, warnings, errors = collect(args.work_dir, args.architecture) + records, warnings, errors = collect( + args.work_dir, args.architecture, + closure_of(args.closure) if args.closure else None) for warning in warnings: sys.stderr.write("lcg-view: warning: %s\n" % warning) if errors: diff --git a/bits_helpers/packages.py b/bits_helpers/packages.py index eb5159a3..9882db16 100644 --- a/bits_helpers/packages.py +++ b/bits_helpers/packages.py @@ -14,6 +14,7 @@ from bits_helpers.checksum_store import load_for_spec, merge_into_spec from bits_helpers.log import banner, debug, dieOnError, warning +from bits_helpers.manifest import sbom_tool from bits_helpers.matchers import (_collect_version_pins, _matcher_active, disabledByArchitectureDefaults, filterByArchitectureDefaults, filterPatches) @@ -35,15 +36,133 @@ def shadowed_defaults_repo(pkg_filename, won_dir, search_dirs, defaults_dirs): return None +# A line of a check's output that is only a version, e.g. `xml2-config --version`. +_VERSION_LINE = re.compile(r"v?\d+(?:\.\d+)+(?:[._+~-][0-9A-Za-z]+)*") + + +def system_version(output): + """The version of a package taken from the system, from its check's output: a + ``bits_system_version: X`` line, else the first line that is only a version.""" + m = re.search(r"^bits_system_version:[ \t]*(\S+)[ \t]*$", output or "", re.MULTILINE) + if m: + return m.group(1) + return next((re.sub(r"^v", "", ln.strip()) for ln in (output or "").splitlines() + if _VERSION_LINE.fullmatch(ln.strip())), "") + + +# q NAME...: the installed packages among NAME... (rpm and dpkg, in that order), as +# "rpm NAME EPOCH VERSION RELEASE ARCH SOURCE-RPM" or "deb NAME - VERSION - ARCH SRC@SRCVER". +_PKG_QUERY = r"""T=; command -v timeout >/dev/null 2>&1 && T="timeout 20" # a locked rpm database +q() { + for p in "$@"; do + command -v rpm >/dev/null 2>&1 && + $T rpm -q --qf 'rpm %{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH} %{SOURCERPM}\n' "$p" 2>/dev/null + command -v dpkg-query >/dev/null 2>&1 && + $T dpkg-query -W -f='${db:Status-Abbrev}deb ${Package} - ${Version} - ${Architecture} ${source:Package}@${source:Version}\n' \ + "$p" 2>/dev/null | sed -n 's/^.i. *//p' + done +} +""" +# A missing file to "." would end the whole sh script (POSIX): none on macOS. +_OS_QUERY = """[ -r /etc/os-release ] && . /etc/os-release && echo "os $ID $VERSION_ID" +true +""" +_FOUND = re.compile(r"(rpm|deb) (\S+) (\S+) (\S+) (\S+) (\S+)(?: (\S+))?") + + +def _query_names(spec): + """Package names to ask the package manager for: the recipe's + ``system_package:`` (one or a list), else the usual spellings of its name.""" + own = spec.get("system_package") + if own: + return [own] if isinstance(own, str) else [str(x) for x in own] + n = spec["package"].lower() + return list(dict.fromkeys([spec["package"], n, n + "-libs"] + + ([] if n.startswith("lib") else ["lib" + n]))) + + +def _purl(found, os_line): + """(version, purl) of a package the query found, as syft writes them.""" + from urllib.parse import quote as pct + kind, name, epoch, version, release, arch, source = found + q = {"arch": arch} + if os_line and os_line.group(2): + q["distro"] = "%s-%s" % os_line.groups() + if kind == "rpm": + full = "%s-%s" % (version, release) + if epoch.isdigit() and epoch != "0": + q["epoch"] = epoch + if source and source != "(none)": + q["upstream"] = source + shown = ("%s:%s" % (epoch, full)) if "epoch" in q else full + else: + full = shown = version + src, _, src_version = (source or "").partition("@") + if src and (src != name or src_version not in ("", version)): + q["upstream"] = src if src_version in ("", version) else source + ns = os_line.group(1) if os_line else ("redhat" if kind == "rpm" else "debian") + return shown, "pkg:%s/%s/%s@%s?%s" % ( + kind, pct(ns), pct(name), pct(full, safe=".-_~+:"), + "&".join("%s=%s" % (k, pct(v, safe=".-_~+")) for k, v in sorted(q.items()))) + + +def distro_packages(output): + """{package: (version, purl)} from the output of the query (see _system_query).""" + os_line = re.search(r"^os (\S+) ?(\S*)$", output or "", re.MULTILINE) + found, current = {}, None + for line in (output or "").splitlines(): + if line.startswith("== "): + current = line[3:] + m = _FOUND.fullmatch(line) + if current and m and current not in found: + found[current] = _purl(m.groups(), os_line) + return found + + +def _system_query(specs): + """One script asking the package manager about the system packages *specs*.""" + return _PKG_QUERY + "".join("echo %s\nq %s\n" % (quote("== " + s["package"]), + " ".join(quote(n) for n in _query_names(s))) + for s in specs) + _OS_QUERY + + +def _record_system(info, taken, query): + """Record in *info* the version of each system package in *taken* ([(spec, check + output)]) and what names it for scanners: the distribution's package (a purl), else + the version its check printed, with the recipe's references: filled with it.""" + if info is None or not taken: + return + from bits_helpers.manifest import _references + try: + err, out = query({"package": "system-packages"}, _system_query([s for s, _ in taken])) + found = {} if err else distro_packages(out) + except Exception as exc: # pylint: disable=broad-except + debug("Could not ask the package manager for the system packages: %s", exc) + found = {} + for spec, output in taken: + if spec["package"] in found: + version, purl = found[spec["package"]] + info[spec["package"]] = {"version": version, "references": [purl]} + else: + version = system_version(output) + info[spec["package"]] = {"version": version, + "references": _references(dict(spec, version=version)) if version else []} + if sbom_tool(spec): # a host tool: the SBOM says so, scans leave it out + info[spec["package"]]["sbom"] = "tool" + + def getPackageList(packages, specs, configDir, preferSystem, noSystem, architecture, disable, defaults, performPreferCheck, performRequirementCheck, performValidateDefaults, overrides, taps, log, force_rebuild=(), - provider_dirs=None, defaults_meta=None, satisfied_requirements=None): + provider_dirs=None, defaults_meta=None, satisfied_requirements=None, + system_info=None): """Resolve the full set of packages required by *packages*. *satisfied_requirements*, when a set, collects the ``system_requirement`` packages whose check passed (taken from the system, like the prefer_system ones in the returned ``systemPackages``, but not reported as such). + *system_info*, when a dict, gets for every package taken from the system its + version and references (see system_version), for the SBOM. *provider_dirs* is an optional ``dict`` returned by ``repo_provider.fetch_repo_providers_iteratively``, mapping each provider @@ -64,6 +183,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, failedRequirements = set() testCache = {} requirementsCache = {} + takenFromSystem = {} # package -> (spec, check output), for system_info trackingEnvCache = {} packages = packages[:] generatedPackages = getGeneratedPackages(configDir) @@ -88,6 +208,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # is the shared base that group repos override on purpose: not a group repo. _ddirs = (defaults_meta or {}).get("_defaults_dirs") or {} _odirs = (defaults_meta or {}).get("_override_dirs") or {} + _vprofiles = (defaults_meta or {}).get("_version_profiles") or {} + _pin_owners = {} # pinned package -> the package whose requires pin it _base = os.path.abspath(_ddirs["release"]) if "release" in _ddirs else None _defaults_dirs = {} for _dname, _ddir in _ddirs.items(): @@ -203,9 +325,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, _ovr_vars = (defaults_meta or {}).get("variables") _ovr_dir = None # repo of the profile whose override changed the sources for override in overrides: - # We downcase the regex in parseDefaults(), so downcase the package name - # as well. FIXME: This is probably a bad idea; we should use - # re.IGNORECASE instead or just match case-sensitively. + # parseDefaults() lowercases an override's package regex, not its matcher, + # so the package name is matched lowercased. FIXME: re.IGNORECASE would do. pkg_re, sep, matcher = override.partition(":") if not re.fullmatch(pkg_re, p.lower()): continue @@ -214,6 +335,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, continue log("Overrides for package %s: %s", spec["package"], overrides[override]) spec.update(overrides.get(override, {}) or {}) + if "version" in (overrides.get(override) or {}): + spec["pinned_by"] = "defaults:" + _vprofiles.get(override, "?") # for the SBOM if SOURCE_KEYS & set(overrides.get(override) or {}): _ovr_dir = _odirs.get(override) or _ovr_dir @@ -280,6 +403,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # No replacement spec name given. Fall back to old system package # behaviour and just disable the package. systemPackages.add(spec["package"]) + takenFromSystem[spec["package"]] = (spec, output) if spec["package"] not in _disable_set: disable.append(spec["package"]) _disable_set.add(spec["package"]) @@ -307,7 +431,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, "recipe_source", "force_revision", "revision_policy"): if _carry in spec and _carry not in replacement: replacement[_carry] = spec[_carry] - spec = replacement + recipeSpec, spec = spec, replacement # Allows generalising the version based on the actual key provided spec["version"] = spec["version"].replace("%(key)s", key) # We need the key to inject the version into the replacement recipe later. @@ -321,6 +445,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, ownPackages.add(spec["package"]) else: systemPackages.add(spec["package"]) + takenFromSystem[spec["package"]] = (recipeSpec, output) else: warning(f"Could not find named replacement spec for {spec['package']}: {key}, " "falling back to building the package ourselves.") @@ -342,6 +467,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, _disable_set.add(spec["package"]) if satisfied_requirements is not None: satisfied_requirements.add(spec["package"]) + takenFromSystem[spec["package"]] = (spec, output) spec["disabled"] = list(disable) if spec["package"] in disable: @@ -373,6 +499,8 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, spec["package"], _version_pins, specs, default_vars=_default_vars, version=_own_version, ) + for _pinned in _version_pins: + _pin_owners.setdefault(_pinned, spec["package"]) # For the moment we treat build_requires just as requires. fn = lambda what: disabledByArchitectureDefaults(architecture, defaults, spec.get(what, []), _default_vars, _own_version) @@ -421,6 +549,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, spec.get("version"), _pin) spec["version"] = _pin spec["tag"] = _pin + spec["pinned_by"] = "requires:" + _pin_owners.get(spec["package"], "?") # External checksum store: the recipe repo's checksums/.checksum, with # those of the repos providing the active defaults profiles merged over it # (for the sources their overrides introduce). Loaded only now, after the @@ -470,4 +599,5 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, for _src, _pkgs in by_source.items(): log(" %s ← %s", _src, ", ".join(_pkgs)) + _record_system(system_info, list(takenFromSystem.values()), performPreferCheck) return (systemPackages, ownPackages, failedRequirements, validDefaults) diff --git a/bits_helpers/plan.py b/bits_helpers/plan.py index aaadbe39..313ac5e5 100644 --- a/bits_helpers/plan.py +++ b/bits_helpers/plan.py @@ -1,37 +1,48 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -"""Read-only reuse plan for ``bits build --dry-run``. +"""Read-only reuse plan, shown before every build and as ``bits build --dry-run``. For each package in build order: compute its hash as the build would, then say where it would come from (installed, local tarball, remote store) or that it would be built. Nothing is downloaded, installed or linked; the remote store is only listed (one request per candidate hash). + +A package only another, reused, package needs to build (its build_requires) is +"not needed": the build neither fetches nor builds it (issue #141). """ +import concurrent.futures +import copy import os from glob import glob from bits_helpers.arch import effective_arch -from bits_helpers.log import banner, debug +from bits_helpers.log import ProgressPrint, banner, debug from bits_helpers.sync import binary_redistributable from bits_helpers.rev_index import revision_from_tarball from bits_helpers.utilities import is_virtual_package, resolve_store_path INSTALLED = "installed" LOCAL_TARBALL = "local tarball" -REMOTE = "from remote store" -OVERLAY = "from reuse overlay" -REMOTE_UNSIGNED = "build (in store, not signed)" +REMOTE = "from store" +OVERLAY = "from overlay" +SYMLINK = "symlink from deployment" +REMOTE_UNSIGNED = "build (unsigned)" # in the store, but no signed manifest vouches for it BUILD = "build" -DEVEL = "build (development package)" +DEVEL = "build (development)" +NOT_NEEDED = "not needed" +REUSED = (INSTALLED, LOCAL_TARBALL, REMOTE, OVERLAY, SYMLINK) +LIST_WORKERS = 16 # store listings at once: one request each, mostly latency def store_can_list(sync_helper): - """Whether *sync_helper* can list store objects (http(s):// and b3:// stores). + """Whether *sync_helper* can list store objects (http(s)://, s3:// and b3:// stores). - The base RemoteSync.list_store_tarballs is a no-op; s3:// (s3cmd) and rsync - stores keep it, so their contents cannot be checked without downloading.""" - from bits_helpers.sync import RemoteSync + The base RemoteSync.list_store_tarballs is a no-op; rsync stores keep it, so + their contents cannot be checked without downloading.""" + from bits_helpers.sync import RemoteSync, MultiRemoteSync + if isinstance(sync_helper, MultiRemoteSync): + return any(store_can_list(r) for r in sync_helper.readers) return getattr(type(sync_helper), "list_store_tarballs", None) not in ( None, RemoteSync.list_store_tarballs) @@ -71,34 +82,50 @@ def _installed(spec, work_dir, eff, rev, h): return False +def _local(spec, eff, work_dir, write_store): + """(INSTALLED or LOCAL_TARBALL, hash, revision) of a tarball of *spec* in the + local store, or None.""" + forced = spec.get("force_revision") if "force_revision" in spec else None + if forced is not None: + remote, local = [spec["remote_revision_hash"]], [] + else: + # With a write store the build never reuses a localN revision. + remote, local = list(spec["remote_hashes"]), [] if write_store else list(spec["local_hashes"]) + for hashes, is_local in ((remote, False), (local, True)): + for h in hashes: + rev = pick_revision(glob(os.path.join(work_dir, resolve_store_path(eff, h), "*.tar.gz")), + spec, eff, local=is_local, forced=forced) + if rev is not None: + state = INSTALLED if _installed(spec, work_dir, eff, rev, h) else LOCAL_TARBALL + return state, h, rev + return None + + def classify(spec, arch, work_dir, sync_helper, can_list, write_store, trusted, - overlay=None): + overlay=None, symlink=None): """Return (state, hash, revision) for *spec*, mirroring build_one_package. - *trusted* is None (signed reuse off) or the verified hash map; *overlay* is a + *trusted* is None (signed reuse off) or the verified {(arch, hash): sha256} map; *overlay* is a predicate for packages the reuse overlay satisfies. Not modelled: revision candidates known only from the signed manifest / rev-index markers, and the sha256 check of a downloaded tarball (so a REMOTE line can still rebuild).""" eff = effective_arch(spec, arch) if spec.get("is_devel_pkg"): return DEVEL, spec["local_revision_hash"], "" + if symlink is not None: + match = symlink(spec) + if match: + from bits_helpers.reuse_symlinks import matching_local_install + if matching_local_install(spec, match, work_dir, arch): + return INSTALLED, match["hash"], match["revision"] + return SYMLINK, match["hash"], match["revision"] if overlay is not None and overlay(spec): return OVERLAY, spec["remote_revision_hash"], "" forced = spec.get("force_revision") if "force_revision" in spec else None - if forced is not None: - remote = [spec["remote_revision_hash"]] - local = [] - else: - remote = list(spec["remote_hashes"]) - # With a write store the build never reuses a localN revision. - local = [] if write_store else list(spec["local_hashes"]) - for hashes, is_local in ((remote, False), (local, True)): - for h in hashes: - rev = pick_revision(glob(os.path.join(work_dir, resolve_store_path(eff, h), "*.tar.gz")), - spec, eff, local=is_local, forced=forced) - if rev is not None: - state = INSTALLED if _installed(spec, work_dir, eff, rev, h) else LOCAL_TARBALL - return state, h, rev + remote = [spec["remote_revision_hash"]] if forced is not None else list(spec["remote_hashes"]) + here = _local(spec, eff, work_dir, write_store) + if here: + return here # A remote tarball the signed manifest does not vouch for is discarded by # the build, which then builds as if the store had none (the hash-policy # fallback below included). @@ -107,7 +134,7 @@ def classify(spec, arch, work_dir, sync_helper, can_list, write_store, trusted, for h in remote: rev = pick_revision(sync_helper.list_store_tarballs(eff, h), spec, eff, forced=forced) if rev is not None: - if trusted is not None and h not in trusted: + if trusted is not None and (eff, h) not in trusted: unsigned = (h, rev) break return REMOTE, h, rev @@ -154,51 +181,296 @@ def satisfied(spec): return satisfied -def print_plan(rows, store_checked): - """Print the per-package plan and a one-line summary.""" - w0 = max([len("Package")] + [len(r[0]) for r in rows]) - w1 = max([len("Version")] + [len(r[1]) for r in rows]) - lines = [" %-*s %-*s %s" % (w0, "Package", w1, "Version", "Plan")] - for pkg, ver, state, note in rows: - lines.append(" %-*s %-*s %s%s" % (w0, pkg, w1, ver, state, note)) - count = lambda s: sum(1 for r in rows if r[2] == s) - reused = count(INSTALLED) + count(LOCAL_TARBALL) + count(REMOTE) + count(OVERLAY) - summary = "%d reused (%d installed, %d local tarball, %d from remote store%s), %d to build" % ( +def needed(build_order, specs, states, targets, build_targets=()): + """The packages the build needs: *targets* and what they require, where a + reused package needs only its runtime and untracked requirements, not its + build_requires. *build_targets* are followed as if built (--only-deps).""" + # defaults-* and development packages are always needed, and so is what they require. + out = set() + todo = list(targets) + [p for p in build_order + if p.startswith("defaults-") or specs[p].get("is_devel_pkg")] + while todo: + p = todo.pop() + if p in out or p not in specs: + continue + out.add(p) + spec = specs[p] + if states.get(p) in REUSED and p not in build_targets: + todo += list(spec.get("runtime_requires") or ()) + list(spec.get("untracked_requires") or ()) + else: + todo += list(spec.get("requires") or ()) + return out + + +def adopt(specs, planned, packages): + """Give the not-needed *packages* the hash and revision the plan found for them + in *planned*: the reused packages that build with them fold those in. Marked + not_needed, they are left out of those packages' init.sh.""" + found = {p: (planned[p]["hash"], planned[p]["revision"]) for p in packages} + for p, (h, rev) in found.items(): + specs[p].update(hash=h, revision=rev, not_needed=True) + + +def plan_rows(build_order, specs, states, need, unuploadable=(), source=None): + """(package, version, plan, note, recipe source) per package; defaults-* left out.""" + rows = [] + for p in build_order: + if p.startswith("defaults-"): + continue + state, note = states[p], "" + if p not in need: + state = NOT_NEEDED + users = [q for q in build_order if q in need and states.get(q) in REUSED + and p in (specs[q].get("full_requires") or specs[q].get("requires") or ())] + if users: + note = " (build-only, for reused %s%s)" % (", ".join(users[:3]), + ", ..." if len(users) > 3 else "") + elif p in unuploadable: + note = " (not uploadable: redistributable)" + rows.append((p, specs[p]["version"], state, note, source(p) if source else "")) + return rows + + +def _unsigned_reason(n): + from bits_helpers.build import _try_import_trust + if _try_import_trust() is None: + return "%s cannot be checked without the Python package cryptography" % ( + "its signature" if n == 1 else "their signatures") + return "no verified signed manifest vouches for %s" % ("it" if n == 1 else "them") + + +NAME_WIDTH, VERSION_WIDTH = 28, 16 # longer ones overflow their own row only + + +def print_plan(rows, store_checked, title="Dry run: build plan (nothing built)"): + """Print the per-package plan and a summary, narrow enough for a terminal: the + recipe source shared by most packages is named once, in the title.""" + sources = [r[4] for r in rows if len(r) > 4 and r[4]] + common = max(dict.fromkeys(sources), key=sources.count) if sources else "" + if common and sources.count(common) * 2 <= len(rows): + common = "" + if common: + title += "\nRecipes from %s, unless shown" % common + shown = lambda r: r[4] if len(r) > 4 and r[4] and r[4] != common else "" # noqa: E731 + w0 = min(max([len("Package")] + [len(r[0]) for r in rows]), NAME_WIDTH) + w1 = min(max([len("Version")] + [len(r[1]) for r in rows]), VERSION_WIDTH) + w2 = max([len("Plan")] + [len(r[2] + r[3]) for r in rows if shown(r)]) + lines = ["%-*s %-*s Plan" % (w0, "Package", w1, "Version")] + for r in rows: + pkg, ver, state, note = r[:4] + line = "%-*s %-*s %s%s" % (w0, pkg, w1, ver, state, note) + if shown(r): + line += "%*s [%s]" % (max(w2 - len(state + note), 0), "", shown(r)) + lines.append(line.rstrip()) + count = lambda s: sum(1 for r in rows if r[2] == s) # noqa: E731 + reused = sum(count(s) for s in REUSED) + summary = ["%d reused (%d installed, %d local tarball, %d from store%s), %d to build%s" % ( reused, count(INSTALLED), count(LOCAL_TARBALL), count(REMOTE), - (", %d from overlay" % count(OVERLAY)) if count(OVERLAY) else "", len(rows) - reused) - if not store_checked: - summary += "; remote store not checked (only http(s):// and b3:// stores can be listed)" - banner("Dry run: build plan (nothing built)\n%s\n\n%s", "\n".join(lines), summary) + ((", %d from overlay" % count(OVERLAY)) if count(OVERLAY) else "") + + ((", %d symlinked" % count(SYMLINK)) if count(SYMLINK) else ""), + len(rows) - reused - count(NOT_NEEDED), + (", %d not needed" % count(NOT_NEEDED)) if count(NOT_NEEDED) else "")] + if count(REMOTE_UNSIGNED): + n = count(REMOTE_UNSIGNED) + summary.append("%d to build %s in the remote store but not reused: %s" % ( + n, "is" if n == 1 else "are", _unsigned_reason(n))) + if store_checked is False: + summary.append("remote store not checked (rsync stores cannot be listed)") + elif store_checked is None: + summary.append("no remote store used") + banner("%s\n%s\n\n%s", title, "\n".join(lines), "\n".join(summary)) -def plan_build(build_order, specs, args, work_dir, sync_helper, raw_architecture, - cfg, trusted_index_fn): - """Compute hashes in build order and print what the build would do.""" +class _Listings: + """The store listings of a plan, made in parallel up front: in front of + *helper*'s list_store_tarballs, one HTTP request per (architecture, hash). + With a *kept* StoreCache and the *trusted* (signed) index, a certified + tarball found in an earlier run is not asked for again: the store's garbage + collection keeps what a signed manifest lists.""" + + def __init__(self, helper, kept=None, trusted=None): + self.helper, self.cache, self.kept, self.trusted = helper, {}, kept, trusted + self.failed = set() # not asked for again ahead, only when needed + + def _from_kept(self, key): + if self.kept is None or self.trusted is None or key not in self.trusted: + return None + return self.kept.get(*key) + + def _found(self, key, names): + self.cache[key] = names + if self.kept is not None and self.trusted is not None and key in self.trusted: + self.kept.put(key[0], key[1], names) + + def list_store_tarballs(self, arch, pkg_hash): + key = (arch, pkg_hash) + if key not in self.cache: # a hash the speculation did not foresee + names = self._from_kept(key) + if names is None: + names = self.helper.list_store_tarballs(arch, pkg_hash) + self._found(key, names) + self.cache[key] = names + return self.cache[key] + + def prefetch(self, keys, what="Checking the remote store"): + """List *keys* not known yet, in parallel; returns how many were asked for.""" + keys = [k for k in dict.fromkeys(keys) if k not in self.cache and k not in self.failed] + for k in keys: + names = self._from_kept(k) + if names is not None: + self.cache[k] = names + known = len(keys) + keys = [k for k in keys if k not in self.cache] + known -= len(keys) + if not keys: + return 0 + progress = ProgressPrint(what) + pool = concurrent.futures.ThreadPoolExecutor(max_workers=LIST_WORKERS) + futures = {pool.submit(self.helper.list_store_tarballs, *k): k for k in keys} + try: + for i, f in enumerate(concurrent.futures.as_completed(futures)): + progress("[%d/%d] packages", i + 1, len(keys)) + try: + self._found(futures[f], f.result()) + except Exception as exc: # listed again, one by one, when needed + debug("store listing failed for %s: %s", futures[f], exc) + self.failed.add(futures[f]) + except BaseException: # Ctrl-C: drop what has not started + for f in futures: + f.cancel() + pool.shutdown(wait=False) + raise + pool.shutdown() + progress.end("done%s" % (", %d known from earlier runs" % known if known else "")) + return len(keys) + + +class _Assumed: + """The store as the speculation sees it: the *known* listings, and a tarball + (signed) under every hash not listed yet: the usual case.""" + + def __init__(self, known, trusted): + self.known, self.trusted, self.spec = known, trusted, None + + def list_store_tarballs(self, arch, pkg_hash): + if (arch, pkg_hash) in self.known: + return self.known[(arch, pkg_hash)] + rev = self.spec.get("force_revision", "1") # "" forced: a revision-less name + return ["%s-%s%s.%s.tar.gz" % (self.spec["package"], self.spec["version"], + "-" + rev if rev else "", arch)] + + def __contains__(self, key): # as the trusted index + return key not in self.known or (self.trusted is not None and key in self.trusted) + + +def _foreseen(build_order, specs, args, raw_architecture, work_dir, write_store, known=None, + trusted=None, overlay=None, symlink=None): + """The (architecture, hash) pairs the plan will list, but not those of + packages already here: computed on a copy of *specs*, each package classified + as the plan will, from the *known* listings ({(arch, hash): names}) and, for + hashes not listed yet, assuming the store has them, signed.""" + from bits_helpers.arch import SHARED_ARCH + from bits_helpers.build import storeHook + from bits_helpers.hashing import storeHashes + store = _Assumed(known or {}, trusted) + signed = store if trusted is not None else None + trial, keys = copy.deepcopy(specs), [] + for p in build_order: + spec = trial[p] + storeHook(p, trial, args.defaults[0]) + storeHashes(p, trial, considerRelocation=( + raw_architecture.startswith("osx") and spec.get("architecture") != SHARED_ARCH)) + store.spec = spec + state, spec["hash"], spec["revision"] = classify( + spec, args.architecture, work_dir, store, True, write_store, signed, overlay, symlink) + if state == DEVEL: + write_store = False # as the plan (and the build) do from here on + if state in (DEVEL, OVERLAY, SYMLINK, INSTALLED, LOCAL_TARBALL) or is_virtual_package(spec): + continue + eff = effective_arch(spec, args.architecture) + hashes = [spec["remote_revision_hash"]] if "force_revision" in spec else spec["remote_hashes"] + keys += [(eff, h) for h in hashes] + return keys + + +MAX_ROUNDS = 4 # listings made again for hashes the earlier ones changed + + +def plan(build_order, specs, args, work_dir, sync_helper, raw_architecture, + cfg, trusted_index_fn): + """({package: state}, packages not uploadable, whether the store was listed), + as the build would decide. Sets each spec's hash and revision: a real build + passes a copy of its specs, so that its own loop decides afresh.""" from bits_helpers.build import storeHook from bits_helpers.hashing import storeHashes from bits_helpers.arch import SHARED_ARCH write_store = bool(getattr(sync_helper, "writeStore", "")) - can_list = store_can_list(sync_helper) + from bits_helpers.sync import NoRemoteSync + # None: there is no store, so nothing to check. + can_list = None if isinstance(sync_helper, NoRemoteSync) else store_can_list(sync_helper) trusted = trusted_index_fn() if (cfg.require_signed_reuse and can_list) else None overlay = _overlay_predicate(cfg) - rows = [] + from bits_helpers.reuse_symlinks import select_symlink_package + sym_trusted = (trusted_index_fn() + if getattr(cfg, "symlink_packages_from", None) and cfg.require_signed_reuse + else None) + symlink = lambda spec: select_symlink_package( # noqa: E731 + spec, cfg, args.architecture, trusted=sym_trusted) + lister, kept = sync_helper, None + if can_list: + from bits_helpers.store_cache import StoreCache, store_id + kept = StoreCache(work_dir, store_id(sync_helper)) + lister = _Listings(sync_helper, kept, trusted) + try: + # A package not in the store is built with its local hash, which changes + # the hashes of all that need it: listed in the next round, not one by one. + what = "Checking the remote store" + for _ in range(MAX_ROUNDS): + if not lister.prefetch(_foreseen(build_order, specs, args, raw_architecture, work_dir, + write_store, lister.cache, trusted, overlay, symlink), + what): + break + what = "Checking the remote store for the hashes that changed" + except Exception as exc: # the plan then lists one package at a time + debug("cannot list the store ahead: %s", exc) + states, unuploadable = {}, set() for p in build_order: spec = specs[p] storeHook(p, specs, args.defaults[0]) storeHashes(p, specs, considerRelocation=( raw_architecture.startswith("osx") and spec.get("architecture") != SHARED_ARCH)) state, spec["hash"], spec["revision"] = classify( - spec, args.architecture, work_dir, sync_helper, can_list, write_store, trusted, - overlay) + spec, args.architecture, work_dir, lister, can_list, write_store, trusted, + overlay, symlink=symlink) debug("Plan %s: %s (hash %s, candidates %s)", p, state, spec["hash"], ", ".join(spec["remote_hashes"])) + states[p] = state if state == DEVEL: write_store = False # the build disables uploads from here on (build.py) - if p.startswith("defaults-"): - continue - note = "" if state in (BUILD, REMOTE_UNSIGNED) and write_store and not binary_redistributable(spec): - note = " (not uploadable: redistributable)" - rows.append((p, spec["version"], state, note)) - print_plan(rows, can_list) - return rows + unuploadable.add(p) + if kept is not None: + kept.save() + return states, unuploadable, can_list + + +def decide(build_order, specs, args, work_dir, sync_helper, raw_architecture, cfg, + trusted_index_fn, targets=None, build_targets=(), title=None, source=None): + """Plan, find what is needed and print it: (rows, states, needed packages). + With cfg.fetch_build_requires every package is needed (the old behaviour).""" + states, unuploadable, can_list = plan(build_order, specs, args, work_dir, sync_helper, + raw_architecture, cfg, trusted_index_fn) + if getattr(cfg, "fetch_build_requires", False): + need = set(build_order) + else: + need = needed(build_order, specs, states, targets or build_order[-1:], build_targets) + rows = plan_rows(build_order, specs, states, need, unuploadable, source) + print_plan(rows, can_list, *([title] if title else [])) + return rows, states, need + + +def plan_build(build_order, specs, args, work_dir, sync_helper, raw_architecture, + cfg, trusted_index_fn, **kw): + """`bits build --dry-run`: print what the build would do; returns the rows.""" + return decide(build_order, specs, args, work_dir, sync_helper, raw_architecture, cfg, + trusted_index_fn, **kw)[0] diff --git a/bits_helpers/prepub.py b/bits_helpers/prepub.py index b389a82a..a0742cdc 100644 --- a/bits_helpers/prepub.py +++ b/bits_helpers/prepub.py @@ -51,6 +51,7 @@ import hashlib import os import time +import uuid from typing import Optional from urllib.parse import urlsplit @@ -78,6 +79,77 @@ _SUBMIT_BACKOFF = 0.5 # seconds; doubled per attempt +class MultipartFile: + """A multipart/form-data body whose one file part is read from disk as it is + sent: requests' files= builds the whole body in memory, the size of the tar + (twice, briefly). fields is in files= form, {name: (None, value)}, plus the + file as {name: (filename, path, content_type)}, sent last. Rewinds (seek) for + urllib3's retry; post it as data=, with its content_type as Content-Type.""" + + def __init__(self, fields): + self.fields = dict(fields) + self.boundary = uuid.uuid4().hex + self.content_type = "multipart/form-data; boundary=" + self.boundary + head, path = [], None + for name, (filename, value, *ctype) in self.fields.items(): + if filename is None: + if value is None: # as requests: a None field is not sent + continue + head.append('--%s\r\nContent-Disposition: form-data; name="%s"\r\n\r\n' + % (self.boundary, name) + str(value) + "\r\n") + elif path is None: + path, file_head = value, ('--%s\r\nContent-Disposition: form-data; name="%s"; ' + 'filename="%s"\r\nContent-Type: %s\r\n\r\n' + % (self.boundary, name, filename, + ctype[0] if ctype else "application/octet-stream")) + else: + raise ValueError("MultipartFile streams one file") + if path is None: + raise ValueError("MultipartFile needs a file field") + self._head = ("".join(head) + file_head).encode() + self._tail = ("\r\n--%s--\r\n" % self.boundary).encode() + self._fh = open(path, "rb") + self._size = os.fstat(self._fh.fileno()).st_size + self._pos = 0 + + def __len__(self): + return len(self._head) + self._size + len(self._tail) + + def tell(self): + return self._pos + + def seek(self, pos, whence=0): + self._pos = {0: pos, 1: self._pos + pos, 2: len(self) + pos}[whence] + return self._pos + + def read(self, n=-1): + end = len(self) if n is None or n < 0 else min(len(self), self._pos + n) + out, head = [], len(self._head) + while self._pos < end: + if self._pos < head: + chunk = self._head[self._pos:end] + elif self._pos < head + self._size: + self._fh.seek(self._pos - head) + chunk = self._fh.read(min(end, head + self._size) - self._pos) + if not chunk: + raise OSError("%s shrank while being sent" % self._fh.name) + else: + off = self._pos - head - self._size + chunk = self._tail[off:off + end - self._pos] + out.append(chunk) + self._pos += len(chunk) + return b"".join(out) + + def close(self): + self._fh.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def _make_session(no_verify_tls: bool = False, signed: bool = True) -> requests.Session: """Return a requests.Session with no credential attached. @@ -299,21 +371,22 @@ def submit_job( resp = None for attempt in range(1, attempts + 1): try: - with open(tar_path, "rb") as tar_fh: - fields = { - "repo": (None, repo), - "path": (None, path), - "tar": (os.path.basename(tar_path), tar_fh, "application/octet-stream"), - "tar_sha256": (None, digest), - } - if webhook_url: - fields["webhook_url"] = (None, webhook_url) - - headers = _auth_headers( - token, "POST", _signed_uri(url), - fields=signed_fields, body_hash=digest, - bearer_auth=bearer_auth, no_verify_tls=no_verify_tls) - resp = session.post(url, files=fields, headers=headers, timeout=300) + fields = { + "repo": (None, repo), + "path": (None, path), + "tar_sha256": (None, digest), + "tar": (os.path.basename(tar_path), tar_path, "application/octet-stream"), + } + if webhook_url: + fields["webhook_url"] = (None, webhook_url) + + headers = _auth_headers( + token, "POST", _signed_uri(url), + fields=signed_fields, body_hash=digest, + bearer_auth=bearer_auth, no_verify_tls=no_verify_tls) + with MultipartFile(fields) as body: + headers["Content-Type"] = body.content_type + resp = session.post(url, data=body, headers=headers, timeout=300) except requests.RequestException as exc: if attempt == attempts: error("Failed to connect to cvmfs-prepub at %s: %s", prepub_url, exc) diff --git a/bits_helpers/publish.py b/bits_helpers/publish.py index 4fa14b43..a63531b0 100644 --- a/bits_helpers/publish.py +++ b/bits_helpers/publish.py @@ -28,6 +28,9 @@ from bits_helpers.log import debug, error, info, warning, banner from bits_helpers.arch import detectArch +_NO_STORE = ("no store: give --remote-store, set $BITS_S3_STORE, or store.operations " + "in a site configuration file") + # --------------------------------------------------------------------------- # Helpers @@ -168,14 +171,19 @@ def _normalize_s3_store(url): https:///[swift/v1/][/...] -> b3:// (+ endpoint, path-style) b3:// / s3:// / rsync:... -> passthrough """ + if "," in (url or ""): + error("this command works on one store, not a list: %s", url) + sys.exit(1) if url.startswith(("b3://", "s3://", "rsync:", "cvmfs://")): return url m = re.match(r"^https?://([^/]+)/(?:swift/v1/)?([^/?#]+)", url) if not m: return url host, bucket = m.group(1), m.group(2) - os.environ.setdefault("BITS_S3_ENDPOINT_URL", "https://%s" % host) - os.environ.setdefault("S3_ADDRESSING_STYLE", "path") # CERN RGW path-style + # This bucket's endpoint (sync.s3_settings), so stores on other hosts keep theirs. + key = "BITS_S3_%s_" % re.sub(r"[^A-Za-z0-9]", "_", bucket).upper() + os.environ.setdefault(key + "ENDPOINT_URL", "https://%s" % host) + os.environ.setdefault(key + "ADDRESSING_STYLE", "path") # CERN RGW path-style return "b3://%s" % bucket @@ -195,12 +203,149 @@ def _run_leaf(): return "%s-%s-%s.json" % (host, ts, rnd) -def _publish_from_manifest(architecture, work_dir, store_url, parser, manifest=None, dry_run=False): +# The build-manifest fields a BOM entry carries (plus tarball and its sha256). +_BOM_FIELDS = ("package", "version", "revision", "effective_architecture", + "hash", "commit_hash", "pkg_family", "built_by", "completed_at", + "license", "redistributable", "view", "requires") + +# How many packages are checked/uploaded at once (BITS_PUBLISH_JOBS). Each is a +# few S3 round trips (rev-index marker, HEAD of the content object), so a +# serial pass over a whole stack spends minutes waiting on the network. +_PUBLISH_JOBS = 8 + + +def _publish_jobs(): + try: + return max(1, min(64, int(os.environ.get("BITS_PUBLISH_JOBS") or _PUBLISH_JOBS))) + except ValueError: + return _PUBLISH_JOBS + + +def _bom_documents(published, manifest_doc, architecture, build_id): + """``(packages, [(effective_arch, BOM)])`` for the *published* ``(entry, + store tarball)`` pairs: the minimal BOM entries, and one BOM per architecture.""" + import getpass, socket, datetime + from bits_helpers.checksum import checksum_file + # completed_at is kept per package so every hash carries "when built". + # license/redistributable are the compliance metadata: the BOM (and + # the signed manifest merged from it) must know what may be laid + # into a public CVMFS tree and what attribution a NOTICE file needs. + # view/requires: what a release made later from this BOM alone (bits + # cvmfs release) needs: the view rules and the dependency order. + packages = [] + for e, tar in published: + m = {k: e[k] for k in _BOM_FIELDS if k in e} + m["tarball"] = e.get("tarball") or basename(tar) + m["tarball_sha256"] = e.get("tarball_sha256") or checksum_file(tar) + packages.append(m) + try: + _user = getpass.getuser() + except Exception: + _user = "unknown" + # A BOM is per-platform: partition the entries by their effective + # architecture ("shared" — noarch — is just another platform) and + # emit one BOM per architecture, with the architecture in the file + # name. This is the invariant that lets certification be scoped per + # platform (a BOM pairs with exactly one common-manifest-) — + # bits_helpers.certify.bom_architecture() refuses mixed BOMs. + by_arch = {} + for m in packages: + by_arch.setdefault(m.get("effective_architecture") or "shared", []).append(m) + provenance = { + "build_id": build_id, + "published_by": "%s@%s" % (_user, socket.gethostname().split(".")[0]), + "published_at": datetime.datetime.now(datetime.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "architecture": architecture, + } + # The CI pipeline that produced this build — the id the console's passkey + # pre-approval is keyed on. The certify job reads it from this BOM to + # consume that pre-approval when it signs. Absent for a local publish. + # FUTURE(build_id-unification 2026-09-20): source_pipeline_id is CI-only + # (empty off-CI), so CLI/external BOMs are not pre-approvable by it. Unify + # the pre-approval anchor on the deterministic manifest build_id. KEEP this + # until that lands. Doc: signing-authz-unified-model-2026-09-20. + pipeline_id = os.environ.get("CI_PIPELINE_ID", "").strip() + if pipeline_id: + provenance["source_pipeline_id"] = pipeline_id + # The build's CVMFS layout (release name, package paths) and what it + # was asked to build: a release is made from these later, from the + # BOM alone, without the build's work directory. + for fld in ("cvmfs_templates", "requested_packages"): + if manifest_doc.get(fld): + provenance[fld] = manifest_doc[fld] + return packages, [(arch, dict(provenance, effective_architecture=arch, packages=by_arch[arch])) + for arch in sorted(by_arch)] + + +# Who and when: the only BOM fields a reused BOM may differ in. +_BOM_RUN_FIELDS = ("published_by", "published_at") + + +def _published_bom(writers, build_id, published, manifest_doc, architecture): + """The BOMs this CI pipeline already published for this build, as + ``[(arch, bom_dict)]``, or None to publish afresh. + + The publish job's BOM step and `bits certify` in the same job publish the + same manifest; the second finds the first's BOMs through the pipeline's build + index (BUILDS/by-pipeline//.json) instead of checking every + package and uploading the BOMs again. Only the store's sha256 of each tarball + is taken from them: the documents are made again from the local manifest + exactly as a publish makes them, and must equal the stored ones but for who + published them and when. Anything else in the bucket is never certified. + """ + import json as _json + pid = os.environ.get("CI_PIPELINE_ID", "").strip() + w = next(iter(writers.values()), None) + if not (pid and build_id and w is not None and getattr(w, "s3", None) + and getattr(w, "writeStore", None)): + return None + + def _get(key): + return _json.loads(w.s3.get_object(Bucket=w.writeStore, Key=key)["Body"].read()) + try: + index = _get("BUILDS/by-pipeline/%s/%s.json" % (pid, build_id)) + keys = index.get("keys") or [] + if index.get("build_id") != build_id or not keys: + return None + stored = [] + for key in keys: + if not str(key).startswith("MANIFESTS/%s/" % build_id): + return None + stored.append(_get(key)) + sha = {} + for doc in stored: + for p in doc.get("packages") or []: + if not (isinstance(p, dict) and p.get("hash") and p.get("tarball_sha256")): + return None + sha.setdefault(p["hash"], p["tarball_sha256"]) + if set(sha) != {e.get("hash") for e, _tar in published}: + raise ValueError("other packages") + _pkgs, mine = _bom_documents([(dict(e, tarball_sha256=sha[e["hash"]]), tar) + for e, tar in published], + manifest_doc, architecture, build_id) + + def _same(a, b): + return ({k: v for k, v in a.items() if k not in _BOM_RUN_FIELDS} + == {k: v for k, v in b.items() if k not in _BOM_RUN_FIELDS}) + if len(stored) != len(mine) or not all( + isinstance(d, dict) and _same(d, m) for d, (_a, m) in zip(stored, mine)): + raise ValueError("not this publish's") + except Exception as exc: # pylint: disable=broad-except + debug("pipeline %s: no BOM to reuse for %s (%s); publishing afresh", pid, build_id, exc) + return None + return [(a, d) for (a, _m), d in zip(mine, stored)] + + +def _publish_from_manifest(architecture, work_dir, store_url, parser, manifest=None, dry_run=False, + reuse_pipeline_bom=False, gauges=True): """Bulk-upload every built package in a manifest to the S3 store. *manifest* is a manifest file path, or 'latest'/None to use the newest under WORKDIR/MANIFESTS. With *dry_run*, list what would be uploaded and touch no - network (no credentials needed). + network (no credentials needed). With *reuse_pipeline_bom* (`bits certify`), + the BOMs this CI pipeline's publish step already wrote for the same packages + are returned instead of publishing again. """ import json as _json from bits_helpers.sync import remote_from_url @@ -219,6 +364,9 @@ def _publish_from_manifest(architecture, work_dir, store_url, parser, manifest=N entries = manifest_doc.get("packages", []) except Exception as exc: parser.error("could not read manifest %s: %s" % (man, exc)) + # The build's own architecture, not the publishing host's: it is what the + # BOMs record and the fallback for entries without an effective one. + architecture = manifest_doc.get("architecture") or architecture if not entries: parser.error("no packages listed in manifest %s" % man) write_store = _normalize_s3_store(store_url) @@ -275,6 +423,7 @@ def _store_tarball(arch, entry): writers = {} done = set() published = [] # (entry, store-tarball-path) actually/would-be uploaded + jobs = [] # (n, entry, arch, tarball, spec) to check/upload n = ok = 0 for e in entries: h = e.get("hash") @@ -310,25 +459,63 @@ def _store_tarball(arch, entry): ok += 1 published.append((e, tar)) continue - writer = writers.get(arch) or writers.setdefault( - arch, remote_from_url(write_store, write_store, arch, work_dir)) - spec = {"package": pkg, "version": e.get("version"), - "revision": e.get("revision"), "hash": h} + if arch not in writers: + writers[arch] = remote_from_url(write_store, write_store, arch, work_dir) + jobs.append((n, e, arch, tar, {"package": pkg, "version": e.get("version"), + "revision": e.get("revision"), "hash": h})) + + if jobs and reuse_pipeline_bom: + from bits_helpers.provenance import build_id_from_manifest + _bid = build_id_from_manifest(manifest_doc) + _reused = _published_bom(writers, _bid, [(j[1], j[3]) for j in jobs], + manifest_doc, architecture) + if _reused is not None: + banner("Using the BOM this pipeline already published for %s (%d package(s))", + _bid, len(jobs)) + return _bid, _reused, _system_from_manifest(manifest_doc) + + def _upload(job): + _n, _e, _arch, _tar, _spec = job + writers[_arch].upload_symlinks_and_tarball(_spec) + + # Independent per-package store checks/uploads, a few at a time; the BOM is + # still assembled in manifest order, so it does not depend on which finished + # first. + from concurrent.futures import ThreadPoolExecutor, as_completed + done_ok = {} + if jobs: + pool = ThreadPoolExecutor(max_workers=min(_publish_jobs(), len(jobs))) + futures = {} try: - writer.upload_symlinks_and_tarball(spec) - # The upload path records the sha256 of the object actually in the - # store (kept-as-found or freshly uploaded). It overrides whatever - # the build manifest recorded: the store object is authoritative, - # and the build's locally-packed bytes may legitimately differ - # (.tar.gz is not byte-reproducible). The BOM below must describe - # the stored bytes or `bits sign` will reject it. - if spec.get("store_tarball_sha256"): - e = dict(e, tarball_sha256=spec["store_tarball_sha256"]) - ok += 1 - published.append((e, tar)) - info(" [%d] %s %s", n, pkg, h[:12]) - except Exception as exc: - error(" FAILED %s (%s): %s", pkg, h[:12], exc) + futures.update({pool.submit(_upload, j): j for j in jobs}) + for fut in as_completed(futures): + _n, _e, _arch, _tar, _spec = futures[fut] + try: + fut.result() + except Exception as exc: # pylint: disable=broad-except + error(" FAILED %s (%s): %s", _spec["package"], _spec["hash"][:12], exc) + continue + # The upload path records the sha256 of the object actually in + # the store (kept-as-found or freshly uploaded). It overrides + # whatever the build manifest recorded: the store object is + # authoritative, and the build's locally-packed bytes may + # legitimately differ (.tar.gz is not byte-reproducible). The BOM + # below must describe the stored bytes or `bits sign` rejects it. + if _spec.get("store_tarball_sha256"): + _e = dict(_e, tarball_sha256=_spec["store_tarball_sha256"]) + done_ok[_n] = (_e, _tar) + info(" [%d/%d] %s %s", len(done_ok), len(jobs), _spec["package"], + _spec["hash"][:12]) + except BaseException: + # Ctrl-C, or a writer giving up (SystemExit): stop at once, not after + # the rest of the queue. + for f in futures: + f.cancel() # (cancel_futures needs Python 3.9) + pool.shutdown(wait=False) + raise + pool.shutdown(wait=True) + ok += len(done_ok) + published.extend(done_ok[k] for k in sorted(done_ok)) # Upload a MINIMAL BOM manifest under MANIFESTS/ (trust-relevant fields only; # drop variables/patches/source_checksums, which bloat it and can leak config) @@ -343,65 +530,15 @@ def _store_tarball(arch, entry): if not dry_run and ok == n: w = next(iter(writers.values()), None) if w is not None and hasattr(w, "s3") and getattr(w, "writeStore", None): - # completed_at is kept per package so every hash carries "when built". - # license/redistributable are the compliance metadata: the BOM (and - # the signed manifest merged from it) must know what may be laid - # into a public CVMFS tree and what attribution a NOTICE file needs. - # view/requires: what a release made later from this BOM alone (bits - # cvmfs release) needs: the view rules and the dependency order. - _keep = ("package", "version", "revision", "effective_architecture", - "hash", "commit_hash", "pkg_family", "built_by", "completed_at", - "license", "redistributable", "view", "requires") - packages = [] - for e, tar in published: - m = {k: e[k] for k in _keep if k in e} - m["tarball"] = e.get("tarball") or basename(tar) - m["tarball_sha256"] = e.get("tarball_sha256") or checksum_file(tar) - packages.append(m) - import tempfile as _tf, getpass, socket, datetime from bits_helpers.provenance import build_id_from_manifest + import tempfile as _tf # Canonical, deterministic build_id (same id the build itself used): # `, never from the events alone. The +worker signs only while the outside copy is in step with this database +(`sign_hold`), so a restored database never signs again before it has caught up. +""" + +import argparse +import json +import logging +import os +import sqlite3 +import sys + +import requests + +from . import db, events + +log = logging.getLogger("bits_orchestrator.outside") + +BATCH = 500 +BATCH_BYTES = 4 << 20 # the service takes at most 8 MB a request +MAX_BATCHES = 10 +FULL_VERIFY_MS = 24 * 3600 * 1000 +ALARM = "outside.alarm" + + +class OutsideError(Exception): + pass + + +def _token(value): + """The token, or the contents of the file it names.""" + if value and value.startswith("/"): + if not os.path.isfile(value): + raise OutsideError("BITS_ORCH_AUDIT_TOKEN names %s, which is not a file" % value) + with open(value) as fh: + return fh.read().strip() + return value + + +def _local_http(url): + """http to localhost, or to a single-label host name: a compose service on + an internal network (audit-log on this host), never one across the network.""" + from urllib.parse import urlparse + u = urlparse(url) + host = u.hostname or "" + return u.scheme == "http" and (host in ("localhost", "127.0.0.1") or (host and "." not in host)) + + +class Client: + """HTTP client of the audit-log service.""" + + def __init__(self, url, token, cafile=None, timeout=20, session=None): + self.url = url.rstrip("/") + self.headers = {"Authorization": "Bearer " + _token(token)} + self.verify = cafile or True + self.timeout = timeout + self.http = session or requests + + @classmethod + def from_settings(cls, settings): + if not settings.orch_audit_url: + return None + if not settings.orch_audit_token: + raise OutsideError("BITS_ORCH_AUDIT_URL is set but BITS_ORCH_AUDIT_TOKEN is not") + url = settings.orch_audit_url + if not (url.startswith("https://") or _local_http(url)): + raise OutsideError("BITS_ORCH_AUDIT_URL must be https (the token is a bearer)," + " or http to a container on this host's internal network") + return cls(settings.orch_audit_url, settings.orch_audit_token, + settings.orch_audit_cafile or None) + + def _call(self, method, path, body=None, ok=(200,)): + try: + r = self.http.request(method, self.url + path, headers=self.headers, json=body, + timeout=self.timeout, verify=self.verify) + except requests.RequestException as e: + raise OutsideError("audit log unreachable: %s" % str(e)[:200]) + try: + data = r.json() + except ValueError: + data = {} + if r.status_code not in ok: + raise OutsideError("audit log %s %s -> %s: %s" % (method, path, r.status_code, + str(data.get("error", ""))[:300])) + return r.status_code, data + + def head(self): + return self._call("GET", "/v1/head")[1] + + def send(self, rows): + status, data = self._call("POST", "/v1/events", {"events": rows}, ok=(200, 409)) + return status == 200, data + + def events(self, after, limit=1000): + return self._call("GET", "/v1/events?after=%d&limit=%d" % (after, limit))[1].get("events", []) + + +FIELDS = ("seq", "ts_ms", "kind", "subject", "actor_kind", "actor_id", "cause", "payload", + "prev_hash", "hash") + + +def _row(r): + return {k: r[k] for k in FIELDS} + + +def _kv(conn, key): + r = conn.execute("SELECT value FROM kv WHERE key = ?", (key,)).fetchone() + return r["value"] if r else None + + +def _alarm(conn, reason, head): + if _kv(conn, ALARM): + return + with db.transaction(conn): + conn.execute("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", + (ALARM, json.dumps({"reason": reason, "head": head, "ms": db.now_ms()}))) + events.append(conn, "outside.diverged", "outside", events.WORKER, + {"reason": reason, "outside_head": head}) + log.error("outside audit copy diverged: %s", reason) + + +def _check_local(conn, now=None): + """Verify this chain before vouching for it: every pass, the events after + the last verified one; once a day, all of it. Returns a problem or None.""" + now = now or db.now_ms() + mark = json.loads(_kv(conn, "outside.verified") or "null") or {} + full = now - int(mark.get("full_ms") or 0) >= FULL_VERIFY_MS + if full: + n, broken = events.verify(conn) + if broken is not None: + return "this database's chain breaks at event %d" % broken + last = conn.execute("SELECT seq, hash FROM events ORDER BY seq DESC LIMIT 1").fetchone() + mark = {"seq": last["seq"] if last else 0, "hash": last["hash"] if last else events.GENESIS, + "full_ms": now} + else: + seq, prev = int(mark.get("seq") or 0), mark.get("hash") or events.GENESIS + if seq: + r = conn.execute("SELECT hash FROM events WHERE seq = ?", (seq,)).fetchone() + if r is None or r["hash"] != prev: + return "this database's event %d changed since it was verified" % seq + for r in conn.execute("SELECT * FROM events WHERE seq > ? ORDER BY seq", (seq,)): + if r["seq"] != seq + 1 or r["prev_hash"] != prev or events._digest(prev, r) != r["hash"]: + return "this database's chain breaks at event %d" % r["seq"] + seq, prev = r["seq"], r["hash"] + mark = dict(mark, seq=seq, hash=prev) + with db.transaction(conn): + conn.execute("INSERT INTO kv (key, value) VALUES ('outside.verified', ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", (json.dumps(mark),)) + return None + + +def status(conn): + alarm = _kv(conn, ALARM) + last = conn.execute("SELECT seq FROM events ORDER BY seq DESC LIMIT 1").fetchone() + shipped = _kv(conn, "outside.head") + return {"local_seq": last["seq"] if last else 0, + "outside": json.loads(shipped) if shipped else None, + "alarm": json.loads(alarm) if alarm else None} + + +def ship(conn, client, batch=BATCH, max_batches=MAX_BATCHES): + """Check the outside head against this chain, then send what follows it. + Returns the number of events sent.""" + if client is None or _kv(conn, ALARM): + return 0 # an alarm stops shipping until it is cleared by hand + problem = _check_local(conn) + if problem: + _alarm(conn, problem, None) + return 0 + head = client.head() + seq, h = int(head.get("seq") or 0), str(head.get("hash") or events.GENESIS) + last = json.loads(_kv(conn, "outside.head") or "null") or {} + if seq < int(last.get("seq") or 0): + _alarm(conn, "the outside copy went back from event %d to %d: it lost events" + % (last["seq"], seq), head) + return 0 + if seq: + mine = conn.execute("SELECT hash FROM events WHERE seq = ?", (seq,)).fetchone() + if mine is None: + _alarm(conn, "the outside copy has event %d, this database does not: it was" + " restored from an older copy, or rewritten" % seq, head) + return 0 + if mine["hash"] != h: + _alarm(conn, "event %d differs from the outside copy: this database was" + " rewritten" % seq, head) + return 0 + elif h != events.GENESIS: + _alarm(conn, "the outside copy is empty but its head is not genesis", head) + return 0 + sent = 0 + for _ in range(max_batches): + rows, size = [], 0 + for r in conn.execute("SELECT * FROM events WHERE seq > ? ORDER BY seq LIMIT ?", + (seq, batch)): + size += len(r["payload"]) + 512 + if rows and size > BATCH_BYTES: + break + rows.append(_row(r)) + if not rows: + break + ok, data = client.send(rows) + if not ok: + _alarm(conn, "the outside copy refused events after %d: %s" + % (seq, str(data.get("error", ""))[:300]), data.get("head")) + return sent + sent += len(rows) + seq, h = rows[-1]["seq"], rows[-1]["hash"] + with db.transaction(conn): + conn.execute("INSERT INTO kv (key, value) VALUES ('outside.head', ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", + (json.dumps({"seq": seq, "hash": h, "ms": db.now_ms()}),)) + return sent + + +def _last(conn): + return conn.execute("SELECT * FROM events ORDER BY seq DESC LIMIT 1").fetchone() + + +def behind(conn, client): + """``(local_seq, outside_seq)`` when the outside copy holds events after this + database's last one and this database is a prefix of it (restored from an + older copy); None otherwise (in step, or diverged: ship says how).""" + head = client.head() + theirs = int(head.get("seq") or 0) + last = _last(conn) + mine = last["seq"] if last else 0 + if theirs <= mine: + return None + if mine: + got = client.events(mine - 1, 1) + if not got or _row(last) != {k: got[0].get(k) for k in FIELDS}: + return None + return mine, theirs + + +def _well_formed(o): + """An event as this database writes them (what events._public can read).""" + try: + canonical = events._canonical(json.loads(o["payload"])) == o["payload"] + except (TypeError, ValueError): + return False + return (canonical and all(isinstance(o.get(k), int) and not isinstance(o.get(k), bool) + for k in ("seq", "ts_ms")) + and all(isinstance(o.get(k), str) and o.get(k) for k in ("kind", "subject", + "actor_id")) + and o.get("actor_kind") in events.ACTOR_KINDS + and (o.get("cause") is None or isinstance(o.get("cause"), str))) + + +def _bodies(missing, mirror): + """The bytes of the *missing* BOMs ({sha256: path}) from *mirror*, a checkout + of the bits-manifests mirror (manifests/), each checked by its sha256.""" + import hashlib + got = {} + for sha, path in missing.items(): + try: + with open(os.path.join(mirror, "manifests", path), "rb") as fh: + body = fh.read() + except OSError: + continue + if hashlib.sha256(body).hexdigest() == sha: + got[sha] = body + return got + + +def recover(conn, client, who, page=1000, mirror=None): + """Append the events this database lacks from the outside copy, each checked + to link onto this chain (all or none), and bring the certified set's tables + up to them; then outside.recovered. BOMs the set gained meanwhile come from + *mirror* (a bits-manifests checkout). Run with the worker stopped. Returns + how many events were appended.""" + n, broken = events.verify(conn) + if broken is not None: + raise OutsideError("this database's own chain breaks at event %d: restore another" + " copy" % broken) + gap = behind(conn, client) + if gap is None: + return 0 + mine, theirs = gap + last = _last(conn) + prev, after, rows = (last["hash"] if last else events.GENESIS), mine, [] + while after < theirs: + got = client.events(after, page) + if not got: + break + for o in got: + seq = o.get("seq") + if (seq != after + 1 or not _well_formed(o) + or o.get("prev_hash") != prev or events._digest(prev, o) != o.get("hash")): + raise OutsideError("the outside copy does not link at event %s: nothing recovered" + % seq) + rows.append({k: o.get(k) for k in FIELDS}) + prev, after = o["hash"], seq + if after == theirs: + break + if after != theirs: + raise OutsideError("the outside copy ends at event %d, before its head %d: nothing" + " recovered" % (after, theirs)) + kinds = {} + for r in rows: + kinds[r["kind"]] = kinds.get(r["kind"], 0) + 1 + from . import certified + # What the set gained comes from the mirror, not from these events alone: the + # outside copy's token could otherwise add to what the worker signs. + try: + live, decertified = certified.recovered_changes(rows) + except (KeyError, ValueError) as e: + raise OutsideError("a recovered event of the set is malformed (%s): nothing recovered" + % e) + bodies = _bodies(live, mirror) if mirror and live else {} + lacking = sorted("%s (%s)" % (path, sha[:12]) for sha, path in live.items() + if sha not in bodies) + if lacking: + raise OutsideError("the certified set gained %d BOM(s) since this copy: pass --mirror" + " ; missing: %s;" + " nothing recovered" % (len(lacking), ", ".join(lacking[:5]))) + with db.transaction(conn): + last = _last(conn) + if (last["seq"] if last else 0) != mine: + raise OutsideError("this database changed while recovering: stop the worker and" + " the console backend, then try again") + conn.execute("DELETE FROM kv WHERE key = 'outside.behind'") + conn.executemany("INSERT INTO events (seq, ts_ms, kind, subject, actor_kind, actor_id," + " cause, payload, prev_hash, hash) VALUES (:seq, :ts_ms, :kind," + " :subject, :actor_kind, :actor_id, :cause, :payload, :prev_hash," + " :hash)", rows) + try: + replayed = certified.replay(conn, rows, bodies) + except (ValueError, KeyError, sqlite3.IntegrityError) as e: + raise OutsideError("the certified set cannot follow the recovered events (%s:" + " %s): nothing recovered" % (type(e).__name__, e)) + if decertified: + # Their pre-approvals in this copy predate it: a new one certifies them. + blocked = json.loads(_kv(conn, "certify.decertified") or "{}") + blocked.update({b: db.now_ms() for b in decertified}) + conn.execute("INSERT INTO kv (key, value) VALUES ('certify.decertified', ?) ON" + " CONFLICT(key) DO UPDATE SET value = excluded.value", + (json.dumps(blocked, sort_keys=True),)) + alarm = _kv(conn, ALARM) + conn.execute("DELETE FROM kv WHERE key = ?", (ALARM,)) + conn.execute("INSERT INTO kv (key, value) VALUES ('outside.head', ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", + (json.dumps({"seq": theirs, "hash": prev, "ms": db.now_ms()}),)) + events.append(conn, "outside.recovered", "outside", events.Actor("person", who), + {"from": mine + 1, "to": theirs, "count": len(rows), "kinds": kinds, + "set_events": replayed, "decertified": decertified, + "first_ms": rows[0]["ts_ms"], "last_ms": rows[-1]["ts_ms"], + "alarm": json.loads(alarm) if alarm else None}) + return len(rows) + + +def sign_hold(conn, settings, now=None): + """Why the worker must not sign now, or None. It signs only while the outside + copy is in step with this database: not diverged, and checked by a recent + pass (BITS_ORCH_SIGN_OUTSIDE_MINUTES).""" + if not settings.orch_audit_url: + return None + alarm = _kv(conn, ALARM) + if alarm: + return ("the outside audit copy diverged (%s): no signing until it is resolved" + % str(json.loads(alarm).get("reason", ""))[:200]) + head = json.loads(_kv(conn, "outside.head") or "null") or {} + age = (now or db.now_ms()) - int(head.get("ms") or 0) + if age > settings.orch_sign_outside_minutes * 60 * 1000: + return ("the outside audit copy has not been checked for %d minutes: no signing until" + " it is" % (age // 60000) if head else + "the outside audit copy has not been checked yet: no signing until it is") + return None + + +def clear(conn, who): + """Clear the alarm once its cause is resolved, so shipping resumes.""" + alarm = _kv(conn, ALARM) + if not alarm: + return False + with db.transaction(conn): + # Forget the last head too: after a re-seeded outside copy, its head is + # the one to follow. + conn.execute("DELETE FROM kv WHERE key IN (?, 'outside.head')", (ALARM,)) + events.append(conn, "outside.cleared", "outside", events.Actor("person", who), + {"alarm": json.loads(alarm)}) + return True + + +def verify(conn, client, page=1000): + """Walk both copies. Returns (checked, problem): problem is None when the + outside copy is an intact prefix of (or equal to) this chain.""" + after, prev, checked = 0, events.GENESIS, 0 + while True: + got = client.events(after, page) + if not got: + return checked, None + for o in got: + seq = int(o.get("seq") or 0) + if seq != after + 1: + return checked, "outside copy skips from %d to %d" % (after, seq) + if o.get("prev_hash") != prev or events._digest(prev, o) != o.get("hash"): + return checked, "outside copy's own chain breaks at %d" % seq + mine = conn.execute("SELECT * FROM events WHERE seq = ?", (seq,)).fetchone() + if mine is None: + return checked, "event %d is outside but not here" % seq + if _row(mine) != {k: o.get(k) for k in _row(mine)}: + return checked, "event %d differs between here and outside" % seq + prev, after, checked = o["hash"], seq, checked + 1 + + +def main(argv=None): + from .. import config + ap = argparse.ArgumentParser(prog="python -m console_backend.orch.outside") + ap.add_argument("command", choices=["verify", "status", "clear", "recover"]) + ap.add_argument("--db", default=None, help="the orchestrator database (default BITS_ORCH_DB)") + ap.add_argument("--by", default=None, help="clear, recover: who did it (recorded)") + ap.add_argument("--mirror", default=None, + help="recover: a checkout of the bits-manifests mirror, for the BOMs the" + " set gained since the restored copy") + args = ap.parse_args(argv) + settings = config.Settings() + conn = db.open_checked(args.db or settings.orch_db) + if args.command == "status": + print(json.dumps(status(conn), indent=1)) + return 0 + if args.command == "clear": + if not args.by: + raise SystemExit("clear needs --by ") + print("cleared" if clear(conn, args.by) else "no alarm") + return 0 + client = Client.from_settings(settings) + if client is None: + raise SystemExit("BITS_ORCH_AUDIT_URL is not set") + if args.command == "recover": + if not args.by: + raise SystemExit("recover needs --by ") + path = args.db or settings.orch_db + try: + with db.worker_lock(path): + n = recover(conn, client, args.by, mirror=args.mirror) + except db.LockedError: + raise SystemExit("the worker is running: stop it first") + except OutsideError as e: + raise SystemExit(str(e)) + if not n: + print("nothing to recover: not behind the outside copy (or diverged from it:" + " see `verify`)") + return 0 + got = events.since(conn, after=conn.execute("SELECT MAX(seq) FROM events") + .fetchone()[0] - 1)[-1]["payload"] + print("recovered %d event(s), %d to %d: %s" % (n, got["from"], got["to"], ", ".join( + "%s %d" % kv for kv in sorted(got["kinds"].items())))) + return 0 + n, local_break = events.verify(conn) + checked, problem = verify(conn, client) + print("local chain: %d events, %s" % (n, "intact" if local_break is None + else "breaks at %d" % local_break)) + print("outside copy: %d events checked, %s" % (checked, problem or "matches this chain")) + return 0 if (problem is None and local_break is None) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/console-backend/console_backend/orch/phases.py b/console-backend/console_backend/orch/phases.py new file mode 100644 index 00000000..e9ba222d --- /dev/null +++ b/console-backend/console_backend/orch/phases.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""What a finished build pipeline's job spent its time on, for the console's +timeline: [kind, start_ms, end_ms] segments. + + queued from the request (the build queue's, else the pipeline's creation) + to the job's start + setup the job's start to the build (bits-phases.txt: build_start) + build bits build (build_start to build_end) + prepub per package, prepub's work: queueing, chunk/compress/upload, lock wait + gateway per package, the commit (gateway ingest and catalogs) + publish the publish step, when prepub recorded no measurements for it + +A failed job's `failed_in` says where it stopped, from the marks of the steps +it passed (build_ok, publish_start, finish_start): in the build, after it +(certification, the BOMs to the store), in the publish, or after the publish +step (finish); '' when it did not fail or its marks were not kept. + +The CI job writes bits-phases.txt (" " lines) and keeps +prepub's measurement records (measurements-.json) as artifacts; a +record covers [ts - total_s, ts], of which the last commit_s is the gateway's. +Overlapping packages are merged per kind. Without the artifacts (older or +failed jobs) the segments come from the job's own times. Derived, so no +events: it can be computed again, and is when the pipeline is retried. +""" + +import datetime +import json +import logging +import math +import re + +from . import build_queue, db + +log = logging.getLogger("bits_orchestrator.phases") + +MAX_AGE_MS = 7 * 24 * 3600 * 1000 # GitLab keeps the job's artifacts a week +PER_PASS = 20 +PHASES_FILE = "bits-phases.txt" +MEASUREMENTS_FILE = "measurements-%d.json" +_FRACTION = re.compile(r"\.(\d+)") + + +def _ms(text): + """GitLab's or Go's time (any offset, up to nanoseconds) in ms, or None.""" + if not text: + return None + try: + text = _FRACTION.sub(lambda m: "." + (m.group(1) + "000000")[:6], str(text), 1) + dt = datetime.datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return int(dt.timestamp() * 1000) + + +def build_job(jobs): + """The latest attempt of the pipeline's build job, or None.""" + build = [j for j in jobs if j.get("stage") == "build" and j.get("started_at")] + return max(build, key=lambda j: j.get("id", 0)) if build else None + + +def marks(text): + """{mark: ms} of bits-phases.txt; the last of a repeated mark wins.""" + out = {} + for line in (text or "").splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + out[parts[0]] = int(parts[1]) * 1000 + return out + + +def _merge(intervals): + out = [] + for s, e in sorted(i for i in intervals if i[1] > i[0]): + if out and s <= out[-1][1]: + out[-1][1] = max(out[-1][1], e) + else: + out.append([s, e]) + return out + + +def publish_segments(records): + """prepub and gateway segments of prepub's measurement records.""" + prepub, gateway = [], [] + for r in records if isinstance(records, list) else []: + if not isinstance(r, dict): + continue + end = _ms(r.get("ts")) + try: + total = float(r.get("total_s") or 0) + commit = min(max(float(r.get("commit_s") or 0), 0.0), total) + except (TypeError, ValueError): + continue + if end is None or not math.isfinite(total) or total <= 0: + continue + start, split = end - int(total * 1000), end - int(commit * 1000) + prepub.append((start, split)) + gateway.append((split, end)) + return ([["prepub", s, e] for s, e in _merge(prepub)] + + [["gateway", s, e] for s, e in _merge(gateway)]) + + +def segments(requested_ms, job, phase_marks, records): + """The segments of one pipeline's build job (see the module docstring), + all but queued within the job's times: three clocks, and a retried job's + measurements still hold the earlier attempt's records.""" + start, end = _ms(job.get("started_at")), _ms(job.get("finished_at")) + if start is None: + return [] + end = end or start + m = phase_marks or {} + out = [] + if requested_ms and requested_ms < start: + out.append(["queued", requested_ms, start]) + b0 = m.get("build_start", start) + b1 = m.get("build_end") or m.get("publish_start") or end + if b0 > start: + out.append(["setup", start, b0]) + out.append(["build", b0, max(b0, b1)]) + p0, p1 = m.get("publish_start"), m.get("publish_end") + pub = publish_segments(records) + if pub: + out += pub + elif p0 and p1 and p1 > p0: + out.append(["publish", p0, p1]) + def clip(t): + return min(max(t, start), end) + clipped = [s if s[0] == "queued" else [s[0], clip(s[1]), clip(s[2])] for s in out] + return sorted((s for s in clipped if s[2] > s[1] or s[0] == "build"), + key=lambda s: (s[1], s[2])) + + +def failed_in(job, phase_marks): + """Where a failed build job stopped, from its marks (see the module docstring).""" + m = phase_marks or {} + if job.get("status") != "failed" or not m: + return "" + if "build_ok" not in m: # build_end is written whatever the build's outcome + return "build" + if "finish_start" not in m: + return "publish" if "publish_start" in m else "certify" + return "finish" + + +def host(job, variables): + """The build host it ran on: the one it was pinned to (bits-host-, as + the queue pins its builds), else its runner's description.""" + try: + tag = str(json.loads(variables or "{}").get("BITS_RUNNER_TAG") or "") + except (ValueError, AttributeError): + tag = "" + if tag.startswith(build_queue.HOST_TAG) and tag != build_queue.HOST_TAG: + return tag[len(build_queue.HOST_TAG):] + return str(job.get("runner") or "") + + +def _artifact(reader, job_id, name): + from .. import forge_ops + try: + raw = reader.job_artifact(job_id, name) + except forge_ops.ForgeError as e: + if e.status in (0, 429) or e.status >= 500: + raise + log.warning("job %s: %s: %s", job_id, name, e) + return None + return raw.decode("utf-8", "replace") if raw is not None else None + + +def step(conn, reader): + """Record the segments of the recently finished build pipelines that have + none, or whose build job was retried since. Returns how many.""" + now = db.now_ms() + rows = conn.execute( + "SELECT p.id, p.jobs, p.variables, p.created_at, f.job_id AS done_job, (SELECT MIN(q.created_ms)" + " FROM build_queue q WHERE q.pipeline_id = p.id) AS queued_ms" + " FROM pipelines p LEFT JOIN pipeline_phases f ON f.pipeline_id = p.id" + " WHERE p.is_build = 1 AND p.outcome != 'running' AND p.first_seen_ms > ?" + " ORDER BY p.id DESC", (now - MAX_AGE_MS,)).fetchall() + n = 0 + for r in rows: + try: + job = build_job(json.loads(r["jobs"] or "[]")) + except ValueError: + job = None + if job is None or job.get("id") == r["done_job"]: + continue + text = _artifact(reader, job["id"], PHASES_FILE) + meas = _artifact(reader, job["id"], MEASUREMENTS_FILE % r["id"]) + try: + records = json.loads(meas) if meas else [] + except ValueError: + records = [] + requested = r["queued_ms"] or _ms(r["created_at"]) + m = marks(text) + segs = segments(requested, job, m, records) + with db.transaction(conn): + conn.execute("INSERT INTO pipeline_phases (pipeline_id, job_id, host, segments," + " failed_in, recorded_ms) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(pipeline_id)" + " DO UPDATE SET job_id = excluded.job_id, host = excluded.host," + " segments = excluded.segments, failed_in = excluded.failed_in," + " recorded_ms = excluded.recorded_ms", + (r["id"], job["id"], host(job, r["variables"]), json.dumps(segs), + failed_in(job, m), now)) + n += 1 + if n >= PER_PASS: + break + return n + + +def for_pipelines(conn, ids): + """{pipeline: {host, segments, failed_in}} of those recorded.""" + ids = sorted({int(i) for i in ids}) + if not ids: + return {} + rows = conn.execute("SELECT * FROM pipeline_phases WHERE pipeline_id IN (%s)" + % ",".join("?" * len(ids)), ids).fetchall() + return {r["pipeline_id"]: {"host": r["host"], "segments": json.loads(r["segments"]), + "failed_in": r["failed_in"]} for r in rows} diff --git a/console-backend/console_backend/orch/reconcile.py b/console-backend/console_backend/orch/reconcile.py new file mode 100644 index 00000000..1bcef6bd --- /dev/null +++ b/console-backend/console_backend/orch/reconcile.py @@ -0,0 +1,547 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Reconcile: what the worker knows of builds, from GitLab and S3, never from +what a job reports about itself. + +Each pass: + +1. hints - pipelines a CI job pointed at (POST /ops/ci/*): looked at now, + even when older than the discovery window. +2. discover - pipelines of the build project updated since the last pass + (with a few minutes' overlap); a new one's variables are read once. +3. settle - for every build pipeline still running, or changed on GitLab, the + outcome is decided from its CI jobs, as the console does: bits' + own progress status can hold a pipeline "running" after every job + finished, or fail it long after its jobs succeeded. +4. index - a pipeline's builds are read from the S3 build index + (BUILDS/by-pipeline//.json) and their BOMs, + once there: a build's index is written after all its tarballs + and BOMs are in the store, while the job goes on to CVMFS. +5. index_cli - a build made with the CLI is recorded once the worker certified + it, under pipeline 0 (CLI_PIPELINE), so it is scanned like the others. + +Each change is an event: pipeline.seen, pipeline.success|failed|canceled|running +(running again: a retry), build.published. +""" + +import datetime +import json +import logging +import re + +from .. import forge_ops +from . import db, events + +log = logging.getLogger("bits_orchestrator") + +BUILD_SOURCES = {"api", "trigger", "web"} +ACTIVE = {"created", "waiting_for_resource", "preparing", "pending", "running", + "scheduled", "manual"} +# A manual job nobody started is finished as far as the build is concerned. +TERMINAL_JOB = {"success", "failed", "canceled", "skipped", "manual"} +_VARS = ("COMMUNITY", "PACKAGE", "VERSION", "PLATFORM", "PUBLISH_CVMFS") +_QUEUE_VARS = ("BITS_RUNNER_TAG", "BITS_QUEUE_ENTRY") +MAX_VARIABLES = 16 * 1024 +OVERLAP = datetime.timedelta(minutes=5) +RECHECK_MS = 120 * 1000 # a running pipeline's jobs, at most this often +INDEX_RETRY_MS = 10 * 60 * 1000 # a finished pipeline with no index, looked at again +INDEX_GIVE_UP_MS = 6 * 3600 * 1000 +STALE_AFTER = datetime.timedelta(days=3) # a build job times out after 24 h +HINT_MAX_AGE_MS = 24 * 3600 * 1000 +MAX_PER_STEP = 50 # pipelines settled or indexed per pass +CLI_PIPELINE = 0 # the builds made with the CLI (db migration 18) + + +def _iso(dt): + return dt.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _when(text): + """GitLab's time (any offset, fractional seconds) as an aware datetime, or None.""" + try: + dt = datetime.datetime.fromisoformat(str(text).replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + return dt if dt.tzinfo else dt.replace(tzinfo=datetime.timezone.utc) + + +def _gone(e): + return isinstance(e, forge_ops.ForgeError) and e.status == 404 + + +def _kv_get(conn, key, default=None): + r = conn.execute("SELECT value FROM kv WHERE key = ?", (key,)).fetchone() + return r["value"] if r else default + + +def _kv_set(conn, key, value): + conn.execute("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", (key, value)) + + +def _subject(pid): + return "pipeline:%d" % int(pid) + + +# --- outcome from jobs ---------------------------------------------------- + +def outcome_from_jobs(gl_status, jobs): + """The pipeline's outcome: 'running', 'success', 'failed' or 'canceled'. + + From its latest job attempts, not from GitLab's pipeline status: decided once + every job is finished, or as soon as a build-stage job has failed (a later + status job may be wedged and cannot rescue it). With no jobs to go by, + GitLab's own status decides.""" + latest = {} + for j in jobs: + key = (j.get("stage"), j.get("name")) + if key not in latest or j.get("id", 0) > latest[key].get("id", 0): + latest[key] = j + jl = list(latest.values()) + if not jl: + if gl_status in ACTIVE: + return "running" + return {"success": "success", "canceled": "canceled", "skipped": "canceled"}.get( + gl_status, "failed") + hard_fail = [j for j in jl if j.get("status") == "failed" and not j.get("allow_failure")] + build = [j for j in jl if j.get("stage") == "build"] + build_failed = (build and all(j.get("status") in TERMINAL_JOB for j in build) + and any(j in hard_fail for j in build)) + if not build_failed and not all(j.get("status") in TERMINAL_JOB for j in jl): + return "running" + if hard_fail: + return "failed" + if any(j.get("status") == "canceled" for j in jl): + return "canceled" + if any(j.get("status") == "success" for j in jl): + return "success" + return "canceled" + + +def job_summary(jobs): + """What the console and the failure collection (M2) need of each job.""" + keep = ("id", "name", "stage", "status", "allow_failure", "failure_reason", + "started_at", "finished_at", "web_url") + out = [] + for j in sorted(jobs, key=lambda j: j.get("id", 0)): + d = {k: j.get(k) for k in keep if j.get(k) is not None} + runner = j.get("runner") or {} + if runner.get("description"): + d["runner"] = runner["description"] + if runner.get("id"): + d["runner_id"] = runner["id"] # descriptions may be empty or shared + out.append(d) + return out + + +# --- recording pipelines -------------------------------------------------- + +def _vars_dict(variables): + out = {} + for v in variables or []: + if isinstance(v, dict) and v.get("key"): + out[str(v["key"])] = str(v.get("value") or "") + return out + + +_NAME = re.compile(r"^\s*\[([^\]]+)\]\s*(\S+)\s*(.*?)\s*\(([^)]*)\)") + + +def _from_name(name): + """COMMUNITY, PACKAGE, VERSION, PLATFORM from the build pipeline's name, which + its workflow sets to '[COMMUNITY] PACKAGE VERSION (PLATFORM)': what + the console shows when the variables cannot be read.""" + m = _NAME.match(name or "") + if not m: + return {} + return dict(zip(("COMMUNITY", "PACKAGE", "VERSION", "PLATFORM"), + (g.strip() for g in m.groups()))) + + +def _var(variables, name): + """A variable by name, case-insensitively (as the backend's pipeline + community check reads COMMUNITY).""" + for k, v in variables.items(): + if k.upper() == name: + return v + return "" + + +def record_pipeline(conn, reader, p): + """Insert or update one pipeline from GitLab's listing *p*. A new one's + variables are read once (they never change). Returns its row.""" + pid = int(p["id"]) + old = conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + # Only what this response carries: the listing has no finished_at, the + # single-pipeline read does; a field it lacks is kept, not blanked. + names = {"status": "gl_status", "name": "name", "ref": "ref", "sha": "sha", + "web_url": "web_url", "created_at": "created_at", "updated_at": "updated_at", + "finished_at": "finished_at"} + fields = {col: str(p.get(key) or "") for key, col in names.items() if key in p} + if old is not None: + if all(old[k] == v for k, v in fields.items()): + return old + # Changed on GitLab: its jobs are looked at again on this pass (a retry + # makes it running again; a wedged status job after a failed build does not). + sets = ", ".join("%s = :%s" % (c, c) for c in sorted(fields)) + with db.transaction(conn): + conn.execute("UPDATE pipelines SET %s, checked_ms = 0 WHERE id = :id" % sets, + dict(fields, id=pid)) + return conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + is_build = (p.get("source") in BUILD_SOURCES) + variables = {} + if is_build: + # Unreadable variables (no ops token, GitLab refusing) never hold up + # discovery: the pipeline's name carries what is needed to record it. + try: + variables = _vars_dict(reader.pipeline_variables(pid)) + except Exception as exc: + log.warning("variables of pipeline %s: %s; using its name", pid, exc) + variables = _from_name(p.get("name")) + is_build = is_build and bool(_var(variables, "PACKAGE")) + text = json.dumps(variables, sort_keys=True) + if len(text) > MAX_VARIABLES: + # What the build queue reads of a pipeline is kept too. + text = json.dumps({k: _var(variables, k) for k in _VARS + _QUEUE_VARS}, sort_keys=True) + row = dict({col: "" for col in names.values()}, **fields) + row.update(id=pid, is_build=int(is_build), source=p.get("source") or "", + community=_var(variables, "COMMUNITY").strip().lower(), + package=_var(variables, "PACKAGE"), version=_var(variables, "VERSION"), + platform=_var(variables, "PLATFORM"), variables=text, + first_seen_ms=db.now_ms()) + with db.transaction(conn): + conn.execute("INSERT INTO pipelines (id, is_build, community, name, source, ref, sha," + " web_url, package, version, platform, variables, gl_status, created_at," + " updated_at, finished_at, first_seen_ms) VALUES (:id, :is_build," + " :community, :name, :source, :ref, :sha, :web_url, :package, :version," + " :platform, :variables, :gl_status, :created_at, :updated_at," + " :finished_at, :first_seen_ms)", row) + if is_build: + events.append(conn, "pipeline.seen", _subject(pid), events.WORKER, + {"pipeline": pid, "community": row["community"], + "package": row["package"], "version": row["version"], + "platform": row["platform"], "status": row["gl_status"]}) + return conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + + +def discover(conn, reader, lookback_days=7.0, max_rounds=20): + """Record the pipelines updated since the last pass. Returns how many. + + Always the first page after a moving start: a listing sorted by update time + shifts while it is read page by page (a pipeline that updates moves to the + end), which can skip one for good. The start moves to the newest update read + (minus a second; already-recorded pipelines cost nothing); when a full page + shares one second, the next page of that second is read instead. The next + pass starts a few minutes back to catch updates recorded out of order, unless + this pass stopped at max_rounds: then it continues exactly from here.""" + cursor = _when(_kv_get(conn, "gitlab.updated_after")) + page = 1 + if cursor is None: + since = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=lookback_days) + elif _kv_get(conn, "gitlab.continue") == "1": + since, page = cursor, int(_kv_get(conn, "gitlab.page", "1") or 1) + else: + since = cursor - OVERLAP + newest, n, truncated = cursor, 0, True + for _ in range(max_rounds): + got = reader.list_pipelines(_iso(since), page=page) + for p in got: + record_pipeline(conn, reader, p) + n += 1 + t = _when(p.get("updated_at")) + if t and (newest is None or t > newest): + newest = t + if len(got) < 100: + truncated = False + break + step = max((_when(p.get("updated_at")) for p in got if _when(p.get("updated_at"))), + default=since) - datetime.timedelta(seconds=1) + if step > since: + since, page = step, 1 + else: + page += 1 + with db.transaction(conn): + if truncated: # the next pass reads exactly this next + _kv_set(conn, "gitlab.updated_after", _iso(since)) + _kv_set(conn, "gitlab.page", str(page)) + elif newest is not None: + _kv_set(conn, "gitlab.updated_after", _iso(newest)) + _kv_set(conn, "gitlab.continue", "1" if truncated else "0") + return n + + +def apply_hints(conn, reader): + """Look now at the pipelines CI jobs pointed at. Unknown ones are fetched + directly; known ones are re-checked and re-indexed on this pass. A hint for + a pipeline GitLab does not have, or older than a day, is dropped.""" + hints = conn.execute("SELECT * FROM hints ORDER BY ts_ms LIMIT ?", + (MAX_PER_STEP,)).fetchall() + now = db.now_ms() + for h in hints: + pid, done = h["pipeline_id"], True + try: + if conn.execute("SELECT 1 FROM pipelines WHERE id = ?", (pid,)).fetchone() is None: + record_pipeline(conn, reader, reader.get_pipeline(pid)) + except Exception as exc: + done = _gone(exc) or now - h["ts_ms"] > HINT_MAX_AGE_MS + log.warning("hint for pipeline %s: %s%s", pid, exc, " (dropped)" if done else "") + with db.transaction(conn): + if done: + conn.execute("UPDATE pipelines SET checked_ms = 0, indexed_ms = 0" + " WHERE id = ?", (pid,)) + conn.execute("DELETE FROM hints WHERE pipeline_id = ? AND ts_ms = ?", + (pid, h["ts_ms"])) + return len(hints) + + +def _set_outcome(conn, r, outcome, jobs_json, extra=None): + with db.transaction(conn): + conn.execute("UPDATE pipelines SET outcome = ?, jobs = ?, checked_ms = ?" + " WHERE id = ?", (outcome, jobs_json, db.now_ms(), r["id"])) + if outcome == r["outcome"]: + if outcome == "failed" and jobs_json != r["jobs"]: + # A job failed after the build did (an on_failure job): collect again; + # jobs already recorded are skipped. + conn.execute("UPDATE pipelines SET collected_ms = 0 WHERE id = ?", (r["id"],)) + return False + # Finished (or running again): its builds are indexed, and its failures + # collected, when it ends. + conn.execute("UPDATE pipelines SET indexed_ms = 0, collected_ms = 0 WHERE id = ?", + (r["id"],)) + payload = {"pipeline": r["id"], "community": r["community"], + "gl_status": r["gl_status"]} + payload.update(extra or {}) + events.append(conn, "pipeline." + outcome, _subject(r["id"]), events.WORKER, payload) + return True + + +def settle(conn, reader): + """Decide the outcome of build pipelines not settled yet. Returns how many + changed. One pipeline GitLab cannot answer for does not hold up the others: + a deleted one (404) is settled from its last known status, any other error + is tried again on a later pass.""" + now = db.now_ms() + # Running ones at most every RECHECK_MS; any one GitLab changed (or a hint + # pointed at) now. + rows = conn.execute( + "SELECT * FROM pipelines WHERE is_build = 1 AND (checked_ms = 0 OR" + " (outcome = 'running' AND checked_ms <= ?)) ORDER BY checked_ms, id LIMIT ?", + (now - RECHECK_MS, MAX_PER_STEP)).fetchall() + stale_before = datetime.datetime.now(datetime.timezone.utc) - STALE_AFTER + changed = 0 + for r in rows: + try: + jobs = reader.pipeline_jobs(r["id"]) + except Exception as exc: + if _gone(exc): + out = outcome_from_jobs(r["gl_status"], []) + changed += _set_outcome(conn, r, "canceled" if out == "running" else out, + r["jobs"], {"gone": True}) + else: + log.warning("jobs of pipeline %s: %s", r["id"], exc) + with db.transaction(conn): + conn.execute("UPDATE pipelines SET checked_ms = ? WHERE id = ?", + (db.now_ms(), r["id"])) + continue + outcome = outcome_from_jobs(r["gl_status"], jobs) + summary = job_summary(jobs) + extra = {"failed_jobs": [j["name"] for j in summary + if j.get("status") == "failed" and not j.get("allow_failure")]} + created = _when(r["created_at"]) + if outcome == "running" and created and created < stale_before: + # Nothing runs this long: GitLab's word, or failed, and no more polling. + outcome = outcome_from_jobs(r["gl_status"], []) + outcome = "failed" if outcome == "running" else outcome + extra["stale"] = True + changed += _set_outcome(conn, r, outcome, json.dumps(summary), extra) + return changed + + +# --- builds from the S3 index --------------------------------------------- + +def _bom_facts(s3, bucket, keys): + """arches, package count, top-level packages and publish time of a build's + BOMs; an unreadable or non-BOM key is skipped (and logged).""" + from bits_helpers.cvmfs_release import get_json, is_bom + arches, packages, top, published = set(), 0, [], "" + for key in keys: + try: + bom = get_json(s3, bucket, key) + except Exception as exc: + log.warning("unreadable BOM %s: %s", key, exc) + continue + if not is_bom(bom): + continue + arches.add(str(bom["effective_architecture"])) + packages += len(bom["packages"]) + if not top and isinstance(bom.get("requested_packages"), list): + top = [str(x) for x in bom["requested_packages"]] + published = max(published, str(bom.get("published_at") or "")) + return sorted(arches), packages, top, published + + +def _index_one(conn, s3, bucket, r): + """Read one pipeline's build index and record its builds, all in one + transaction with indexed_ms, so a failure part way records nothing.""" + from bits_helpers.cvmfs_release import get_json, list_keys + found = {} + for key in list_keys(s3, bucket, "BUILDS/by-pipeline/%d/" % r["id"]): + try: + idx = get_json(s3, bucket, key) + except Exception as exc: + log.warning("unreadable build index %s: %s", key, exc) + continue + if isinstance(idx, dict) and idx.get("build_id"): + found[str(idx["build_id"])] = [k for k in idx.get("keys") or [] if isinstance(k, str)] + known = {b["build_id"] for b in conn.execute( + "SELECT build_id FROM builds WHERE pipeline_id = ?", (r["id"],))} + new = [(b, keys, _bom_facts(s3, bucket, keys)) for b, keys in sorted(found.items()) + if b not in known] + with db.transaction(conn): + conn.execute("UPDATE pipelines SET indexed_ms = ? WHERE id = ?", (db.now_ms(), r["id"])) + for build_id, keys, (arches, n, top, published) in new: + conn.execute("INSERT INTO builds (pipeline_id, build_id, community, arches," + " bom_keys, top_level, packages, published_at, recorded_ms)" + " VALUES (?,?,?,?,?,?,?,?,?)", + (r["id"], build_id, r["community"], json.dumps(arches), + json.dumps(keys), json.dumps(top), n, published, db.now_ms())) + events.append(conn, "build.published", _subject(r["id"]), events.WORKER, + {"pipeline": r["id"], "build_id": build_id, + "community": r["community"], "arches": arches, "packages": n}) + return len(new) + + +def index_cli(conn, store): + """Record the builds made with the CLI that the worker certified, as + pipeline CLI_PIPELINE's (again when certified again): their BOMs those it + signed, their community the pre-approval's group (a community with another + certify_group has its CLI builds apart). Returns how many were recorded.""" + if store is None: + return 0 + s3, bucket = store + rows = conn.execute( + "SELECT c.build_id, c.approval_id, c.updated_ms, a.groups FROM certify_runs c" + " JOIN approvals a ON a.id = c.approval_id LEFT JOIN builds b ON" + " b.pipeline_id = c.pipeline_id AND b.build_id = c.build_id" + " WHERE c.pipeline_id = ? AND c.arch = '*' AND c.state = 'signed'" + " AND (b.build_id IS NULL OR b.recorded_ms < c.updated_ms)" + " ORDER BY c.updated_ms LIMIT ?", (CLI_PIPELINE, MAX_PER_STEP)).fetchall() + n = 0 + for r in rows: + keys = sorted({k for (b,) in conn.execute( + "SELECT boms FROM certify_runs WHERE pipeline_id = ? AND build_id = ? AND arch != '*'" + " AND state = 'signed' AND approval_id = ?", (CLI_PIPELINE, r["build_id"], r["approval_id"])) + for k, _sha in json.loads(b or "[]")}) + arches, packages, top, published = _bom_facts(s3, bucket, keys) + # A BOM not read now is tried again on the next pass, for INDEX_GIVE_UP_MS. + if len(arches) < len(keys) and db.now_ms() - r["updated_ms"] < INDEX_GIVE_UP_MS: + continue + community = str((json.loads(r["groups"] or "[]") or [""])[0]).lower() + with db.transaction(conn): + conn.execute("INSERT INTO builds (pipeline_id, build_id, community, arches," + " bom_keys, top_level, packages, published_at, recorded_ms)" + " VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(pipeline_id, build_id) DO UPDATE" + " SET community = excluded.community, arches = excluded.arches," + " bom_keys = excluded.bom_keys, top_level = excluded.top_level," + " packages = excluded.packages, published_at = excluded.published_at," + " recorded_ms = excluded.recorded_ms", + (CLI_PIPELINE, r["build_id"], community, json.dumps(arches), + json.dumps(keys), json.dumps(top), packages, published, db.now_ms())) + events.append(conn, "build.published", "build:%s" % r["build_id"], events.WORKER, + {"pipeline": CLI_PIPELINE, "build_id": r["build_id"], + "community": community, "arches": arches, "packages": packages}) + n += 1 + return n + + +def index(conn, store): + """Record the builds of pipelines from the S3 build index, running ones + too. Returns how many builds were recorded. *store* is (s3, bucket) or None.""" + if store is None: + return 0 + s3, bucket = store + now = db.now_ms() + # Indexed when it starts or finishes (indexed_ms reset to 0), when a hint + # points at it, and, while it has no builds yet, every INDEX_RETRY_MS while it + # runs and until INDEX_GIVE_UP_MS after it finished (checked_ms). + rows = conn.execute( + "SELECT * FROM pipelines p WHERE is_build = 1 AND" + " (indexed_ms = 0 OR (indexed_ms <= ? AND checked_ms >= ? AND NOT EXISTS" + " (SELECT 1 FROM builds b WHERE b.pipeline_id = p.id))) ORDER BY indexed_ms, id" + " LIMIT ?", (now - INDEX_RETRY_MS, now - INDEX_GIVE_UP_MS, MAX_PER_STEP)).fetchall() + recorded = 0 + for r in rows: + try: + recorded += _index_one(conn, s3, bucket, r) + except Exception as exc: + log.warning("build index of pipeline %s: %s", r["id"], exc) + with db.transaction(conn): # tried again after INDEX_RETRY_MS + conn.execute("UPDATE pipelines SET indexed_ms = ? WHERE id = ?", + (db.now_ms(), r["id"])) + return recorded + + +# --- reads for the API ---------------------------------------------------- + +def _pipeline_public(r, builds): + return {"pipeline": r["id"], "community": r["community"], "name": r["name"], + "package": r["package"], "version": r["version"], "platform": r["platform"], + "source": r["source"], "ref": r["ref"], "web_url": r["web_url"], + "gl_status": r["gl_status"], "outcome": r["outcome"], + "created_at": r["created_at"], "updated_at": r["updated_at"], + "finished_at": r["finished_at"], + "builds": [{"build_id": b["build_id"], "state": b["state"], + "arches": json.loads(b["arches"]), "packages": b["packages"], + "top_level": json.loads(b["top_level"]), + "published_at": b["published_at"]} for b in builds]} + + +def list_builds(conn, communities=None, outcome=None, pipelines=None, limit=50): + """Build pipelines newest first, with their builds. *communities* (lowercase) + None = all; *pipelines* restricts to those ids.""" + q, args = "SELECT * FROM pipelines WHERE is_build = 1", [] + if communities is not None: + names = sorted(communities) + if not names: + return [] + q += " AND community IN (%s)" % ",".join("?" * len(names)) + args.extend(names) + if outcome: + q += " AND outcome = ?" + args.append(outcome) + if pipelines is not None: + ids = sorted({int(p) for p in pipelines})[:200] + if not ids: + return [] + q += " AND id IN (%s)" % ",".join("?" * len(ids)) + args.extend(ids) + q += " ORDER BY id DESC LIMIT ?" + args.append(max(1, min(int(limit), 200))) + rows = conn.execute(q, args).fetchall() + out = [] + for r in rows: + b = conn.execute("SELECT * FROM builds WHERE pipeline_id = ? ORDER BY build_id", + (r["id"],)).fetchall() + out.append(_pipeline_public(r, b)) + return out + + +def get_build(conn, pid): + r = conn.execute("SELECT * FROM pipelines WHERE id = ? AND is_build = 1", + (int(pid),)).fetchone() + if r is None: + return None + b = conn.execute("SELECT * FROM builds WHERE pipeline_id = ? ORDER BY build_id", + (r["id"],)).fetchall() + out = _pipeline_public(r, b) + out["jobs"] = json.loads(r["jobs"]) + out["variables"] = json.loads(r["variables"]) + out["events"] = events.since(conn, 0, 200, subject=_subject(r["id"])) + return out + + +def add_hint(conn, pid, kind): + with db.transaction(conn): + conn.execute("INSERT INTO hints (pipeline_id, kind, ts_ms) VALUES (?, ?, ?)" + " ON CONFLICT(pipeline_id) DO UPDATE SET kind = excluded.kind," + " ts_ms = excluded.ts_ms", (int(pid), kind, db.now_ms())) diff --git a/console-backend/console_backend/orch/replay.py b/console-backend/console_backend/orch/replay.py new file mode 100644 index 00000000..399d009b --- /dev/null +++ b/console-backend/console_backend/orch/replay.py @@ -0,0 +1,296 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The worker's own certification of a build, replayed beside the CI's (M8a). + +When the shadow (M4) compares a certification of the bits-manifests CI, the +worker also works out what it would have certified for the same build, on its +own terms, and compares that with what the CI signed (sign runs mode 'replay', +the evidence for the M8a gate): + + * the build: the one the push added BOMs for, by their source_pipeline_id, as + the CI job finds it (one build per push; a CLI build, a delete-only push or + BOMs from several builds are not the worker's to certify in M8a); + * its approval: a console pre-approval of that pipeline, approved before the + CI job started and neither expired nor revoked then. Nothing else approves + a certification: no pre-approval, no certification; + * its BOMs: the build's own copies in the store (the keys its build index + names, under MANIFESTS//), not the merge request's, each matched + to the MR's file of the same architecture and put at that file's path, so + the inputs are ordered as the CI's are; + * its group: the build's certify group (certify_group in the community's + ui-config.yaml at the pipeline's commit, else the community), which every + BOM's directory must be; + * the checks bits-manifests does: one architecture per BOM, the pre-approver + an admin of every group in the result (the backend's admin policy, not + ADMINS), the signing key allowed for every group by the key policy (and no + key policy is a refusal), the store holding the bytes (each object hashed + by the worker itself once, its digest kept while its key, ETag and size + stay the same: the digest cache the shadow uses); + * the baseline: the tree the CI certified, the build's files replaced by + the store's copies, and the architectures the CI certified, so the result + can equal the CI's exactly. + +Nothing is signed or written. A refusal, or a build the worker does not get to, +is a sign run too ('refused', 'missing'), with the reason. +""" + +import json +import logging +import os +import shutil + +from . import certified, db, shadow + +log = logging.getLogger("bits_orchestrator.replay") +MODE = "replay" +_UI_MAX = 1 << 20 + + +class Refuse(Exception): + """The worker would not certify this build (the reason is the message).""" + + +class Missing(Exception): + """The worker has nothing to certify this build from (the reason).""" + + +def _docs(root, paths): + out = {} + for p in paths: + try: + out[p] = json.loads(shadow._read(os.path.join(root, p))) + except ValueError: + out[p] = None + return out + + +def build_of(new_root, old_root): + """(pipeline id, {path: MR BOM}) of the build a push added BOMs for, as the + CI job's BUILD_INFO decides; Missing for what the worker leaves alone.""" + new, old = shadow._files(new_root), shadow._files(old_root) if old_root else {} + changed = sorted(p for p in set(new) | set(old) + if p not in new or p not in old + or shadow._read(new[p]) != shadow._read(old[p])) + added = [p for p in changed if p in new] + if not added: + raise Missing("a delete-only push (revocation)") + docs = _docs(new_root, added) + sids, bids = set(), set() + for d in docs.values(): + if not isinstance(d, dict): + continue + sid = str(d.get("source_pipeline_id") or "").strip() + if sid: + sids.add(sid) + else: + bids.add(str(d.get("build_id") or "").strip()) + if sids and bids or len(sids) > 1: + raise Missing("BOMs from more than one build in one push") + if not sids: + raise Missing("a CLI build (bits certify): not certified by the worker in M8a") + sid = next(iter(sids)) + if not sid.isdigit(): + raise Missing("source_pipeline_id %r is not a pipeline id" % sid) + return int(sid), docs + + +def preapproval_at(conn, build_id, at_ms): + """The console pre-approval of *build_id* in force at *at_ms*, or None.""" + return conn.execute( + "SELECT * FROM approvals WHERE kind = 'preapproval' AND subject = ? AND via = 'console'" + " AND created_ms <= ? AND expires_ms > ? AND (revoked_ms IS NULL OR revoked_ms > ?)" + " ORDER BY id DESC LIMIT 1", (str(build_id), at_ms, at_ms, at_ms)).fetchone() + + +def community_config(console, pipeline): + """(community, its ui-config.yaml at the pipeline's commit as a dict) of a + build pipeline; {} when the file is not there.""" + try: + variables = json.loads(pipeline["variables"] or "{}") + except ValueError: + variables = {} + community = str(variables.get("COMMUNITY") or pipeline["community"] or "").strip() + if not community: + raise Refuse("the build pipeline has no COMMUNITY") + if "/" in community or community.startswith("."): + raise Refuse("COMMUNITY %r is not a directory name" % community) + if console is None: + raise Missing("no reader for the build project (its ui-config.yaml)") + cfg = {} + if pipeline["sha"]: + import yaml + raw = console.file_raw("communities/%s/ui-config.yaml" % community, pipeline["sha"], + max_bytes=_UI_MAX) + if raw is not None: + try: + cfg = yaml.safe_load(raw) or {} + except yaml.YAMLError as e: + raise Refuse("communities/%s/ui-config.yaml does not parse: %s" + % (community, str(e)[:200])) + return community, (cfg if isinstance(cfg, dict) else {}) + + +def certify_group(console, pipeline): + """The group the build's certify job uses: its community's certify_group in + ui-config.yaml at the pipeline's commit, else the community lowercased (as + bits-certify.sh does).""" + community, cfg = community_config(console, pipeline) + return (str(cfg.get("certify_group") or "").strip() or community).lower() + + +def store_boms(conn, store, mr_docs, pid): + """{path: (key, the build's store BOM for that MR file's architecture, + whether it differs from the MR's)}, from the keys the pipeline's build + index names (a build id is shared by every run of the same release: its + store directory holds other pipelines' copies too).""" + from bits_helpers import certify + s3, bucket = store + out, by_build = {}, {} + for path, doc in sorted(mr_docs.items()): + if not isinstance(doc, dict): + raise Refuse("%s is not a JSON object" % path) + try: + arch = certify.bom_architecture(doc) or "" + except certify.CertifyError as e: + raise Refuse("%s: %s" % (path, e)) + bid = str(doc.get("build_id") or "") + if bid not in by_build: + # (Recorded when the build was first seen: a publish job retried in + # the same pipeline rewrites the same keys, almost always with the + # same content; a difference shows as store_differs_from_mr.) + row = conn.execute("SELECT bom_keys FROM builds WHERE pipeline_id = ? AND build_id = ?", + (pid, bid)).fetchone() + if row is None: + raise Missing("build %s of pipeline %s is not in the store's build index" % (bid, pid)) + docs = [] + for key in json.loads(row["bom_keys"] or "[]"): + try: + sdoc = json.loads(s3.get_object(Bucket=bucket, Key=key)["Body"].read()) + except ValueError: + continue + except Exception as e: + if shadow._missing(e): + continue # gone from the store + raise # transient: tried again; else an error run + if isinstance(sdoc, dict): + docs.append((key, sdoc)) + by_build[bid] = docs + hits = [] + for key, sdoc in by_build[bid]: + try: + if (certify.bom_architecture(sdoc) or "") == arch: + hits.append((key, sdoc)) + except certify.CertifyError: + continue + if not hits: + raise Missing("no store copy of %s's BOM for %s" % (bid or path, arch or "shared")) + if len(hits) > 1: + raise Missing("%d store BOMs of build %s for %s" % (len(hits), bid, arch or "shared")) + key, sdoc = hits[0] + if str(sdoc.get("source_pipeline_id") or "") != str(pid): + raise Refuse("the store's BOM %s is of pipeline %s, not %s" + % (key, sdoc.get("source_pipeline_id"), pid)) + out[path] = (key, sdoc, certified._strip(sdoc) != certified._strip(doc)) + return out + + +def _groups(manifests): + """The groups of the result, as the sign path reads them (main._groups_of): + an entry without one, or a manifest without entries, is 'common'.""" + groups = set() + for m in manifests.values(): + pkgs = (m or {}).get("packages") or [] + if not pkgs: + groups.add("common") + for p in pkgs: + g = p.get("group") if isinstance(p, dict) else None + groups.add(g if isinstance(g, str) and g else "common") + return sorted(groups) or ["common"] + + +def runs(conn, settings, console, store, sha, new_root, old_root, job, parsed, key_id, + compute): + """The replay's sign runs for this certification. *compute(root)* is the + shadow's computation (the CI's architectures).""" + try: + return _runs(conn, settings, console, store, new_root, old_root, job, parsed, + key_id, compute) + except Missing as e: + return [("*", "missing", "", 0, "", "", {"error": str(e)[:1000]})] + except Refuse as e: + return [("*", "refused", "", 0, "", "", {"error": str(e)[:1000]})] + except Exception as e: + if shadow.transient(e) or shadow._transient_cause(e): + raise # GitLab or the store: the step is tried again + log.warning("replay of %s: %s", sha[:12], e) + return [("*", "error", "", 0, "", "", {"error": "%s: %s" % (type(e).__name__, + str(e)[:1000])})] + + +def _runs(conn, settings, console, store, new_root, old_root, job, parsed, key_id, compute): + from .. import authz + from bits_helpers import trust + if key_id is None: + raise Missing("the CI's signed manifests do not verify against the trust anchor") + pid, mr_docs = build_of(new_root, old_root) + pipeline = conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + if pipeline is None: + raise Missing("build pipeline %s is not recorded" % pid) + at = shadow._ms((job or {}).get("started_at") or (job or {}).get("created_at")) or db.now_ms() + pre = preapproval_at(conn, pid, at) + if pre is None: + raise Refuse("no console pre-approval of pipeline %s in force when the CI signed" % pid) + user = pre["approver"] + group = certify_group(console, pipeline) + wrong = sorted({p.split("/")[0] for p in mr_docs} - {group}) + if wrong: + raise Refuse("BOMs under %s, but the build certifies into %s" % (", ".join(wrong), group)) + boms = store_boms(conn, store, mr_docs, pid) + archs = sorted({shadow._bom_arch(json.dumps(d)) or "?" for _k, d, _x in boms.values()}) + if "?" in archs: + raise Refuse("a store BOM whose architecture the CI cannot tell") + # The tree the CI certified (its deletions applied), the build's files + # replaced by the store's copies. + root = os.path.join(os.path.dirname(os.path.abspath(new_root)) + ".replay", "manifests") + shutil.rmtree(os.path.dirname(root), ignore_errors=True) + try: + shutil.copytree(new_root, root) + for path, (_key, doc, _x) in boms.items(): + dest = os.path.join(root, path) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "w") as fh: + json.dump(dict(doc, certified_by=[user]), fh, indent=1) + mine, refused = compute(root) + finally: + shutil.rmtree(os.path.dirname(root), ignore_errors=True) + if refused: + return [(refused[0], "refused") + refused[2:]] + groups = _groups(mine) + resolved = authz.resolved_admin_policy(settings, None) + denied = [g for g in groups if not authz.is_admin_for(user, g, resolved)] + if denied: + raise Refuse("%s (who pre-approved pipeline %s) is not an admin for %s" + % (user, pid, ", ".join(denied))) + policy = trust.load_key_policy() + if policy is None: + raise Refuse("no key policy (key-policy.json): the worker fails closed") + badk = [g for g in groups if not trust.key_authorized(key_id, g, policy)] + if badk: + raise Refuse("signing key %s is not allowed for %s" % (key_id, ", ".join(badk))) + extra = {"build": pid, "approval": pre["id"], "approver": user, "group": group, + "store": sorted(k for k, _d, _x in boms.values())} + differs = sorted(k for k, _d, x in boms.values() if x) + if differs: + extra["store_differs_from_mr"] = differs + ci_archs = sorted(a for a in parsed if parsed[a][2] != "superseded") + if archs != ci_archs: + # The push certified other architectures too (a deletion, or a file + # whose architecture it could not tell): computed as the CI did. + extra["archs"] = {"build": archs, "ci": ci_archs} + out = [] + for r in shadow.judge(mine, parsed): + d = dict(r[6]) + if r[1] != "match" or differs or "archs" in extra: + d["replay"] = extra + out.append(r[:6] + (d,)) + return out diff --git a/console-backend/console_backend/orch/sbom_history.py b/console-backend/console_backend/orch/sbom_history.py new file mode 100644 index 00000000..d1b2d915 --- /dev/null +++ b/console-backend/console_backend/orch/sbom_history.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Risk over time per release: what the Vulnerabilities page charts. + +A release is a community's top-level package (the pipeline's PACKAGE, else +the first package the build asked for): its compiler and build-type variants +are built from the same sources, so its newest scanned build stands for it (of +two published on one day, the higher build id). Its open findings (not +triaged, in shipped packages) count by severity, and their risk is grype's (by +severity alone where the scan has none, as in the upgrade plans). + +Once a day the worker records each release's counts and risk (sbom_history), +so a build grows riskier as new vulnerabilities are published. A day with no +record but a build of the release (before the first record, or the worker +down) gets that build's counts as they are when it is recorded, BACKFILL +builds a pass. Today's point is computed afresh.""" + +import datetime +import json + +from . import db, sbom_scan, vuln_triage +from .sbom_plan import SEVERITY_RISK +from .sbom_scan import FINDINGS_INDEX, SEVERITIES + +DAY_KEY = "sbom.history.day" # kv: the day whose records are complete +BACKFILL = 50 # earlier builds recorded in one pass, at most + + +def _day(ms): + return datetime.datetime.fromtimestamp(ms / 1000, datetime.timezone.utc).strftime("%Y-%m-%d") + + +def _builds(conn, communities=None): + """The indexed builds with a good scan: {build_id: (community, top, day)}; + a build of several pipelines is of the newest one's release, as + sbom_scan.listing shows it, and of the day it was first published.""" + out, pid = {}, {} + for r in conn.execute( + "SELECT b.pipeline_id, b.build_id, b.community, b.top_level, b.published_at," + " b.recorded_ms, p.package FROM builds b JOIN sbom_scans s ON s.build_id = b.build_id" + " LEFT JOIN pipelines p ON p.id = b.pipeline_id" + " WHERE s.index_version >= ? AND s.db_built != ''", (FINDINGS_INDEX,)): + bid = r["build_id"] + day = (r["published_at"] or "")[:10] or _day(r["recorded_ms"]) + if bid in out and r["pipeline_id"] < pid[bid]: + out[bid] = out[bid][:2] + (min(out[bid][2], day),) + continue + try: + asked = json.loads(r["top_level"] or "[]") + except ValueError: + asked = [] + top = r["package"] or (str(asked[0]) if asked else "") or bid + out[bid] = (r["community"], top, min(day, out[bid][2]) if bid in out else day) + pid[bid] = r["pipeline_id"] + if communities is not None: + out = {b: v for b, v in out.items() if v[0] in communities} + return out + + +def counts(conn, builds, now=None): + """{build_id: {severity: open findings, ..., "risk": their risk, "risks": + {severity: their risk}}} of *builds* ({build_id: community}).""" + out = {b: dict({k: 0 for k in SEVERITIES}, risk=0.0, risks={k: 0.0 for k in SEVERITIES}) + for b in builds} + if not builds: + return out + rules = vuln_triage.Rules.load(conn, set(builds.values()), now=now) + ids = sorted(builds) + for i in range(0, len(ids), 500): # under SQLite's variable limit + for f in sbom_scan._findings(conn, {b: builds[b] for b in ids[i:i + 500]}, rules): + if f["not_shipped"] or f["rule"]: + continue + c, risk = out[f["build_id"]], f["risk"] or SEVERITY_RISK.get(f["severity"], 0.5) + c[f["severity"]] += 1 + c["risks"][f["severity"]] += risk + for c in out.values(): + c["risks"] = {k: round(v, 2) for k, v in c["risks"].items()} + c["risk"] = round(sum(c["risks"].values()), 2) + return out + + +def _newest(builds, before=None): + """{(community, top): build_id}: each release's newest build (of a day + before *before*, if given: {(community, top, day): build_id} then).""" + newest = {} + for bid, (community, top, day) in builds.items(): + key = (community, top) if before is None else (community, top, day) + if before is not None and day >= before: + continue + if key not in newest or (day, bid) > (builds[newest[key]][2], newest[key]): + newest[key] = bid + return newest + + +def _record(conn, rows, got, replace): + """Record [((community, top, day), build_id)] with the builds' counts *got*.""" + for (community, top, day), bid in rows: + c = got[bid] + conn.execute( + "INSERT OR %s INTO sbom_history (day, community, top, build_id, critical, high," + " medium, low, other, risk, risks) VALUES (?,?,?,?,?,?,?,?,?,?,?)" + % ("REPLACE" if replace else "IGNORE"), + (day, community, top, bid, *[c[k] for k in SEVERITIES], c["risk"], + json.dumps(c["risks"], sort_keys=True))) + + +def snapshot(conn, now=None): + """Record each release's counts and risk for today, once a day, and the days + before with a build but no record (BACKFILL builds a pass; the day is + complete once none is left). Returns how many rows were recorded.""" + now = db.now_ms() if now is None else now + today = _day(now) + if (conn.execute("SELECT value FROM kv WHERE key = ?", (DAY_KEY,)).fetchone() or [""])[0] == today: + return 0 + builds = _builds(conn) + have = {(r["community"], r["top"], r["day"]) for r in conn.execute( + "SELECT community, top, day FROM sbom_history")} + rows = [((c, t, today), bid) for (c, t), bid in _newest(builds).items()] + missing = sorted((key, bid) for key, bid in _newest(builds, before=today).items() + if key not in have) + # Counted before the write lock is taken (today's again each pass of a backfill). + got = counts(conn, {bid: key[0] for key, bid in rows + missing[:BACKFILL]}, now=now) + with db.transaction(conn): + _record(conn, rows, got, replace=True) + _record(conn, missing[:BACKFILL], got, replace=False) + if len(missing) <= BACKFILL: + conn.execute("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key)" + " DO UPDATE SET value = excluded.value", (DAY_KEY, today)) + return len(rows) + len(missing[:BACKFILL]) + + +def history(conn, communities=None, days=90, now=None): + """{"releases": [{community, top, points: [{day, build_id, , risk, risks}]}], + "builds": [{community, top, build_id, day}]}: the releases of *communities* + (None: all) over the last *days* (0: all), oldest point first: the records, + and today's point afresh.""" + now = db.now_ms() if now is None else now + since = _day(now - days * 86400000) if days else "" + today = _day(now) + builds = _builds(conn, communities) + points = {} # (community, top) -> {day: point} + for r in conn.execute("SELECT * FROM sbom_history WHERE day >= ? AND day < ?", (since, today)): + if communities is None or r["community"] in communities: + points.setdefault((r["community"], r["top"]), {})[r["day"]] = dict( + {k: r[k] for k in SEVERITIES}, day=r["day"], build_id=r["build_id"], risk=r["risk"], + risks=json.loads(r["risks"] or "{}")) + newest = _newest(builds) + got = counts(conn, {b: builds[b][0] for b in newest.values()}, now=now) + for key, bid in newest.items(): + points.setdefault(key, {})[today] = dict(got[bid], day=today, build_id=bid) + releases = [{"community": c, "top": t, "points": [p[d] for d in sorted(p)]} + for (c, t), p in sorted(points.items())] + marks = sorted(({"community": c, "top": t, "build_id": b, "day": d} + for b, (c, t, d) in builds.items() if d >= since), + key=lambda m: (m["day"], m["build_id"])) + return {"releases": releases, "builds": marks} diff --git a/console-backend/console_backend/orch/sbom_licences.py b/console-backend/console_backend/orch/sbom_licences.py new file mode 100644 index 00000000..08983b0f --- /dev/null +++ b/console-backend/console_backend/orch/sbom_licences.py @@ -0,0 +1,434 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Licences and dependencies of the published builds, from their SBOMs (the +index sbom_scan keeps: each package's licence, redistribution, runtime and build +links, and the packages the release asks for, its top). + +Licences, per release line (its newest indexed build, as sbom_plan): every +package's licence (an SPDX expression) in coarse classes, and the findings: + +* conflict: a package whose runtime closure holds licences it cannot be + combined with (GPL-2.0-only with GPL-3.0, AGPL or Apache-2.0; GPL-3.0/AGPL + with GPL-2.0-only), each with its dependency path; +* unknown: a package with no licence, NOASSERTION, a LicenseRef or a name that + is no SPDX id (unreviewed). + +Build-only links form no combined work, so they are not followed, nor are the +standalone build tools a recipe may list among its runtime requires; packages +taken from the host system are not distributed by us. Copyleft reach lists, per +top package, the GPL and AGPL code in its runtime closure (its own apart), and +restricted the packages whose binaries may not be redistributed (left out of +publishing): both inform, they are not findings. The admins accept a finding with a reason, as a +vulnerability triage rule (vuln_triage) on the id licence:. + +Dependencies, per build: the packages most others depend on (how many rebuild +when one changes), those needed only to build, those nothing needs, and what +changed since the line's previous build; across lines, the packages built in +several versions. +""" + +import functools +import json +import re + +from . import vuln_triage + +KINDS = ("conflict", "unknown") +COPYLEFT = ("gpl2only", "gpl2plus", "gpl3", "agpl") +CAN_CONFLICT = ("gpl2only", "gpl3", "agpl") # the classes conflict() knows a rule for +STRICTNESS = {"permissive": 0, "weak": 1, "lgpl": 2, "gpl2plus": 3, "gpl2only": 4, "gpl3": 4, + "agpl": 5, "custom": 6, "unknown": 7} +# Exceptions that exist to permit combining: a licence carrying one grants it. +_EXCEPTION_OK = re.compile(r"^(GCC-exception|GCC-Runtime|Classpath-exception|LLVM-exception|" + r"Universal-FOSS-exception|Autoconf-exception|Bison-exception|" + r"Font-exception)", re.I) +# File-level (weak) copyleft: shown apart, no conflict rule. +_WEAK = re.compile(r"^(MPL|EPL|CDDL|EUPL|OSL|CPL|MS-RL|CECILL)-", re.I) +MAX_WITH = 20 # dependencies listed per conflict finding +MAX_ALTERNATIVES = 64 # ways to satisfy one licence expression, at most +# Standalone build tools, at times in a recipe's requires: no combined work. +BUILD_TOOLS = {n.lower() for n in ( + "CMake", "ninja", "gmake", "make", "m4", "autotools", "automake", "autoconf", "libtool", + "pkg_config", "pkgconfig", "gettext", "gperf", "bison", "flex", "swig", "ccache", + "patchelf", "GCC-Toolchain", "Clang", "bits-recipe-tools", "yacc-like", "doxygen")} +TOP_N = 50 +LICENCE_INDEX = 3 # sbom_scan's index this analysis needs (licences, links, top) + + +def finding_id(kind): + """The vuln_triage id a finding of *kind* is accepted under.""" + return "licence:%s" % kind + + +@functools.lru_cache(maxsize=1) +def _spdx_ids(): + from bits_helpers.spdx_ids import LICENSES + return frozenset(x.upper() for x in LICENSES) + + +def _id_class(lid): + """The class of one licence id: a GPL family member by its SPDX id (the + deprecated GPL-2.0 is -only, GPL-2.0+ -or-later), a LicenseRef or a name + that is no SPDX id (ids match whatever their case) is unreviewed, any + other SPDX id permissive or weak.""" + u = lid.upper() + if u in ("NOASSERTION", "NONE"): + return "unknown" + m = re.match(r"^(A|L)?GPL-(\d)\.\d(-ONLY|-OR-LATER|\+)?$", u) + if m: + kind, major, scope = m.group(1) or "", int(m.group(2)), m.group(3) or "" + if kind == "A": + return "agpl" + if kind == "L": + return "lgpl" + if major >= 3: + return "gpl3" + return "gpl2only" if major == 2 and scope in ("", "-ONLY") else "gpl2plus" + if u not in _spdx_ids() and u.rstrip("+") not in _spdx_ids(): + return "custom" + return "weak" if _WEAK.match(lid) else "permissive" + + +def _parse(lic): + """The alternatives of an SPDX expression: [(class, ids)], one per way to + satisfy it (an OR adds alternatives, an AND combines them: the strictest + class, the ids that apply). A licence WITH an exception that permits + combining is permissive, with no ids. Text that is not an expression is + one unreviewed alternative.""" + toks = re.findall(r"\(|\)|[^\s()]+", lic) + pos = [0] + + def peek(): + return toks[pos[0]].upper() if pos[0] < len(toks) else None + + def take(): + pos[0] += 1 + return toks[pos[0] - 1] + + def atom(): + if peek() == "(": + take() + out = either() + if peek() != ")": + raise ValueError("no )") + take() + return out + if peek() in (None, ")", "AND", "OR", "WITH"): + raise ValueError("a licence id expected") + lid = take() + if peek() == "WITH": + take() + if peek() in (None, "(", ")", "AND", "OR", "WITH"): + raise ValueError("an exception id expected") + if _EXCEPTION_OK.match(take()): + return [("permissive", frozenset())] + return [(_id_class(lid), frozenset([lid.upper()]))] + + def both(): + alts = atom() + while peek() == "AND": + take() + right = atom() + alts = [(max(a, b, key=STRICTNESS.get), x | y) for a, x in alts for b, y in right] + alts = alts[:MAX_ALTERNATIVES] + return alts + + def either(): + alts = both() + while peek() == "OR": + take() + alts = (alts + both())[:MAX_ALTERNATIVES] + return alts + + try: + out = either() + if pos[0] != len(toks): + raise ValueError("text after the expression") + return out + except (ValueError, IndexError): + return [("custom", frozenset())] + + +@functools.lru_cache(maxsize=4096) +def licence(lic): + """The alternatives of a licence as an SBOM gives it (see _parse).""" + lic = str(lic or "").strip() + return tuple(_parse(lic)) if lic else (("unknown", frozenset()),) + + +def licence_class(lic): + """The coarse class of a licence: its most permissive alternative's.""" + return min((c for c, _ in licence(lic)), key=STRICTNESS.get) + + +def _pair(a, b, ids, own, dep): + if a == "gpl2only" and b in ("gpl3", "agpl"): + return "GPL-2.0-only cannot be combined with %s" % dep + if a in ("gpl3", "agpl") and b == "gpl2only": + return "%s cannot be combined with GPL-2.0-only" % own + if a == "gpl2only" and "APACHE-2.0" in ids: + return "GPL-2.0-only cannot be combined with Apache-2.0" + return None + + +def conflict(own, dep): + """Why a package licensed *own* cannot use one licensed *dep*, or None: only + when no alternative of either licence can be combined with one of the other.""" + reasons = [_pair(a, b, ids, own, dep) for a, _ in licence(own) for b, ids in licence(dep)] + return reasons[0] if reasons and all(reasons) else None + + +def restricted(value): + """Do the binaries of a package marked *value* stay unpublished?""" + return str(value or "").strip().lower() in ("none", "sources", "false") + + +# ── a build's graph ────────────────────────────────────────────────────────── + +def _loads(text): + try: + out = json.loads(text or "[]") + except ValueError: + return [] + return [x for x in out if isinstance(x, str)] if isinstance(out, list) else [] + + +def graph(conn, build_id): + """{ref: package} of a build: its row, with `runtime` and `links` (all) lists.""" + out = {} + for r in conn.execute("SELECT * FROM sbom_packages WHERE build_id = ?", (build_id,)): + p = dict(r) + p["runtime"], p["links"] = _loads(p["runtime_on"]), _loads(p["depends_on"]) + p["label"] = p["alias"] or p["name"] + out[p["ref"]] = p + return out + + +def _follows(p): + return p is not None and not p["system"] and p["label"].lower() not in BUILD_TOOLS + + +def _walk(packages, starts, key="runtime", follow=_follows): + """{ref: parent ref (None at a start)} of what *starts* reach over *key* + links, breadth first, so each parent chain is a shortest path.""" + seen, todo = {s: None for s in starts if s in packages}, [s for s in starts if s in packages] + while todo: + nxt = [] + for ref in todo: + if not follow(packages[ref]) and seen[ref] is not None: + continue + for d in packages[ref][key]: + if d in packages and d not in seen: + seen[d] = ref + nxt.append(d) + todo = nxt + return seen + + +def _path(packages, parents, ref): + out = [] + while ref is not None: + out.append(packages[ref]["label"]) + ref = parents[ref] + return out[::-1] + + +def licences(packages, community, rules): + """The licence analysis of a build's *packages*: class counts, findings (each + with the triage rule accepting it, or None), copyleft reach, restricted.""" + classes, findings, reach = {}, [], [] + tops = [r for r, p in packages.items() if p["top"]] or list(packages) + + def add(kind, p, **kw): + rule = rules.rule(community, finding_id(kind), p["name"], p["version"], p["alias"]) + findings.append(dict(kind=kind, package=p["label"], version=p["version"], + licence=p["license"], rule=rule, **kw)) + + for ref, p in sorted(packages.items(), key=lambda kv: kv[1]["label"].lower()): + if p["system"]: + continue + cls = licence_class(p["license"]) + classes[cls] = classes.get(cls, 0) + 1 + if cls in ("custom", "unknown"): + add("unknown", p) + if cls not in CAN_CONFLICT or not _follows(p): + continue + parents, hits = _walk(packages, [ref]), [] + for d in parents: + q = packages[d] + why = d != ref and _follows(q) and conflict(p["license"], q["license"]) + if why: + hits.append({"package": q["label"], "licence": q["license"], "reason": why, + "path": _path(packages, parents, d)}) + if hits: + hits.sort(key=lambda h: (len(h["path"]), h["package"].lower())) + add("conflict", p, using=hits[:MAX_WITH], more=max(0, len(hits) - MAX_WITH)) + for ref in tops: + parents = _walk(packages, [ref]) + hits = [d for d in parents if d != ref and _follows(packages[d]) + and licence_class(packages[d]["license"]) in COPYLEFT] + if hits: + reach.append({"top": packages[ref]["label"], "licence": packages[ref]["license"], + "copyleft": [{"package": packages[d]["label"], + "licence": packages[d]["license"], + "path": _path(packages, parents, d)} for d in sorted( + hits, key=lambda d: packages[d]["label"].lower())]}) + kept = sorted(p["label"] for p in packages.values() if not p["system"] and restricted( + p["redistributable"])) + return {"classes": classes, "findings": findings, "copyleft": reach, "restricted": kept} + + +def dependencies(packages): + """The dependency analysis of a build's *packages*: the most depended-on + (rebuilds: how many depend on it, build links too; runtime: at run time), the + packages only builds need, and those nothing the release asks for needs.""" + users, runtime_users = {r: set() for r in packages}, {r: set() for r in packages} + for r, p in packages.items(): + for d in p["links"]: + if d in users: + users[d].add(r) + for d in p["runtime"]: + if d in runtime_users: + runtime_users[d].add(r) + + def closure(edges, ref): + seen, todo = set(), [ref] + while todo: + for u in edges[todo.pop()]: + if u not in seen: + seen.add(u) + todo.append(u) + return seen + + tops = [r for r, p in packages.items() if p["top"]] + shipped = set(_walk(packages, tops, follow=lambda p: True)) + needed = set(_walk(packages, tops, key="links", follow=lambda p: True)) + rows = [] + for r, p in packages.items(): + if p["system"]: + continue + rows.append({"package": p["label"], "version": p["version"], + "rebuilds": len(closure(users, r)), "runtime": len(closure(runtime_users, r)), + "direct": len(users[r])}) + rows.sort(key=lambda x: (-x["rebuilds"], x["package"].lower())) + names = lambda refs: sorted(packages[r]["label"] for r in refs if not packages[r]["system"]) + return {"packages": sum(not p["system"] for p in packages.values()), + "top": names(tops), "most_depended": rows[:TOP_N], + "build_only": names(needed - shipped) if tops else [], + "unneeded": names(set(packages) - needed) if tops else []} + + +def diff(old, new): + """What changed from build *old* to *new* (two graphs): packages added, + removed, in another version, under another licence.""" + key = lambda pk: {p["label"].lower(): p for p in pk.values() if not p["system"]} + a, b = key(old), key(new) + out = {"added": [], "removed": [], "version": [], "licence": []} + for k in sorted(set(a) | set(b)): + if k not in a: + out["added"].append({"package": b[k]["label"], "version": b[k]["version"]}) + elif k not in b: + out["removed"].append({"package": a[k]["label"], "version": a[k]["version"]}) + else: + if a[k]["version"] != b[k]["version"]: + out["version"].append({"package": b[k]["label"], "from": a[k]["version"], + "to": b[k]["version"]}) + if a[k]["license"] != b[k]["license"] and a[k]["license"] and b[k]["license"]: + out["licence"].append({"package": b[k]["label"], "from": a[k]["license"], + "to": b[k]["license"]}) + return out + + +# ── the release lines ──────────────────────────────────────────────────────── + +def _lines(conn, communities): + """The newest build of each release line (sbom_plan), with whether it is + indexed for this analysis.""" + from .sbom_plan import release_lines + lines = release_lines(conn, communities) + for r in lines: + v = conn.execute("SELECT index_version FROM sbom_scans WHERE build_id = ?", + (r["build_id"],)).fetchone() + r["indexed"] = bool(v and v["index_version"] >= LICENCE_INDEX) + return lines + + +def _head(r): + return {k: r.get(k) for k in ("build_id", "community", "line", "platform", "package", + "version", "pipeline_id", "published_at", "indexed")} + + +def licence_lines(conn, communities=None, now=None): + """{"releases": each release line of *communities*, with its open and + accepted findings by kind (once indexed)}.""" + lines = _lines(conn, communities) + rules = vuln_triage.Rules.load(conn, {r["community"] for r in lines}, now=now) + out = [] + for r in lines: + row = _head(r) + if r["indexed"]: + fs = licences(graph(conn, r["build_id"]), r["community"], rules)["findings"] + row["open"] = {k: sum(f["kind"] == k and not f["rule"] for f in fs) for k in KINDS} + row["accepted"] = sum(bool(f["rule"]) for f in fs) + out.append(row) + return {"releases": out} + + +def licence_report(conn, build_id, communities=None, now=None): + """The licence analysis of a release line's newest build *build_id*; None + when it is not one of them.""" + r = next((x for x in _lines(conn, communities) if x["build_id"] == build_id), None) + if r is None: + return None + out = _head(r) + if r["indexed"]: + rules = vuln_triage.Rules.load(conn, {r["community"]}, now=now) + out.update(licences(graph(conn, build_id), r["community"], rules)) + return out + + +def previous(conn, release): + """The build the line of *release* published before it, or None.""" + from .sbom_scan import FINDINGS_INDEX + rows = conn.execute( + "SELECT b.build_id, MAX(b.published_at) AS published_at FROM builds b" + " JOIN sbom_scans s ON s.build_id = b.build_id" + " LEFT JOIN pipelines p ON p.id = b.pipeline_id" + " WHERE b.community = ? AND COALESCE(p.platform, '') = ? AND b.build_id != ?" + " AND s.index_version >= ? GROUP BY b.build_id", + (release["community"], release["platform"] or "", release["build_id"], + FINDINGS_INDEX)).fetchall() + from .sbom_plan import _LINE + older = [x for x in rows if _LINE.sub("", x["build_id"]) == release["line"] + and x["published_at"] <= (release["published_at"] or "")] + return max(older, key=lambda x: x["published_at"])["build_id"] if older else None + + +def dependency_lines(conn, communities=None): + """{"releases": the release lines, "versions": the packages the newest builds + of the lines have in several versions}.""" + lines = _lines(conn, communities) + seen = {} + for r in lines: + for p in graph(conn, r["build_id"]).values(): + if not p["system"]: + seen.setdefault(p["label"].lower(), {"package": p["label"], "versions": {}} + )["versions"].setdefault(p["version"], []).append( + "%s/%s (%s)" % (r["community"], r["line"], r["platform"] or "?")) + versions = sorted((v for v in seen.values() if len(v["versions"]) > 1), + key=lambda v: (-len(v["versions"]), v["package"].lower())) + return {"releases": [_head(r) for r in lines], "versions": versions} + + +def dependency_report(conn, build_id, communities=None): + """The dependency analysis of a release line's newest build *build_id*, with + the diff from the line's previous build; None when it is not one of them.""" + release = next((r for r in _lines(conn, communities) if r["build_id"] == build_id), None) + if release is None: + return None + new = graph(conn, build_id) + out = dict(_head(release), **dependencies(new)) + before = previous(conn, release) + out["previous"] = before + out["diff"] = diff(graph(conn, before), new) if before else None + return out diff --git a/console-backend/console_backend/orch/sbom_plan.py b/console-backend/console_backend/orch/sbom_plan.py new file mode 100644 index 00000000..242b4ef2 --- /dev/null +++ b/console-backend/console_backend/orch/sbom_plan.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Upgrade plans from the scanned builds (sbom_scan's index): what to upgrade, to +which version and in which order, to remove the most security risk. + +A release line is a community's build of one release name on one platform; its +plan is made from its newest indexed build. For each package with open findings +(not covered by the triage; a component the stack does not ship has none), the +target is the smallest stable version newer than the current one that the +scanner names as fixing each of them. The packages fall +into waves, each meant as one set of version bumps and one rebuild, so the +release stays consistent after each: 0 known exploited (CISA KEV), 1 upgrades +within the major version, 2 major upgrades; packages taken from the host system +are listed apart (an image or host update), and findings no version fixes are +left to the triage. A wave is ordered by the risk it removes (grype's score, which +weighs severity, EPSS and KEV) per package rebuilt (the package and all that +depend on it). + +The converged plan then picks one version per package for all the release lines +(of all the communities in scope), the highest any of them needs, preferring a +version already built and scanned clean; a release line whose version is pinned +(a defaults override, another package's requires) is reported as an exception. +""" + +import functools +import json +import re + +from . import vuln_triage + +# A finding's risk when the scan has none of grype's (0-100): by severity alone. +SEVERITY_RISK = {"critical": 50.0, "high": 25.0, "medium": 10.0, "low": 2.0, "other": 0.5} +WAVES = ((0, "Known exploited"), (1, "Upgrades within the major version"), (2, "Major upgrades")) +_LINE = re.compile(r"-[0-9a-f]{12}$") + + +_PRE_WORDS = {"alpha", "beta", "rc", "pre", "preview", "dev", "snapshot"} +_PRE_LETTERS = {"a", "b", "c"} # 1.0a1, 2.0b3; not 1.1.1c (OpenSSL) nor hashes + + +_EPOCH, _TOKEN, _V = re.compile(r"^\d+:"), re.compile(r"\d+|[A-Za-z]+"), re.compile(r"^[vV](?=\d)") +_LETTER = re.compile(r"\d([A-Za-z])\d+$") + + +@functools.lru_cache(maxsize=65536) +def _vkey(v): + """A sort key for versions: an epoch (1:...) first, numbers numerically, then + words; a pre-release (rc, beta, ...) before the end, other words after it.""" + epoch, _, rest = str(v).partition(":") if _EPOCH.match(str(v)) else ("0", "", str(v)) + tokens = _TOKEN.findall(_V.sub("", rest)) + letter = _LETTER.search(rest) # 1.0a1: a letter between the last digits only + parts = [int(epoch or 0)] + for i, t in enumerate(tokens): + w = t.lower() + pre = w in _PRE_WORDS or (letter is not None and i + 2 == len(tokens) and w in _PRE_LETTERS) + parts.append((0, int(t)) if t.isdigit() else (-2, w) if pre else (-1, w)) + return tuple(parts) + ((-1.5, ""),) + + +def _prerelease(v): + return any(p[0] == -2 for p in _vkey(v)[1:]) + + +def _major(v): + """The first number of a version (its epoch aside).""" + k = _vkey(v) + return k[1] if len(k) > 2 else None + + +def release_lines(conn, communities=None): + """The newest indexed build of each release line of *communities* (None: all).""" + from .sbom_scan import FINDINGS_INDEX, _in + q = ("SELECT b.build_id, b.community, MAX(b.pipeline_id) AS pipeline_id, p.package," + " p.version, p.platform, MAX(b.published_at) AS published_at, s.scanned_ms," + " s.components, s.identified" + " FROM builds b JOIN sbom_scans s ON s.build_id = b.build_id" + " LEFT JOIN pipelines p ON p.id = b.pipeline_id WHERE s.index_version >= ?") + args = [FINDINGS_INDEX] + if communities is not None: + if not communities: + return [] + cond, vals = _in("b.community", communities) + q += " AND " + cond + args += vals + newest = {} + for r in conn.execute(q + " GROUP BY b.build_id", args): + r = dict(r) + r["line"] = _LINE.sub("", r["build_id"]) + key = (r["community"], r["line"], r["platform"] or "") + if key not in newest or (r["published_at"], r["scanned_ms"]) > ( + newest[key]["published_at"], newest[key]["scanned_ms"]): + newest[key] = r + return sorted(newest.values(), key=lambda r: (r["community"], r["line"], r["platform"] or "")) + + +def _need(fixed_in, current): + """The smallest stable fixing version newer than *current*, or None.""" + newer = [f for f in fixed_in if not _prerelease(f) and _vkey(f) > _vkey(current)] + return min(newer, key=_vkey) if newer else None + + +def _dependents(packages): + """A function: ref -> the refs that depend on it, directly or not (memoised).""" + users, memo = {r: set() for r in packages}, {} + for r, p in packages.items(): + for d in p["depends_on"]: + if d in users: + users[d].add(r) + + def of(ref): + if ref not in memo: + seen, todo = set(), [ref] + while todo: + for u in users[todo.pop()]: + if u not in seen: + seen.add(u) + todo.append(u) + memo[ref] = seen - {ref} + return memo[ref] + return of + + +def plan_release(conn, rel, rules): + """The plan of the release line *rel* (from release_lines), with *rules*.""" + bid, community = rel["build_id"], rel["community"] + packages = {} + for r in conn.execute("SELECT * FROM sbom_packages WHERE build_id = ?", (bid,)): + try: + on = [d for d in json.loads(r["depends_on"]) if isinstance(d, str)] + except (TypeError, ValueError): + on = [] + packages[r["ref"]] = dict(r, depends_on=on, findings=[], triaged=0, unshipped=0) + for f in conn.execute("SELECT * FROM sbom_findings WHERE build_id = ?", (bid,)): + p = packages.get(f["ref"]) + if p is None: + continue + if p["not_shipped"]: # a host tool, what only tools use, build-only + p["unshipped"] += 1 + continue + if rules.rule(community, f["vuln_id"], p["name"], p["version"], p["alias"]): + p["triaged"] += 1 + continue + try: + fixed = [x for x in json.loads(f["fixed_in"]) if isinstance(x, str)] + except (TypeError, ValueError): + fixed = [] + p["findings"].append({"id": f["vuln_id"], "severity": f["severity"], "kev": f["kev"], + "epss": f["epss"], "need": _need(fixed, p["version"]), + "risk": f["risk"] or SEVERITY_RISK.get(f["severity"], 0.1)}) + users = _dependents(packages) + items, host, unfixed = [], [], [] + for ref, p in packages.items(): + if not p["findings"]: + continue + fixed = [f for f in p["findings"] if f["need"]] + left = [f for f in p["findings"] if not f["need"]] + entry = {"ref": ref, "name": p["alias"] or p["name"], "sbom_name": p["name"], + "version": p["version"], "pinned_by": p["pinned_by"], "system": bool(p["system"]), + "fixes": sorted(f["id"] for f in fixed), "unfixed": sorted(f["id"] for f in left), + "kev": sorted(f["id"] for f in fixed if f["kev"]), + "risk_removed": round(sum(f["risk"] for f in fixed), 2), + "risk_left": round(sum(f["risk"] for f in left), 2), "triaged": p["triaged"], + "rebuild": len(users(ref)) + 1 if fixed else 0} + if left: + unfixed.append(entry) + if not fixed: + continue + entry["target"] = max((f["need"] for f in fixed), key=_vkey) + entry["major"] = _major(entry["target"]) != _major(p["version"]) + if p["system"]: + host.append(entry) + continue + entry["wave"] = 0 if entry["kev"] else 2 if entry["major"] else 1 + items.append(entry) + items.sort(key=lambda e: (e["wave"], -e["risk_removed"] / e["rebuild"], e["name"].lower())) + total = sum(f["risk"] for p in packages.values() for f in p["findings"]) + waves, left_risk, rebuilt = [], total, set() + for w, title in WAVES: + ws = [e for e in items if e["wave"] == w] + if not ws: + continue + removed = sum(e["risk_removed"] for e in ws) + left_risk -= removed + refs = set() + for e in ws: + refs |= users(e["ref"]) | {e["ref"]} + rebuilt |= refs + waves.append({"wave": w, "title": title, "packages": ws, "rebuild": len(refs), + "risk_removed": round(removed, 2), "risk_left": round(max(left_risk, 0), 2)}) + return dict(rel, components=len(packages), risk=round(total, 2), waves=waves, + host=sorted(host, key=lambda e: -e["risk_removed"]), + unfixed=sorted(unfixed, key=lambda e: -e["risk_left"]), + findings=sum(len(p["findings"]) for p in packages.values()), + triaged=sum(p["triaged"] for p in packages.values()), + not_shipped=sum(p["unshipped"] for p in packages.values())) + + +def converge(conn, plans): + """One version per package for all the release lines of *plans*: for each + package one of them needs to upgrade, the highest version any needs, or a + higher one already in use and scanned clean; per release line, the upgrade, + or why it cannot follow (a pin).""" + seen = {} # package (lowercase bits name) -> [(plan, its package row, its plan entry)] + for pl in plans: + entries = {e["ref"]: e for w in pl["waves"] for e in w["packages"]} + pl["_open"] = {e["ref"] for e in pl["unfixed"]} | set(entries) + for r in conn.execute("SELECT ref, name, alias, version, system, pinned_by FROM sbom_packages" + " WHERE build_id = ? AND system = 0 AND not_shipped = ''", + (pl["build_id"],)): + seen.setdefault((r["alias"] or r["name"]).lower(), []).append((pl, dict(r), entries.get(r["ref"]))) + out = [] + for key, uses in seen.items(): + if not any(e for _, _, e in uses): + continue + # what each line needs: its target, or its own version where it has no open finding + needs = [e["target"] if e else r["version"] for pl, r, e in uses + if e or r["ref"] not in pl["_open"]] + version = max(needs, key=_vkey) + while True: # a version in use that is itself vulnerable: what it needs + higher = [e["target"] for _, r, e in uses if e and r["version"] == version] + if not higher or max(higher, key=_vkey) == version: + break + version = max(higher, key=_vkey) + clean = [pl["build_id"] for pl, r, e in uses + if r["version"] == version and r["ref"] not in pl["_open"]] + lines = [] + for pl, r, e in uses: + line = {"build_id": pl["build_id"], "community": pl["community"], "line": pl["line"], + "platform": pl["platform"], "version": r["version"], "pinned_by": r["pinned_by"], + "risk_removed": e["risk_removed"] if e else 0, "kev": e["kev"] if e else [], + "target": e["target"] if e else ""} + if r["version"] == version: + line["action"] = "keep" + elif r["pinned_by"]: + line["action"] = "pinned" # its own target, or change the pin + else: + line["action"] = "upgrade" + line["major"] = _major(version) != _major(r["version"]) + lines.append(line) + name = uses[0][1]["alias"] or uses[0][1]["name"] + out.append({"name": name, "version": version, "in_use_clean": clean, + "kev": sorted({k for ln in lines for k in ln["kev"]}), + "risk_removed": round(sum(ln["risk_removed"] for ln in lines), 2), + "pinned": sum(ln["action"] == "pinned" for ln in lines), + "releases": sorted(lines, key=lambda ln: (ln["community"], ln["line"]))}) + out.sort(key=lambda c: (not c["kev"], -c["risk_removed"], c["name"].lower())) + return out + + +def plan(conn, communities=None, now=None): + """{"releases": the plan of each release line, "converged": converge(...)}.""" + lines = release_lines(conn, communities) + rules = vuln_triage.Rules.load(conn, {r["community"] for r in lines}, now=now) + plans = [plan_release(conn, r, rules) for r in lines] + converged = converge(conn, plans) + for pl in plans: + pl.pop("_open", None) + return {"releases": plans, "converged": converged} diff --git a/console-backend/console_backend/orch/sbom_scan.py b/console-backend/console_backend/orch/sbom_scan.py new file mode 100644 index 00000000..d39cc58c --- /dev/null +++ b/console-backend/console_backend/orch/sbom_scan.py @@ -0,0 +1,542 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Vulnerability scans of the published builds' SBOMs. + +The worker step sends each build's sbom.cdx.json (MANIFESTS//) to the +sbom-scanner service (BITS_ORCH_SCANNER_URL, bits-services), which matches it with +grype and returns it with a CycloneDX `vulnerabilities` section. A build is +scanned once, again whenever the scanner's vulnerability database is newer than +the one of its last good scan, so old builds learn of new CVEs, and again when a +newer pipeline publishes the build (a re-run writes its SBOM anew). + +sbom_scans keeps each build's counts by severity (findings: a vulnerability in a +package) and the last error; the `vulnerabilities` section itself is a gzip file +beside the database (sbom-scans/, rebuilt by scanning again, so not in its +backups). A failed scan keeps the last good counts. A change of a build's counts +is an event (sbom.scanned). + +A good scan also indexes the SBOM: its components (sbom_packages: which builds +contain a package), what matched (sbom_findings), and how many components a +scanner can identify at all (a CPE, or a purl of an ecosystem it knows): a build +with none is not scannable, and its zero findings say nothing. The findings the +admins' triage covers (vuln_triage) are left out of the effective counts, as are +those of components not shipped with the stack (not_shipped): host tools (a +recipe's `sbom: tool`), what only they use, and system packages nothing the +release asks for uses at run time. Each +package's licence, redistribution, runtime links and whether the release asks +for it (top) are kept for the licence analysis (sbom_licences).""" + +import gzip +import hashlib +import json +import logging +import os +import re +import time + +from . import db, events, vuln_triage + +log = logging.getLogger("bits_orchestrator.sbom_scan") + +PER_PASS = 3 # builds scanned in one worker pass, at most +BUDGET_S = 20 # no new scan once a pass has spent this long +RETRY_MS = 3600 * 1000 # a failed scan is tried again after this +SETTLE_MS = 10 * 60 * 1000 # a new build's SBOM follows its build index +TIMEOUT_S = (5, 60) # connect, read: the scanner gives up at 45 s +SEVERITIES = ("critical", "high", "medium", "low", "other") +SBOM = "sbom.cdx.json" +INDEX_VERSION = 5 # a good scan older than this indexes its build again +FINDINGS_INDEX = 2 # the index the findings and the plans need +# purl types grype matches (pkg:generic and pkg:github it cannot) +MATCHED_PURLS = {"apk", "cargo", "cocoapods", "composer", "conan", "deb", "gem", "golang", + "hackage", "hex", "maven", "npm", "nuget", "pub", "pypi", "rpm", "swift"} + + +class Failed(Exception): + pass + + +class Busy(Exception): + """The scanner cannot take a scan now (updating its database): stop the pass.""" + + +def _get(url): + import requests + r = requests.get(url, timeout=TIMEOUT_S) + r.raise_for_status() + return r.json() + + +def _post(url, body): + import requests + r = requests.post(url, data=body, timeout=TIMEOUT_S, + headers={"Content-Type": "application/json"}) + if r.status_code == 503: + raise Busy(r.text[:200]) + if r.status_code >= 400: + raise Failed("scanner answered %d: %s" % (r.status_code, r.text[:200])) + return r.json() + + +def _missing(e): + # Without ListBucket a missing key answers 403: there is nothing to read either way. + code = str(getattr(e, "response", {}).get("Error", {}).get("Code", "")) + return isinstance(e, KeyError) or code in ("NoSuchKey", "404", "NotFound", "AccessDenied", "403") + + +def _path(directory, bid): + safe = re.sub(r"[^A-Za-z0-9._-]", "_", bid) + if safe != bid or safe.startswith("."): + safe = hashlib.sha256(bid.encode()).hexdigest() + return os.path.join(directory, safe + ".vulns.json.gz") + + +def _write(directory, bid, vulns): + os.makedirs(directory, exist_ok=True) + path = _path(directory, bid) + tmp = path + ".tmp" + with open(tmp, "wb") as f: + f.write(gzip.compress(json.dumps(vulns, sort_keys=True).encode(), mtime=0)) + os.replace(tmp, path) + + +def vulnerabilities(directory, bid): + """Build *bid*'s `vulnerabilities` from its last good scan, or None.""" + if not directory: + return None + try: + with open(_path(directory, bid), "rb") as f: + return json.loads(gzip.decompress(f.read())) + except FileNotFoundError: + return None + + +def _props(c): + """A CycloneDX component's properties as {name: value} (strings).""" + props = c.get("properties") + return {str(p.get("name")): str(p.get("value")) for p in props if isinstance(p, dict) + } if isinstance(props, list) else {} + + +def _severity(v): + r = (v.get("ratings") or [{}])[0] + s = str(r.get("severity") if isinstance(r, dict) else "").lower() + return s if s in SEVERITIES else "other" + + +def _list(x): + return x if isinstance(x, list) else [] + + +def _float(x): + try: + return float(x) + except (TypeError, ValueError): + return 0.0 + + +def _licence(c): + """A CycloneDX component's licence as one SPDX expression ('' if none).""" + out = [] + for x in _list(c.get("licenses")): + if isinstance(x, dict) and x.get("expression"): + out.append(str(x["expression"])) + elif isinstance(x, dict) and isinstance(x.get("license"), dict): + v = x["license"].get("id") or x["license"].get("name") + if v: + out.append(str(v)) + return " AND ".join(out) + + +def _closure(deps, starts, skip=frozenset()): + seen, todo = set(), [t for t in starts if t not in skip] + while todo: + ref = todo.pop() + if ref not in seen: + seen.add(ref) + todo += [d for d in deps.get(ref, []) if d not in skip] + return seen + + +def _shipped(comps, deps, tops): + """(shipped, used by tools only) refs: the runtime closure of the release's + *tops* without host tools (bits:sbom = tool), and what only through them; + None without tops (older SBOMs).""" + if not tops: + return None + tool = {str(c["bom-ref"]) for c in comps if _props(c).get("bits:sbom") == "tool"} + shipped = _closure(deps, tops, tool) + return shipped, _closure(deps, tops) - shipped - tool + + +def _not_shipped(props, ref, shipped): + """Why a component is not shipped with the stack ('' if it is).""" + if props.get("bits:sbom") == "tool": + return "tool" + if shipped is None or ref in shipped[0]: + return "" + if ref in shipped[1]: + return "tool-only" + return "build-only" if props.get("bits:provided_by") == "system" else "" + + +def index(doc, vulns): + """(components, identified, sbom_packages rows, sbom_findings rows) of the + CycloneDX document *doc* and its scan's *vulns*, without the build id.""" + packages, identified = [], 0 + comps = [c for c in _list(doc.get("components")) + if isinstance(c, dict) and c.get("bom-ref")] + by_name = {} # bits name -> bom-ref, for build_requires + for c in comps: + props = _props(c) + by_name.setdefault(str(props.get("bits:package") or c.get("name") or ""), str(c["bom-ref"])) + deps = {} + for d in _list(doc.get("dependencies")): + if isinstance(d, dict) and isinstance(d.get("dependsOn"), list): + deps[str(d.get("ref"))] = [str(r) for r in d["dependsOn"]] + meta = doc.get("metadata") if isinstance(doc.get("metadata"), dict) else {} + root = meta.get("component") if isinstance(meta.get("component"), dict) else {} + tops = set(deps.get(str(root.get("bom-ref")), [])) if root.get("bom-ref") else set() + shipped = _shipped(comps, deps, tops) + for c in comps: + props, ref = _props(c), str(c["bom-ref"]) + purl = str(c.get("purl") or "") + cpe = str(c.get("cpe") or props.get("syft:cpe23") or "") + if cpe or purl.partition(":")[2].partition("/")[0] in MATCHED_PURLS: + identified += 1 + build = [by_name[str(p.get("value"))] for p in _list(c.get("properties")) + if isinstance(p, dict) and p.get("name") == "bits:build_requires" + and str(p.get("value")) in by_name] + on = sorted(set(deps.get(ref, []) + build) - {ref}) + version, rev = str(c.get("version") or ""), props.get("bits:revision") + if rev and version.endswith("-" + rev): # older SBOMs: version-revision + version = version[:-len(rev) - 1] + packages.append((ref, str(c.get("name") or ""), str(props.get("bits:package") or ""), + version, purl, cpe, + int(props.get("bits:provided_by") == "system"), + str(props.get("bits:pinned_by") or ""), json.dumps(on), + _licence(c), str(props.get("bits:redistributable") or ""), + json.dumps(sorted(set(deps.get(ref, [])) - {ref})), int(ref in tops), + _not_shipped(props, ref, shipped))) + refs = {p[0] for p in packages} + findings = {} + for v in _list(vulns): + if not isinstance(v, dict) or not v.get("id"): + continue + threat = _props(v) + for a in _list(v.get("affects")): + if not isinstance(a, dict) or not isinstance(a.get("ref"), str) or a["ref"] not in refs: + continue + fixed = sorted({str(x.get("version")) for x in _list(a.get("versions")) + if isinstance(x, dict) and x.get("status") == "unaffected" and x.get("version")}) + findings[(str(v["id"]), a["ref"])] = ( + _severity(v), _float(threat.get("grype:risk")), _float(threat.get("grype:epss")), + str(threat.get("grype:kev") or ""), json.dumps(fixed)) + return len(packages), identified, packages, sorted(k + f for k, f in findings.items()) + + +def due(conn, db_built, now, limit=None): + """Build ids to scan, newest first: never scanned (recorded SETTLE_MS ago: + a running pipeline's build is recorded before its SBOM is uploaded), last good + scan with an older database, older than the build's newest publication or not + indexed (INDEX_VERSION), or failed more than RETRY_MS ago.""" + rows = conn.execute( + "SELECT b.build_id, MAX(b.recorded_ms) AS seen, s.db_built, s.error, s.scanned_ms," + " s.index_version" + " FROM builds b LEFT JOIN sbom_scans s ON s.build_id = b.build_id" + " GROUP BY b.build_id ORDER BY seen DESC").fetchall() + out, limit = [], limit or PER_PASS + for r in rows: + if r["scanned_ms"] is None: + if now - (r["seen"] or 0) >= SETTLE_MS: + out.append(r["build_id"]) + elif r["error"]: + if now - r["scanned_ms"] > RETRY_MS: + out.append(r["build_id"]) + elif (r["db_built"] != db_built or r["seen"] > r["scanned_ms"] + or r["index_version"] < INDEX_VERSION): + out.append(r["build_id"]) + if len(out) >= limit: + break + return out + + +def _failed(conn, bid, error): + """Record a failed scan, keeping the last good one's counts.""" + with db.transaction(conn): + conn.execute( + "INSERT INTO sbom_scans (build_id, scanned_ms, error) VALUES (?, ?, ?)" + " ON CONFLICT(build_id) DO UPDATE SET scanned_ms = excluded.scanned_ms," + " error = excluded.error", (bid, db.now_ms(), error[:300])) + + +def _scanned(conn, directory, bid, sha, db_built, summary, vulns, doc): + s = summary or {} + n_components, n_identified, packages, findings = index(doc, vulns) + counts = [int(s.get(k) or 0) for k in SEVERITIES] + vsha = hashlib.sha256(json.dumps(vulns, sort_keys=True).encode()).hexdigest() + old = conn.execute("SELECT * FROM sbom_scans WHERE build_id = ?", (bid,)).fetchone() + if old is None or old["vulns_sha256"] != vsha or vulnerabilities(directory, bid) is None: + _write(directory, bid, vulns) + good = old is not None and old["db_built"] # a good scan before this one + before = [old[k] for k in SEVERITIES] if good else None + with db.transaction(conn): + conn.execute( + "INSERT INTO sbom_scans (build_id, sbom_sha256, db_built, scanned_ms, critical, high," + " medium, low, other, packages, vulnerabilities, vulns_sha256, error, components," + " identified, index_version) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,'',?,?,?)" + " ON CONFLICT(build_id) DO UPDATE SET sbom_sha256 = excluded.sbom_sha256," + " db_built = excluded.db_built, scanned_ms = excluded.scanned_ms," + " critical = excluded.critical, high = excluded.high, medium = excluded.medium," + " low = excluded.low, other = excluded.other, packages = excluded.packages," + " vulnerabilities = excluded.vulnerabilities, vulns_sha256 = excluded.vulns_sha256," + " error = '', components = excluded.components, identified = excluded.identified," + " index_version = excluded.index_version", + (bid, sha, db_built, db.now_ms(), *counts, int(s.get("packages") or 0), + int(s.get("vulnerabilities") or 0), vsha, n_components, n_identified, INDEX_VERSION)) + indexed = old is not None and old["index_version"] >= INDEX_VERSION + if not (indexed and old["sbom_sha256"] == sha): # the same SBOM: the same rows + conn.execute("DELETE FROM sbom_packages WHERE build_id = ?", (bid,)) + conn.executemany("INSERT OR IGNORE INTO sbom_packages (build_id, ref, name, alias," + " version, purl, cpe, system, pinned_by, depends_on, license," + " redistributable, runtime_on, top, not_shipped)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [(bid, *p) for p in packages]) + if not (indexed and old["sbom_sha256"] == sha and old["vulns_sha256"] == vsha): + conn.execute("DELETE FROM sbom_findings WHERE build_id = ?", (bid,)) + conn.executemany("INSERT INTO sbom_findings (build_id, vuln_id, ref, severity, risk," + " epss, kev, fixed_in) VALUES (?,?,?,?,?,?,?,?)", + [(bid, *f) for f in findings]) + if counts != (before or [0] * len(SEVERITIES)): # a first clean scan: no event + events.append(conn, "sbom.scanned", "build:%s" % bid, events.WORKER, + {"build_id": bid, "db_built": db_built, + **dict(zip(SEVERITIES, counts)), "was": before}) + + +def step(conn, store, url, directory, get=_get, post=_post, clock=time.monotonic): + """Scan the builds that are due; returns how many were scanned.""" + if not url: + return 0 + if store is None or not directory: + raise RuntimeError("SBOM scans need the store (BITS_ORCH_STORE) and BITS_ORCH_DB") + base = url.rstrip("/") + status = get(base + "/v1/status") + if status.get("updating"): + return 0 # back once its database is in + db_built = str((status.get("db") or {}).get("built") or "") + if not db_built: + raise RuntimeError("the sbom-scanner has no valid vulnerability database") + s3, bucket = store + start, n = clock(), 0 + for bid in due(conn, db_built, db.now_ms()): + if clock() - start > BUDGET_S: + break + try: + body = s3.get_object(Bucket=bucket, Key="MANIFESTS/%s/%s" % (bid, SBOM))["Body"].read() + except Exception as e: # pylint: disable=broad-except + if not _missing(e): + raise + _failed(conn, bid, "no %s in the store" % SBOM) + continue + try: + doc = json.loads(body) + if not isinstance(doc, dict): + raise ValueError("not an object") + except (ValueError, RecursionError) as e: + _failed(conn, bid, "%s is not JSON: %s" % (SBOM, e)) + continue + try: + res = post(base + "/v1/scan", body) + except Busy: + break # nothing recorded: next pass + except Exception as e: # pylint: disable=broad-except + log.warning("scan of %s: %s", bid, e) + _failed(conn, bid, str(e)) + continue + try: + _scanned(conn, directory, bid, hashlib.sha256(body).hexdigest(), + str((res.get("db") or {}).get("built") or db_built), res.get("summary"), + (res.get("sbom") or {}).get("vulnerabilities") or [], doc) + except (TypeError, ValueError, AttributeError, KeyError) as e: # an odd SBOM + log.warning("indexing the scan of %s: %r", bid, e) + _failed(conn, bid, "could not index the scan: %r" % e) + continue + n += 1 + return n + + +def _in(column, values): + return "%s IN (%s)" % (column, ",".join("?" * len(values))), sorted(values) + + +def _findings(conn, community, rules): + """Findings of the builds of *community* ({build id: its community}) as dicts, + each with the triage rule covering it (None if none), its package, version and + why it is not shipped ('' if it is).""" + if not community: + return [] + cond, vals = _in("f.build_id", community) + rows = conn.execute( + "SELECT f.build_id, f.vuln_id, f.ref, f.severity, f.risk, p.name, p.alias, p.version," + " p.not_shipped FROM sbom_findings f JOIN sbom_packages p ON p.build_id = f.build_id AND p.ref = f.ref" + " WHERE " + cond, vals).fetchall() + out = [] + for r in rows: + d = dict(r) + d["rule"] = rules.rule(community[d["build_id"]], d["vuln_id"], d["name"], d["version"], + d["alias"]) + out.append(d) + return out + + +def listing(conn, communities=None, flagged=False, limit=200, now=None): + """The scanned builds, most severe first (*flagged*: those with open findings + or not scannable): counts, when, against which database, the pipelines that + published them and their package; how many components a scanner could + identify, and the counts without the findings the triage covers (`effective`, + once indexed), by which they are ordered, nor those of components not shipped + (`not_shipped`: how many).""" + q = ("SELECT s.build_id, s.db_built, s.scanned_ms, s.critical, s.high, s.medium, s.low," + " s.other, s.packages, s.vulnerabilities, s.error, s.components, s.identified," + " s.index_version, MAX(b.pipeline_id) AS pipeline_id," + " b.community, p.package, p.version, p.platform, b.published_at" + " FROM sbom_scans s JOIN builds b ON b.build_id = s.build_id" + " LEFT JOIN pipelines p ON p.id = b.pipeline_id") + where, args = [], [] + if communities is not None: + if not communities: + return [] + cond, vals = _in("b.community", communities) + where.append(cond) + args += vals + if flagged: + where.append("((s.critical + s.high + s.medium + s.low + s.other) > 0" + " OR (s.components > 0 AND s.identified = 0))") + if where: + q += " WHERE " + " AND ".join(where) + q += (" GROUP BY s.build_id ORDER BY s.critical DESC, s.high DESC, s.medium DESC," + " s.low DESC, s.scanned_ms DESC LIMIT ?") + rows = [dict(r) for r in conn.execute(q, args + [int(limit)])] + indexed = {r["build_id"] for r in rows if r["index_version"] >= FINDINGS_INDEX} + rules = vuln_triage.Rules.load(conn, {r["community"] for r in rows}, now=now) + effective = {b: dict.fromkeys(SEVERITIES, 0) for b in indexed} + suppressed, not_shipped = dict.fromkeys(indexed, 0), dict.fromkeys(indexed, 0) + for f in _findings(conn, {r["build_id"]: r["community"] for r in rows + if r["build_id"] in indexed}, rules): + if f["not_shipped"]: + not_shipped[f["build_id"]] += 1 + elif f["rule"]: + suppressed[f["build_id"]] += 1 + else: + effective[f["build_id"]][f["severity"]] += 1 + for r in rows: + indexed_now = r.pop("index_version") >= FINDINGS_INDEX + r["effective"] = effective.get(r["build_id"]) if indexed_now else None + r["suppressed"] = suppressed.get(r["build_id"], 0) + r["not_shipped"] = not_shipped.get(r["build_id"], 0) + if flagged: + rows = [r for r in rows if any(_open(r)) or _blind(r)] + return sorted(rows, key=lambda r: [-n for n in _open(r)[:4]] + [-r["scanned_ms"]]) + + +def _blind(row): + """Is a listed scan of an SBOM none of whose components a scanner can identify?""" + return row["effective"] is not None and row["components"] > 0 and not row["identified"] + + +def _open(row): + """A listed scan's counts by severity, without what the triage covers.""" + return [(row["effective"] or row)[k] for k in SEVERITIES] + + +def _version_match(version): + """SQL for a version and every version it is a prefix of up to a separator + ("2.9" or "2.9.x": 2.9, 2.9.13, 2.9-1; not 2.90); None for no version.""" + v = re.sub(r"[.]?[x*]$", "", version.strip()) + if not v: + return None + esc = v.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return ("(p.version = ? OR p.version LIKE ? ESCAPE '\\' OR p.version LIKE ? ESCAPE '\\'" + " OR p.version LIKE ? ESCAPE '\\')", [v, esc + ".%", esc + "-%", esc + "+%"]) + + +def packages(conn, name, version="", communities=None, limit=500, now=None): + """The indexed builds that contain the package *name* (its name in the SBOM + or its bits name, any case), at *version* (see _version_match) if given, + newest first: the build, its release, the component and its findings.""" + if communities is not None and not communities: + return [] + where = ["(p.name = ? COLLATE NOCASE OR p.alias = ? COLLATE NOCASE)"] + args = [name.strip(), name.strip()] + match = _version_match(version) + if match: + where.append(match[0]) + args += match[1] + if communities is not None: + cond, vals = _in("b.community", communities) + where.append(cond) + args += vals + rows = [dict(r) for r in conn.execute( + "SELECT p.build_id, p.ref, p.name, p.alias, p.version AS component_version, p.purl, p.cpe," + " p.system, p.not_shipped, b.community, MAX(b.pipeline_id) AS pipeline_id, pl.package," + " pl.version, pl.platform, MAX(b.published_at) AS published_at, s.scanned_ms" + " FROM sbom_packages p JOIN builds b ON b.build_id = p.build_id" + " JOIN sbom_scans s ON s.build_id = p.build_id" + " LEFT JOIN pipelines pl ON pl.id = b.pipeline_id WHERE " + " AND ".join(where) + + " GROUP BY p.build_id, p.ref ORDER BY published_at DESC, p.build_id DESC LIMIT ?", + args + [int(limit)])] + rules = vuln_triage.Rules.load(conn, {r["community"] for r in rows}, now=now) + by = {} + for f in _findings(conn, {r["build_id"]: r["community"] for r in rows}, rules): + by.setdefault((f["build_id"], f["ref"]), []).append( + {"id": f["vuln_id"], "severity": f["severity"], + "suppressed": f["rule"]["id"] if f["rule"] else None}) + for r in rows: + r["system"] = bool(r["system"]) + r["findings"] = sorted(by.get((r["build_id"], r["ref"]), []), + key=lambda f: (SEVERITIES.index(f["severity"]), f["id"])) + return rows + + +def names(conn, communities=None): + """The package names (and bits names) in the indexed builds, for completion.""" + if communities is not None and not communities: + return [] + q = "SELECT DISTINCT p.name, p.alias FROM sbom_packages p" + args = [] + if communities is not None: + cond, args = _in("b.community", communities) + q += " JOIN builds b ON b.build_id = p.build_id WHERE " + cond + out = set() + for r in conn.execute(q, args): + out.update(n for n in (r["name"], r["alias"]) if n) + return sorted(out, key=str.lower) + + +def affected(conn, vuln_id, communities=None, limit=500, now=None): + """The indexed builds a vulnerability was found in, newest first: the build, + its release, the package and version, and the triage rule covering it.""" + if communities is not None and not communities: + return [] + where, args = ["f.vuln_id = ? COLLATE NOCASE"], [vuln_id.strip()] + if communities is not None: + cond, vals = _in("b.community", communities) + where.append(cond) + args += vals + rows = [dict(r) for r in conn.execute( + "SELECT f.build_id, f.vuln_id, f.ref, f.severity, p.name, p.alias," + " p.version AS component_version," + " p.system, p.not_shipped, b.community, MAX(b.pipeline_id) AS pipeline_id, pl.package," + " pl.version, pl.platform, MAX(b.published_at) AS published_at" + " FROM sbom_findings f JOIN sbom_packages p ON p.build_id = f.build_id AND p.ref = f.ref" + " JOIN builds b ON b.build_id = f.build_id LEFT JOIN pipelines pl ON pl.id = b.pipeline_id" + " WHERE " + " AND ".join(where) + + " GROUP BY f.build_id, f.ref ORDER BY published_at DESC, f.build_id DESC LIMIT ?", + args + [int(limit)])] + rules = vuln_triage.Rules.load(conn, {r["community"] for r in rows}, now=now) + for r in rows: + rule = rules.rule(r["community"], r["vuln_id"], r["name"], r["component_version"], + r["alias"]) + r["system"] = bool(r["system"]) + r["suppressed"] = rule["id"] if rule else None + return rows diff --git a/console-backend/console_backend/orch/schedules.py b/console-backend/console_backend/orch/schedules.py new file mode 100644 index 00000000..76b53f61 --- /dev/null +++ b/console-backend/console_backend/orch/schedules.py @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Nightly schedules (ADR bits 0007): what a community builds every night, on +which platforms, on which weekdays at what time of the bits instance's time zone. + +A schedule (a row of the nightly matrix) builds one package at one version, with +one defaults chain and recipe repositories, on several platforms, and each of +its variants (ADR bits 0009: defaults chains put before its own). Its content +hash covers what is built and where it goes, not when: a standing approval is +bound to it (approvals.standing_subject("schedule:", sha256)), so moving a +nightly to another time keeps its approval and adding a platform needs a new one. +Every change is an event with what it was before and after. +""" + +import datetime as dt +import hashlib +import json +import re +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from . import db, events + +# Matched whole (fullmatch), ASCII only: these become CI variables and bits arguments. +_RE = lambda p: re.compile(p, re.ASCII) +_NAME = _RE(r"[A-Za-z0-9][A-Za-z0-9 ._+()-]{0,99}") +_PKG = _RE(r"[A-Za-z0-9._+-]{1,100}") +_VERSION = _RE(r"[A-Za-z0-9._+-]{0,100}") +DEFAULTS_RE = _RE(r"[A-Za-z0-9._:+-]{0,200}") +_PLATFORM = _RE(r"[A-Za-z0-9._-]{1,64}") +_REPO = _RE(r"https://[A-Za-z0-9.-]+(:[0-9]+)?/[A-Za-z0-9._~/-]+(@[A-Za-z0-9._/][A-Za-z0-9._/-]*)?") +_REPOS = _RE(r"[^\s]{0,2000}") +_SET_KEY = _RE(r"[A-Za-z_][A-Za-z0-9_.-]{0,63}") +_SET_VALUE = _RE(r"[A-Za-z0-9._:/@+=-]{0,200}") +_ARGS = _RE(r"[A-Za-z0-9 \t=_.,/:@#+%^-]{0,500}") # lib-pipeline-dispatch.sh's, spaces as such +_GROUP = _RE(r"[a-z0-9._-]{1,64}") +_DIR = _RE(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +_TIME = _RE(r"([01][0-9]|2[0-3]):[0-5][0-9]") +MAX_PLATFORMS = 32 +MAX_VARIANTS = 16 +MAX_CELLS = 64 # platforms x variants: the builds of one night +MAX_CHAIN = 160 # a variant::defaults, leaving room for a community's overlay +MAX_WATCHES = 8 +EVERY_DAY = 0b1111111 # bit 0 Monday ... bit 6 Sunday + + +class Invalid(ValueError): + """A schedule that cannot be stored: why, for the person editing it.""" + + +class Conflict(Exception): + """The schedule changed since it was read (another revision).""" + + +def time_zone(): + """The bits instance's time zone (site configuration console.time_zone).""" + from bits_helpers import site + name = str(site.get("console.time_zone") or "UTC") + try: + return ZoneInfo(name) + except (ZoneInfoNotFoundError, ValueError): + raise Invalid("console.time_zone %r is not a time zone" % name) + + +# --- what a schedule is ------------------------------------------------------- + +def _flag(p, key, default=False): + v = p.get(key, default) + if not isinstance(v, bool): + raise Invalid("%s must be true or false" % key) + return v + + +def _text(p, key, pattern, what): + v = p.get(key, "") + v = "" if v is None else v + if not isinstance(v, str) or not pattern.fullmatch(v.strip()): + raise Invalid("%s: %s" % (key, what)) + return v.strip() + + +def _port_ok(url): + m = re.match(r"https://[^/:]+:([0-9]+)/", url) + return not m or 1 <= int(m.group(1)) <= 65535 + + +def check(p, community): + """The stored fields of a schedule from the request *p*, or Invalid.""" + if not isinstance(p, dict): + raise Invalid("expected a JSON object") + # community_dir: the community's directory in the console (communities//). + s = {"community": community.strip().lower(), "community_dir": community.strip()} + if not _DIR.fullmatch(s["community_dir"]): + raise Invalid("community: not a community name") + s["name"] = _text(p, "name", _NAME, "1-100 letters, digits, spaces and ._+()-") + s["package"] = _text(p, "package", _PKG, "a package name") + s["version"] = _text(p, "version", _VERSION, "a version, or empty for the recipe's") + s["defaults"] = _text(p, "defaults", DEFAULTS_RE, "a defaults chain (a::b), or empty") + repos = [r for r in _text(p, "extra_repos", _REPOS, "url@branch;...").split(";") if r] + for r in repos: + if not _REPO.fullmatch(r): + raise Invalid("extra_repos: %r is not https://host/path[@branch]" % r) + s["extra_repos"] = ";".join(repos) + watch = [w for w in _text(p, "watch", _REPOS, "url@branch;...").split(";") if w] + for w in watch: + if not _REPO.fullmatch(w) or "@" not in w.split("/", 3)[-1] or not _port_ok(w): + raise Invalid("watch: %r is not https://host/path@branch" % w) + if len(watch) > MAX_WATCHES or len(set(watch)) != len(watch): + raise Invalid("watch: up to %d different branches" % MAX_WATCHES) + s["watch"] = ";".join(sorted(watch)) + sv = p.get("set") or {} + if not isinstance(sv, dict) or len(sv) > 32 or not all( + isinstance(k, str) and _SET_KEY.fullmatch(k) and isinstance(v, str) and _SET_VALUE.fullmatch(v) + for k, v in sv.items()): + raise Invalid("set: up to 32 variables, name=value without spaces") + s["set"] = dict(sorted(sv.items())) + plats = p.get("platforms") + if not isinstance(plats, list) or not 0 < len(plats) <= MAX_PLATFORMS or not all( + isinstance(x, str) and _PLATFORM.fullmatch(x) for x in plats) or len(set(plats)) != len(plats): + raise Invalid("platforms: 1 to %d different platform names" % MAX_PLATFORMS) + s["platforms"] = sorted(plats) + var = p.get("variants") or [] + if not isinstance(var, list) or not all(isinstance(v, str) and DEFAULTS_RE.fullmatch(v) for v in var): + raise Invalid("variants: defaults chains (a::b)") + var = ["::".join(x for x in v.split("::") if x) for v in var] # "a::" is "a" + if len(var) > MAX_VARIANTS or not all(var) or len(set(var)) != len(var): + raise Invalid("variants: up to %d different defaults chains (a::b)" % MAX_VARIANTS) + if any(len(v) + 2 + len(s["defaults"]) > MAX_CHAIN for v in var): + raise Invalid("variants: a variant and the defaults together over %d characters" % MAX_CHAIN) + s["variants"] = sorted(var) + if len(plats) * max(1, len(var)) > MAX_CELLS: + raise Invalid("platforms x variants: at most %d builds a night" % MAX_CELLS) + s["rebuild"], s["debug"] = _flag(p, "rebuild"), _flag(p, "debug") + jobs = p.get("jobs", 0) + if isinstance(jobs, bool) or not isinstance(jobs, int) or not 0 <= jobs <= 256: + raise Invalid("jobs: 0 (the build host's) to 256") + s["jobs"] = jobs + s["extra_build_args"] = _text(p, "extra_build_args", _ARGS, "bits build arguments without quotes") + if re.search(r"(^|\s)--defaults(=|\s|$)", s["extra_build_args"]): + raise Invalid("extra_build_args: give the defaults in defaults (the build job resolves its CVMFS target with them)") + days = p.get("days", EVERY_DAY) + if isinstance(days, bool) or not isinstance(days, int) or not 0 < days <= EVERY_DAY: + raise Invalid("days: a weekday mask, 1 (Monday) to 127 (every day)") + s["days"] = days + s["time"] = _text(p, "time", _TIME, "HH:MM") + s["certify"], s["cvmfs"] = _flag(p, "certify"), _flag(p, "cvmfs") + groups = p.get("certify_groups") or [s["community"]] + if not isinstance(groups, list) or not 0 < len(groups) <= 8 or not all( + isinstance(g, str) and _GROUP.fullmatch(g) for g in groups): + raise Invalid("certify_groups: 1 to 8 group names") + s["certify_groups"] = sorted(set(groups)) + s["enabled"] = _flag(p, "enabled", True) + return s + + +_CONTENT = ("community", "package", "version", "defaults", "extra_repos", "set", "platforms", + "rebuild", "debug", "jobs", "extra_build_args", "certify", "cvmfs", "certify_groups") + + +def content_sha256(s): + """What is built and where it goes, not when (days, time, the watch list) or + whether (enabled). Variants count only when set: a schedule without keeps its hash.""" + content = {k: s[k] for k in _CONTENT} + if s.get("variants"): # only when set: older schedules keep their hash + content["variants"] = s["variants"] + text = json.dumps(content, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode()).hexdigest() + + +def cells(s): + """[(platform, variant)] of a night: every platform x every variant ('' when none).""" + return [(p, v) for p in s["platforms"] for v in (s.get("variants") or [""])] + + +def watches(s): + """[(repo, branch)] the schedule follows: its watch list, url@branch.""" + return [tuple(w.rsplit("@", 1)) for w in s.get("watch", "").split(";") if w] + + +def cell_key(platform, variant): + """A cell's name in the matrix: its platform, and its variant if it has one.""" + return "%s %s" % (platform, variant) if variant else platform + + +# --- when it runs --------------------------------------------------------------- + +def _local(day, hhmm, tz): + """The moment of *hhmm* on local *day*: a time skipped in spring is the first + valid minute after it, one repeated in autumn its first occurrence.""" + h, m = (int(x) for x in hhmm.split(":")) + naive = dt.datetime(day.year, day.month, day.day, h, m) + for step in range(0, 180): + t = naive + dt.timedelta(minutes=step) + aware = t.replace(tzinfo=tz, fold=0) + if aware.astimezone(dt.timezone.utc).astimezone(tz).replace(tzinfo=None) == t: + return aware + return naive.replace(tzinfo=tz) + + +def occurrences(s, after_ms, before_ms, tz): + """[(night, ms)] of the schedule's times in (after_ms, before_ms], oldest + first: night is the local date (ISO) of its scheduled time.""" + start = dt.datetime.fromtimestamp(after_ms / 1000, tz).date() - dt.timedelta(days=1) + end = dt.datetime.fromtimestamp(before_ms / 1000, tz).date() + dt.timedelta(days=1) + out, day = [], start + while day <= end: + if s["days"] >> day.weekday() & 1: + ms = int(_local(day, s["time"], tz).timestamp() * 1000) + if after_ms < ms <= before_ms: + out.append((day.isoformat(), ms)) + day += dt.timedelta(days=1) + return out + + +def next_run_ms(s, now_ms, tz): + """The schedule's next time after *now_ms*, or None when it never runs.""" + found = occurrences(s, now_ms, now_ms + 8 * 86400 * 1000, tz) + return found[0][1] if found else None + + +# --- the rows ------------------------------------------------------------------- + +def _row(r): + d = dict(r) + for k in ("set", "platforms", "certify_groups", "variants"): + d[k] = json.loads(d.pop("set_vars") if k == "set" else d[k]) + for k in ("rebuild", "debug", "certify", "cvmfs", "enabled"): + d[k] = bool(d[k]) + return d + + +def get(conn, sid): + r = conn.execute("SELECT * FROM schedules WHERE id = ? AND deleted_ms = 0", (int(sid),)).fetchone() + return _row(r) if r else None + + +def listing(conn, communities=None): + """The schedules of *communities* (lowercase; None = all), by community and name.""" + q, args = "SELECT * FROM schedules WHERE deleted_ms = 0", [] + if communities is not None: + names = sorted(communities) + if not names: + return [] + q += " AND community IN (%s)" % ",".join("?" * len(names)) + args.extend(names) + return [_row(r) for r in conn.execute(q + " ORDER BY community, name, id", args)] + + +def _public(s): + return {k: v for k, v in s.items() if k not in ("deleted_ms",)} + + +def _store(conn, s, sid=None): + cols = ("community", "community_dir", "name", "package", "version", "defaults", "extra_repos", "set_vars", + "platforms", "rebuild", "debug", "jobs", "extra_build_args", "days", "time", "certify", + "cvmfs", "certify_groups", "enabled", "variants", "watch", "content_sha256") + vals = (s["community"], s["community_dir"], s["name"], s["package"], s["version"], s["defaults"], s["extra_repos"], + json.dumps(s["set"]), json.dumps(s["platforms"]), int(s["rebuild"]), int(s["debug"]), + s["jobs"], s["extra_build_args"], s["days"], s["time"], int(s["certify"]), + int(s["cvmfs"]), json.dumps(s["certify_groups"]), int(s["enabled"]), + json.dumps(s["variants"]), s["watch"], content_sha256(s)) + return cols, vals + + +def _taken(conn, s, sid=None): + r = conn.execute("SELECT id FROM schedules WHERE community = ? AND name = ? AND deleted_ms = 0" + " AND id != ?", (s["community"], s["name"], int(sid or 0))).fetchone() + if r: + raise Invalid("name: the community has a schedule %r already" % s["name"]) + + +def create(conn, s, user): + """Store a checked schedule (check); returns it.""" + now = db.now_ms() + with db.transaction(conn): + _taken(conn, s) + cols, vals = _store(conn, s) + cur = conn.execute("INSERT INTO schedules (%s, revision, last_fired_ms, created_by," + " created_ms, updated_by, updated_ms) VALUES (%s, 1, ?, ?, ?, ?, ?)" + % (", ".join(cols), ", ".join("?" * len(cols))), + vals + (now, user, now, user, now)) + new = get(conn, cur.lastrowid) + events.append(conn, "schedule.created", "schedule:%d" % new["id"], + events.Actor("person", user), {"after": _public(new)}) + return new + + +def update(conn, sid, s, user, revision): + """Replace a schedule's fields; Conflict when it is no longer at *revision*. + Pause and resume alone change whether it runs, and a new time fires nothing + that went by before the change.""" + if isinstance(revision, bool) or not isinstance(revision, int): + raise Invalid("revision: the revision the change is made to") + now = db.now_ms() + with db.transaction(conn): + old = get(conn, sid) + if old is None: + return None + if s["community"] != old["community"]: + raise Invalid("community: a schedule stays in its community") + if revision != old["revision"]: + raise Conflict("changed since: revision %d, not %d" % (old["revision"], revision)) + _taken(conn, s, sid) + cols, vals = _store(conn, dict(s, enabled=old["enabled"], community_dir=old["community_dir"])) + conn.execute("UPDATE schedules SET %s, revision = revision + 1, updated_by = ?," + " updated_ms = ?, last_fired_ms = MAX(last_fired_ms, ?) WHERE id = ?" + % ", ".join("%s = ?" % c for c in cols), vals + (user, now, now, int(sid))) + new = get(conn, sid) + events.append(conn, "schedule.updated", "schedule:%d" % new["id"], + events.Actor("person", user), {"before": _public(old), "after": _public(new)}) + return new + + +def set_enabled(conn, sid, enabled, user): + """Pause or resume; the schedule, or None when there is none (unchanged when + it already is so: no event).""" + now = db.now_ms() + with db.transaction(conn): + old = get(conn, sid) + if old is None: + return None + if old["enabled"] != bool(enabled): + # Resumed: from now on, no night missed while it was paused. + conn.execute("UPDATE schedules SET enabled = ?, revision = revision + 1, updated_by = ?," + " updated_ms = ?, last_fired_ms = MAX(last_fired_ms, ?) WHERE id = ?", + (int(bool(enabled)), user, now, now if enabled else 0, int(sid))) + events.append(conn, "schedule.resumed" if enabled else "schedule.paused", + "schedule:%d" % old["id"], events.Actor("person", user), + {"before": {"enabled": old["enabled"]}, "after": {"enabled": bool(enabled)}}) + return get(conn, sid) + + +def delete(conn, sid, user): + """Gone from the matrix and from firing; its runs stay. False when there is none.""" + now = db.now_ms() + with db.transaction(conn): + old = get(conn, sid) + if old is None: + return False + conn.execute("UPDATE schedules SET deleted_ms = ?, enabled = 0, updated_by = ?, updated_ms = ?" + " WHERE id = ?", (now, user, now, int(sid))) + events.append(conn, "schedule.deleted", "schedule:%d" % old["id"], + events.Actor("person", user), {"before": _public(old)}) + return True diff --git a/console-backend/console_backend/orch/shadow.py b/console-backend/console_backend/orch/shadow.py new file mode 100644 index 00000000..482f3536 --- /dev/null +++ b/console-backend/console_backend/orch/shadow.py @@ -0,0 +1,714 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Certification in shadow (M4): recompute what the bits-manifests CI signed. + +For every pipeline of the bits-manifests main branch whose certify job signed, +the worker works out what that job certified and computes the same manifests +itself, with bits' own merge and store check (`certify.build_by_arch`), from the +same inputs: + + * the manifests/ tree at the pipeline's commit (the GitLab archive API); + * the architectures the push changed, from the tree at the pipeline's + before_sha, worked out exactly as the CI job's script does (a manifest's + `sources` depends on which architectures were certified together); + * the store, through the same probe, with the sha256 of each object kept in + store_digests: objects never change, so only new ones are hashed. + +It compares the result with the manifests the job signed (its artifacts, or the +store's copy while it still carries the pipeline's commit), once their +signature verifies against the trust anchor. Timestamps and certified_by are +left out of the comparison; everything else must be equal. Each architecture is +a sign run (mode 'shadow') and a shadow.match or shadow.mismatch event. Nothing +is signed, committed or written to the store. + +Where git and the archive differ, the two can disagree, and the comparison +shows it: git lists only the new path of a renamed BOM; a change of file mode +alone, symlinks and .gitattributes export rules are not seen in the archive. +None of these occur in bits-manifests today. An outage of GitLab or S3 fails the +step (tried again next pass) without counting against a certification. + +This runs on its own thread with its own connection (`Runner`): the first +certification of an architecture hashes every object in it, which takes as long +as the CI job does, and must not hold up the worker's actions. +""" + +import hashlib +import io +import json +import logging +import os +import re +import shutil +import tarfile +import threading +import time +import zipfile + +from . import db, events + +log = logging.getLogger("bits_orchestrator.shadow") +_SHA = re.compile(r"[0-9a-f]{7,64}") + +VOLATILE = ("created_at", "expires", "certified_at", "certified_by") +MAX_ATTEMPTS = 8 +KEEP_TREES = 4 +MAX_TREE = 512 << 20 # the extracted manifests/ tree +MAX_SIGNED = 64 << 20 # the signed manifests in a job's artifacts +_LIST = 20 + + +class Skip(Exception): + """Nothing to compare for this certification (with the reason).""" + + +def transient(e): + """An error of GitLab, S3 or the network rather than of this certification: + the step fails and is tried again, without counting against it.""" + from .. import forge_ops + if isinstance(e, forge_ops.ForgeError): + return e.status == 0 or e.status == 429 or e.status >= 500 + resp = getattr(e, "response", None) + if isinstance(resp, dict): + status = (resp.get("ResponseMetadata") or {}).get("HTTPStatusCode") or 0 + code = str((resp.get("Error") or {}).get("Code", "")) + return status >= 500 or status == 429 or code in ("SlowDown", "Throttling", + "RequestTimeout", "InternalError", + "ServiceUnavailable") + mod = type(e).__module__ or "" + return isinstance(e, (OSError, TimeoutError)) or mod.startswith(("botocore", "urllib3", + "requests")) + + +# --- inputs ------------------------------------------------------------------- + +def _safe_member(m): + name = m.name + return (m.isfile() and not name.startswith("/") and ".." not in name.split("/") + and name.endswith(".json") and "/manifests/" in "/" + name) + + +def tree(reader, sha, cache_dir): + """The manifests/ directory of the bits-manifests tree at *sha*, extracted + under *cache_dir* (kept for the next certification, whose before_sha it + usually is). Only regular .json files under manifests/ are extracted.""" + if not _SHA.fullmatch(sha or ""): + raise Skip("not a commit sha: %r" % (sha,)) + dest = os.path.join(cache_dir, sha) + done = os.path.join(dest, ".complete") + if os.path.exists(done): + os.utime(dest) # recently used: pruned last + return os.path.join(dest, "manifests") + data = reader.repository_archive(sha, path="manifests") + if data is None: + raise Skip("commit %s is not in bits-manifests" % sha[:12]) + shutil.rmtree(dest, ignore_errors=True) + root = os.path.join(dest, "manifests") + os.makedirs(root) + total = 0 + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: + for m in tar: + if not _safe_member(m): + continue + total += m.size + if total > MAX_TREE: + raise RuntimeError("the manifests tree at %s is over %d bytes" % (sha[:12], MAX_TREE)) + # /manifests//.json -> manifests//... + rel = m.name.split("/manifests/", 1)[1] if "/manifests/" in m.name else m.name[len("manifests/"):] + out = os.path.join(root, rel) + if not os.path.realpath(out).startswith(os.path.realpath(root) + os.sep): + continue + os.makedirs(os.path.dirname(out), exist_ok=True) + with tar.extractfile(m) as src, open(out, "wb") as fh: + shutil.copyfileobj(src, fh) + open(done, "w").close() + return root + + +def prune(cache_dir, keep=KEEP_TREES, protect=()): + """Keep the *keep* most recently used trees, and those in *protect*.""" + try: + dirs = [os.path.join(cache_dir, d) for d in os.listdir(cache_dir)] + except OSError: + return + dirs = sorted((d for d in dirs if os.path.isdir(d)), key=os.path.getmtime, reverse=True) + for d in dirs[keep:]: + if os.path.basename(d) not in protect: + shutil.rmtree(d, ignore_errors=True) + + +def _files(root): + out = {} + for dirpath, _dirs, names in os.walk(root): + for n in names: + if n.endswith(".json"): + p = os.path.join(dirpath, n) + out[os.path.relpath(p, root).replace(os.sep, "/")] = p + return out + + +def _read(path): + with open(path, "rb") as fh: + return fh.read() + + +def _bom_arch(text): + """The CI job's reading of one BOM's architecture: None when it cannot tell.""" + try: + m = json.loads(text) + except ValueError: + return None + if not isinstance(m, dict): + return None + s = {e.get("effective_architecture") or "shared" + for e in m.get("packages") or [] + if isinstance(e, dict) and e.get("hash") and e.get("tarball_sha256")} + return m.get("effective_architecture") or (s.pop() if len(s) == 1 else None) + + +def changed_archs(new_root, old_root): + """(changed, archs) as the CI job computes them for `git diff before HEAD -- + manifests/`: the .json files that differ, and the architectures to certify, + or None for every architecture (a file whose architecture it cannot tell).""" + new, old = _files(new_root), _files(old_root) if old_root else {} + changed = sorted(p for p in set(new) | set(old) + if p not in new or p not in old or _read(new[p]) != _read(old[p])) + archs, ok = set(), True + for p in changed: + got = _bom_arch(_read(new[p]).decode("utf-8", "replace")) if p in new else ( + _bom_arch(_read(old[p]).decode("utf-8", "replace"))) + if got: + archs.add(got) + else: + ok = False + return changed, (sorted(archs) if ok and archs else None) + + +def signed_outputs(reader, store, prefix, job_id, sha, archs): + """{arch: (manifest bytes, signature envelope, source)} the certify job + signed: from its artifacts when it kept them, else the store's copy of each + architecture in *archs* (None: every one in the store), marked 'superseded' + once a later certification replaced it.""" + out = {} + data = reader.job_artifacts(job_id) if job_id else None + if data: + with zipfile.ZipFile(io.BytesIO(data)) as z: + names = set(z.namelist()) + wanted = [i for i in z.infolist() if i.filename.rsplit("/", 1)[-1].startswith( + "common-manifest-")] + if sum(i.file_size for i in wanted) > MAX_SIGNED: + raise RuntimeError("job %s: signed manifests over %d bytes" % (job_id, MAX_SIGNED)) + for n in sorted(names): + base = n.rsplit("/", 1)[-1] + if base.startswith("common-manifest-") and base.endswith(".json"): + arch = base[len("common-manifest-"):-len(".json")] + sig = z.read(n + ".sig") if n + ".sig" in names else b"" + out[arch] = (z.read(n), _envelope(sig), "artifacts") + if out: + return out + if store is None: + return out + s3, bucket = store + if archs is None: + archs = _store_archs(s3, bucket, prefix) + for arch in archs: + key = "%s-%s.json" % (prefix, arch) + try: + body = s3.get_object(Bucket=bucket, Key=key)["Body"].read() + sig = s3.get_object(Bucket=bucket, Key=key + ".sig")["Body"].read() + except Exception as e: + if _missing(e): + continue + raise + try: + commit = json.loads(body).get("source_commit") + except (ValueError, AttributeError): + commit = None + out[arch] = (body, _envelope(sig), "store" if commit == sha else "superseded") + return out + + +def _store_archs(s3, bucket, prefix): + archs, token = [], None + while True: + kw = {"Bucket": bucket, "Prefix": prefix + "-"} + if token: + kw["ContinuationToken"] = token + page = s3.list_objects_v2(**kw) + for o in page.get("Contents", []): + k = o["Key"] + if k.endswith(".json"): + archs.append(k[len(prefix) + 1:-len(".json")]) + token = page.get("NextContinuationToken") + if not page.get("IsTruncated") or not token: + return sorted(archs) + + +def _missing(e): + code = str(getattr(e, "response", {}).get("Error", {}).get("Code", "")) + return code in ("NoSuchKey", "404", "NotFound") + + +def _envelope(raw): + try: + env = json.loads(raw or b"null") + except ValueError: + return None + return env if isinstance(env, dict) else None + + +# --- comparing ---------------------------------------------------------------- + +def _canonical(obj): + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def content(manifest): + return {k: v for k, v in manifest.items() if k not in VOLATILE} + + +def digest(manifest): + return hashlib.sha256(_canonical(content(manifest)).encode()).hexdigest() + + +def _label(p): + return "%s@%s-%s %s" % (p.get("package", "?"), p.get("version", "?"), + p.get("revision", "?"), str(p.get("hash", ""))[:12]) + + +def diff(ours, signed): + """What differs between our manifest and the signed one, capped.""" + key = lambda p: (p.get("effective_architecture") or "", p.get("hash")) # noqa: E731 + a = {key(p): p for p in ours.get("packages") or [] if isinstance(p, dict)} + b = {key(p): p for p in signed.get("packages") or [] if isinstance(p, dict)} + only_ours = sorted(set(a) - set(b), key=str) + only_signed = sorted(set(b) - set(a), key=str) + changed = sorted((k for k in set(a) & set(b) if _canonical(a[k]) != _canonical(b[k])), key=str) + out = {} + if only_ours: + out["only_ours"] = {"count": len(only_ours), "first": [_label(a[k]) for k in only_ours[:_LIST]]} + if only_signed: + out["only_signed"] = {"count": len(only_signed), + "first": [_label(b[k]) for k in only_signed[:_LIST]]} + if changed: + fields = sorted({f for k in changed for f in set(a[k]) | set(b[k]) + if a[k].get(f) != b[k].get(f)}) + out["changed"] = {"count": len(changed), "fields": fields, + "first": [_label(a[k]) for k in changed[:_LIST]]} + for f in sorted((set(ours) | set(signed)) - set(VOLATILE) - {"packages"}): + x, y = ours.get(f), signed.get(f) + if x == y: + continue + if isinstance(x, list) and isinstance(y, list): + out[f] = {"only_ours": sorted(map(str, set(map(str, x)) - set(map(str, y))))[:_LIST], + "only_signed": sorted(map(str, set(map(str, y)) - set(map(str, x))))[:_LIST], + "order": sorted(map(str, x)) == sorted(map(str, y))} + else: + out[f] = {"ours": x, "signed": y} + return out + + +# --- the digest cache ----------------------------------------------------------- + +class DigestCache: + """store_digests as the probe's cache (see certify.s3_probe). It trusts an + object's key, ETag and size to stand for its bytes: fine for a comparison, + while CI still hashes every object; signing (M8) must hash for itself.""" + + def __init__(self, conn): + self.conn = conn + + def get(self, key, etag, size): + r = self.conn.execute("SELECT etag, size, sha256 FROM store_digests WHERE key = ?", + (key,)).fetchone() + if r and r["etag"] == etag and r["size"] == int(size): + return r["sha256"] + return None + + def put(self, key, etag, size, sha256): + with db.transaction(self.conn): + self.conn.execute( + "INSERT INTO store_digests (key, etag, size, sha256, checked_ms) VALUES (?,?,?,?,?)" + " ON CONFLICT(key) DO UPDATE SET etag = excluded.etag, size = excluded.size," + " sha256 = excluded.sha256, checked_ms = excluded.checked_ms", + (key, etag, int(size), sha256, db.now_ms())) + + +# --- the steps ------------------------------------------------------------------ + +def _subject(pid): + return "certification:%d" % int(pid) + + +def discover(conn, reader, settings, now_ms=None): + """Record the bits-manifests main pipelines that finished, and look again at + one not yet compared whose pipeline changed (a retried certify job).""" + now = now_ms or db.now_ms() + since = now - int(settings.orch_shadow_lookback_days * 86400 * 1000) + first = conn.execute("SELECT value FROM kv WHERE key = 'shadow.since_ms'").fetchone() + if first is None: + with db.transaction(conn): + conn.execute("INSERT INTO kv (key, value) VALUES ('shadow.since_ms', ?)", (str(since),)) + floor = since + else: + floor = int(first["value"]) + n = 0 + for p in reader.finished_pipelines(settings.orch_manifests_branch): + if not isinstance(p, dict) or p.get("source") not in ("push", "web", "schedule"): + continue + created = _ms(p.get("created_at")) + if created is not None and created < floor: + continue + pid, updated = int(p["id"]), str(p.get("updated_at") or "") + row = conn.execute("SELECT state, updated_at FROM certifications WHERE pipeline_id = ?", + (pid,)).fetchone() + if row is None: + with db.transaction(conn): + conn.execute("INSERT OR IGNORE INTO certifications (pipeline_id, sha, updated_at," + " created_ms, updated_ms) VALUES (?,?,?,?,?)", + (pid, str(p.get("sha") or ""), updated, now, now)) + n += 1 + elif row["updated_at"] != updated and row["state"] in ("skipped", "failed"): + # A retried certify job: look again. + with db.transaction(conn): + conn.execute("UPDATE certifications SET state = 'pending', reason = ''," + " attempts = 0, next_ms = 0, updated_at = ?, updated_ms = ?" + " WHERE pipeline_id = ?", (updated, now, pid)) + n += 1 + return n + + +def _ms(iso): + if not iso: + return None + import datetime + try: + return int(datetime.datetime.fromisoformat(str(iso).replace("Z", "+00:00")).timestamp() * 1000) + except ValueError: + return None + + +def _set(conn, pid, **fields): + fields["updated_ms"] = db.now_ms() + cols = ", ".join("%s = ?" % k for k in fields) + conn.execute("UPDATE certifications SET %s WHERE pipeline_id = ?" % cols, + list(fields.values()) + [pid]) + + +def compare_one(conn, reader, store, settings, cache_dir, build_by_arch=None, + trusted=None, probe_factory=None, upto=None, console=None): + """Compare the oldest due certification (with a pipeline id up to *upto*, + the newest the certified set has followed, when given). Returns its + pipeline id, or None when none is due.""" + now = db.now_ms() + row = conn.execute("SELECT * FROM certifications WHERE state = 'pending' AND next_ms <= ?" + " AND pipeline_id <= ? ORDER BY pipeline_id LIMIT 1", + (now, upto if upto is not None else 1 << 62)).fetchone() + if row is None: + return None + pid = row["pipeline_id"] + try: + _compare(conn, row, reader, store, settings, cache_dir, build_by_arch, trusted, + probe_factory, console) + except Skip as s: + with db.transaction(conn): + _set(conn, pid, state="skipped", reason=str(s)[:500]) + except Exception as e: + if transient(e) or _transient_cause(e): + raise # the step fails; this one is tried again as it is + attempts = row["attempts"] + 1 + reason = "%s: %s" % (type(e).__name__, str(e)[:400]) + with db.transaction(conn): + if attempts >= MAX_ATTEMPTS: + _set(conn, pid, state="failed", attempts=attempts, reason=reason) + events.append(conn, "shadow.failed", _subject(pid), events.WORKER, + {"error": reason, "attempts": attempts}) + else: + _set(conn, pid, attempts=attempts, reason=reason, + next_ms=now + min(3600, 60 * 2 ** attempts) * 1000) + log.warning("certification %s: %s (attempt %d)", pid, reason, attempts) + return pid + + +def _transient_cause(e): + """certify wraps store errors in CertifyError: look at the cause.""" + c = e.__cause__ or e.__context__ + return c is not None and transient(c) + + +def _certify_job(jobs, name): + mine = [j for j in jobs if isinstance(j, dict) and j.get("name") == name] + return max(mine, key=lambda j: int(j.get("id") or 0)) if mine else None + + +def _compare(conn, row, reader, store, settings, cache_dir, build_by_arch, trusted, + probe_factory, console=None): + pid = row["pipeline_id"] + pipeline = reader.get_pipeline(pid) + job = _certify_job(reader.pipeline_jobs(pid), settings.orch_certify_job) + status = (job or {}).get("status") or "" + with db.transaction(conn): + _set(conn, pid, job_id=(job or {}).get("id"), job_status=status, + updated_at=str(pipeline.get("updated_at") or row["updated_at"])) + # Whether that version of the certified set was signed (M8 needs to know). + conn.execute("UPDATE set_versions SET job_status = ? WHERE pipeline_id = ?", + (status or "none", pid)) + if job is None: + raise Skip("no %s job" % settings.orch_certify_job) + if status != "success": + raise Skip("%s %s" % (settings.orch_certify_job, status)) + sha = str(pipeline.get("sha") or row["sha"]) + before = str(pipeline.get("before_sha") or "") + if not before.strip("0"): + parents = (reader.commit(sha) or {}).get("parent_ids") or [] + before = parents[0] if parents else "" + try: + new_root = tree(reader, sha, cache_dir) + old_root = tree(reader, before, cache_dir) if before else None + _compare_trees(conn, pid, sha, before, new_root, old_root, job, reader, store, + settings, build_by_arch, trusted, probe_factory, console) + finally: + prune(cache_dir, protect=(sha, before)) + + +def _compare_trees(conn, pid, sha, before, new_root, old_root, job, reader, store, settings, + build_by_arch, trusted, probe_factory, console=None): + from bits_helpers import certify, trust + build_by_arch = build_by_arch or certify.build_by_arch + changed, archs = changed_archs(new_root, old_root) + with db.transaction(conn): + _set(conn, pid, sha=sha, before_sha=before, archs=json.dumps(archs or [])) + if not changed: + raise Skip("no manifests changed") + signed = signed_outputs(reader, store, settings.orch_manifest_prefix, job["id"], sha, archs) + if not signed: + raise Skip("the signed manifests are gone (no artifacts, and the store has newer ones)") + trusted = trust.load_trusted_keys() if trusted is None else trusted + parsed, key_id = {}, None + for arch, (body, env, source) in signed.items(): + kid = trust.verify_bytes(body, env, trusted) if env else None + try: + doc = json.loads(body) + except ValueError: + doc = None + parsed[arch] = (doc if isinstance(doc, dict) else None, kid, source) + key_id = key_id or kid + if probe_factory is None: + s3, bucket = store + base = certify.s3_probe(s3, bucket, settings.orch_default_arch, cache=DigestCache(conn)) + else: + base = probe_factory(conn) + memo = {} + + def probe(arch, h, tarball=None): + # The tree's, the set's and the replay's computations ask about the + # same objects. + k = (arch, h, tarball) + if k not in memo: + memo[k] = base(arch, h, tarball) + return memo[k] + + def compute(root, only=archs): + try: + return dict(build_by_arch(root, probe=probe, source_commit=sha, + only_archs=only, + key_id=(lambda: key_id) if key_id else None)), None + except certify.CertifyError as e: + if _transient_cause(e): + raise # the store failed to answer, not this certification + return None, ("*", "mismatch", "", 0, "", "", + {"error": "the worker refuses what was signed: %s" % str(e)[:1000]}) + + # The worker's own certification of the build (M8a). + from . import replay + replay_runs = replay.runs(conn, settings, console, store, sha, new_root, old_root, job, + parsed, key_id, compute) + ours, refused = compute(new_root) + runs = [refused] if refused else judge(ours, parsed) + # The same from the certified set as of this certification (M5): the set + # must give exactly what the tree gives. + from . import certified + version = certified.version_of(conn, pid) + if version is None: + set_runs = [("*", "unsynced", "", 0, "", "", + {"error": "the certified set did not follow this pipeline"})] + else: + # The set as of this pipeline must hold exactly the tree's BOMs. + have, want = certified.members(conn, version), { + p: hashlib.sha256(_read(f)).hexdigest() for p, f in _files(new_root).items()} + if have != want: + set_runs_tree = [("(tree)", "mismatch", "", len(have), certified.digest_of(have), + certified.digest_of(want), { + "only_set": sorted(set(have) - set(want))[:_LIST], + "only_tree": sorted(set(want) - set(have))[:_LIST], + "different": sorted(p for p in set(have) & set(want) + if have[p] != want[p])[:_LIST]})] + else: + set_runs_tree = [("(tree)", "match", "", len(have), certified.digest_of(have), + certified.digest_of(want), {})] + set_root = certified.materialize(conn, version, os.path.join( + os.path.dirname(os.path.abspath(new_root)) + ".set")) + try: + mine, refused = compute(set_root) + finally: + shutil.rmtree(os.path.dirname(set_root), ignore_errors=True) + set_runs = set_runs_tree + ([refused] if refused else judge(mine, parsed)) + with db.transaction(conn): + # The CI's side of the canary's comparison (M8a). + from . import canary + canary.record_ci(conn, settings, pid, sha, before, new_root, old_root, parsed) + for mode, rows in (("shadow", runs), ("set", set_runs), (replay.MODE, replay_runs)): + for arch, state, source, n, ours_sha, signed_sha, d in rows: + _record(conn, pid, sha, mode, arch, state, source, n, ours_sha, signed_sha, d) + _set(conn, pid, state="compared", reason="") + + +def judge(ours, parsed): + """Sign runs for each architecture: our manifests against the signed ones.""" + runs = [] + for arch in sorted(set(ours) | set(parsed)): + mine = ours.get(arch) + doc, kid, source = parsed.get(arch, (None, None, "")) + if source == "superseded": + # The store's copy is of a later (or, for an architecture this run + # did not sign, an earlier) certification: not this one's. + if mine is not None: + runs.append((arch, "superseded", source, len(mine["packages"]), + digest(mine), "", {})) + continue + if mine is None: + runs.append((arch, "mismatch", source, 0, "", digest(doc) if doc else "", + {"error": "signed, but not in the worker's result"})) + continue + if arch not in parsed: + runs.append((arch, "mismatch", "", len(mine["packages"]), digest(mine), "", + {"error": "in the worker's result, but not signed"})) + continue + if doc is None or kid is None: + runs.append((arch, "unverified", source, len(mine["packages"]), digest(mine), + "", {"error": "the signed manifest does not verify"})) + continue + d, a, b = diff(mine, doc), digest(mine), digest(doc) + if a != b and not d: + d = {"order": "the same packages and fields, in another order"} + runs.append((arch, "match" if a == b else "mismatch", source, + len(mine["packages"]), a, b, d)) + return runs + + +def _record(conn, pid, sha, mode, arch, state, source, n, ours_sha, signed_sha, d): + conn.execute( + "INSERT INTO sign_runs (certification, mode, arch, state, source, source_commit," + " packages, manifest_sha256, signed_sha256, diff, created_ms)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + " ON CONFLICT(certification, mode, arch) DO UPDATE SET state = excluded.state," + " source = excluded.source, packages = excluded.packages," + " manifest_sha256 = excluded.manifest_sha256," + " signed_sha256 = excluded.signed_sha256, diff = excluded.diff," + " created_ms = excluded.created_ms", + (pid, mode, arch, state, source, sha, n, ours_sha, signed_sha, _canonical(d), db.now_ms())) + prefix = {"shadow": "shadow.", "set": "shadow.set_"}.get(mode, mode + ".") + if state == "unverified" and mode == "shadow": + events.append(conn, "shadow.unverified", _subject(pid), events.WORKER, + {"arch": arch, "source": source, "commit": sha}) + # The tree's result is an event either way; the set's only when it differs. + if (state == "mismatch" or (state == "match" and mode in ("shadow", "replay")) + or state in ("unsynced", "refused", "missing", "error")): + summary = {k: (v.get("count") if isinstance(v, dict) and "count" in v else True) + for k, v in d.items()} + if "error" in d: + summary = {"error": d["error"][:500]} + events.append(conn, prefix + state, _subject(pid), events.WORKER, + {"arch": arch, "packages": n, "source": source, "commit": sha, + **({"differs": summary} if state != "match" else {})}) + + +def runs(conn, limit=50, state=""): + """Certifications, newest first, each with its sign runs.""" + limit = max(1, min(int(limit), 500)) + if state: + rows = conn.execute("SELECT * FROM certifications WHERE state = ?" + " ORDER BY pipeline_id DESC LIMIT ?", (state, limit)).fetchall() + else: + rows = conn.execute("SELECT * FROM certifications ORDER BY pipeline_id DESC LIMIT ?", + (limit,)).fetchall() + out = [] + for r in rows: + c = {k: r[k] for k in r.keys()} + c["archs"] = json.loads(c["archs"] or "[]") + c["sign_runs"] = [dict({k: s[k] for k in s.keys()}, diff=json.loads(s["diff"] or "{}")) + for s in conn.execute("SELECT * FROM sign_runs WHERE certification = ?" + " ORDER BY arch", (r["pipeline_id"],))] + out.append(c) + return out + + +def summary(conn, since_ms=0, mode="shadow"): + """Counts of sign runs by state since *since_ms*: the gate's numbers. *mode* + 'shadow' (from the git tree, M4), 'set' (from the certified set, M5) or + 'replay' (the worker's own certification of the build, M8a).""" + return {r["state"]: r["n"] for r in conn.execute( + "SELECT state, COUNT(*) AS n FROM sign_runs WHERE mode = ? AND created_ms >= ?" + " GROUP BY state", (mode, int(since_ms)))} + + +# --- the thread ----------------------------------------------------------------- + +class Runner(threading.Thread): + """Discover and compare certifications on a thread of its own, with its own + connection, every *interval* seconds.""" + + def __init__(self, path, settings, deps, interval, stop, note): + """*note(conn, step, error)* records the step's failure or recovery.""" + super().__init__(name="shadow", daemon=True) + self.path, self.settings, self.deps = path, settings, deps + self.interval, self.stop_event, self.note = interval, stop, note + self.cache_dir = os.path.join(os.path.dirname(os.path.abspath(path)), "shadow") + + def run(self): + while not self.stop_event.is_set(): + try: + conn = db.connect(self.path) + except Exception: + log.exception("shadow: cannot open %s; trying again", self.path) + self.stop_event.wait(max(60.0, self.interval)) + continue + try: + while not self.stop_event.is_set(): + began = time.monotonic() + self.once(conn) + self.stop_event.wait(max(1.0, self.interval - (time.monotonic() - began))) + finally: + conn.close() + + def _console(self): + """The build project, for a build's ui-config.yaml (None: not readable).""" + try: + return self.deps.reader() + except Exception as e: + log.warning("replay: the build project cannot be read: %s", e) + return None + + def once(self, conn): + try: + reader = self.deps.manifests_reader() + store = self.deps.store() + if reader is None: + return + if store is None: + raise RuntimeError("BITS_ORCH_STORE is not set: certifications cannot be" + " checked against the store") + discover(conn, reader, self.settings) + from . import certified + certified.sync(conn, reader, self.cache_dir, branch=self.settings.orch_manifests_branch) + upto = certified.followed(conn) or 0 + console = self._console() + while not self.stop_event.is_set(): + if compare_one(conn, reader, store, self.settings, self.cache_dir, + upto=upto, console=console) is None: + break + certified.match_store(conn, store) + except Exception as exc: + log.exception("shadow certification failed") + self.note(conn, "shadow", "%s: %s" % (type(exc).__name__, exc)) + else: + self.note(conn, "shadow", None) diff --git a/console-backend/console_backend/orch/trace.py b/console-backend/console_backend/orch/trace.py new file mode 100644 index 00000000..bddaa2c1 --- /dev/null +++ b/console-backend/console_backend/orch/trace.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Who built and who approved what is on CVMFS (`bits trace`). + +The index: every package path prepub published, with the build (the CI pipeline +id) that published it, read from prepub's measurement records (GET +/api/v1/measurements on each of BITS_ORCH_PREPUB_URLS), which prepub keeps for +every build. A build is read again while it is recent, since records still come. + +`lookup` answers for one CVMFS path: the publishes of the longest published path +above it, and for each publishing pipeline and for the build that compiled the +package, who requested it (the console's verified BITS_CERTIFIER, or the queue's +requester) and its passkey approval, with the approval's event in the audit chain. +""" + +import json +import logging +import time +from datetime import datetime + +from . import db + +log = logging.getLogger("bits_orchestrator.trace") + +PER_PASS = 20 # per prepub +RECENT_MS = 2 * 24 * 3600 * 1000 # a build this recent is read again: records still come +REREAD_MS = 15 * 60 * 1000 +TIMEOUT_S = 10 + + +def _get(url): + import requests + r = requests.get(url, timeout=TIMEOUT_S) + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + + +def _ms(text): + try: + return int(datetime.fromisoformat(str(text).replace("Z", "+00:00")).timestamp() * 1000) + except ValueError: + return 0 + + +def _due(conn, prepub, builds, now): + """The builds of one prepub to read this pass: new ones, then recent ones not + read for a while, at most PER_PASS.""" + seen = {r["build_id"]: r for r in conn.execute( + "SELECT * FROM traced_builds WHERE prepub = ?", (prepub,))} + new = [b for b in builds if b not in seen] + again = [b for b in builds if b in seen and now - seen[b]["last_ms"] < RECENT_MS + and now - seen[b]["checked_ms"] > REREAD_MS] + return (new + again)[:PER_PASS] + + +def _unreachable(exc): + """The prepub itself cannot be reached (not one build's bad answer).""" + import requests + if isinstance(exc, (requests.HTTPError, ValueError)): # an answer, if a bad one + return False + return isinstance(exc, (requests.RequestException, OSError)) + + +def step(conn, prepubs, get=_get): + """Read the measurement records of new and recent builds into the index. A + build prepub cannot answer for is marked read, and tried again with the + recent ones; an unreachable prepub is left for the next pass.""" + n, now = 0, db.now_ms() + for prepub in prepubs: + base = prepub.rstrip("/") + try: + builds = get(base + "/api/v1/measurements") or [] + except Exception as exc: # one unreachable prepub does not stop the others + log.warning("%s: no measurements (%s)", base, exc) + continue + for build in _due(conn, base, [str(b) for b in builds if b and b != "latest"], now): + try: + records = get("%s/api/v1/measurements/%s" % (base, build)) or [] + except Exception as exc: + log.warning("%s: build %s unreadable (%s)", base, build, exc) + if _unreachable(exc): + break + records = [] + rows = [r for r in records if isinstance(r, dict)] if isinstance(records, list) else [] + # No records yet: recent from now, so it is read again while they come. + last = max([_ms(r.get("ts")) for r in rows] or [now]) + with db.transaction(conn): + for r in rows: + if r.get("outcome") != "published" or not r.get("path") or not r.get("job_id"): + continue + conn.execute( + "INSERT OR IGNORE INTO published_paths (repo, path, build_id, job_id," + " ts_ms, replaced, prepub) VALUES (?, ?, ?, ?, ?, ?, ?)", + (str(r.get("repo") or ""), str(r["path"]).strip("/"), build, + str(r["job_id"]), _ms(r.get("ts")), int(bool(r.get("replaced"))), base)) + conn.execute( + "INSERT INTO traced_builds (prepub, build_id, records, last_ms, checked_ms)" + " VALUES (?, ?, ?, ?, ?) ON CONFLICT(prepub, build_id) DO UPDATE SET" + " records = excluded.records, checked_ms = excluded.checked_ms," + " last_ms = CASE WHEN excluded.records > 0 THEN excluded.last_ms" + " ELSE traced_builds.last_ms END", (base, build, len(rows), last, now)) + n += 1 + return n + + +def _approval(conn, subject): + """The passkey pre-approval of a build id (the newest not revoked, else the + newest), with its audit event, or None.""" + a = conn.execute("SELECT * FROM approvals WHERE kind = 'preapproval' AND subject = ?" + " ORDER BY revoked_ms IS NOT NULL, id DESC LIMIT 1", + (str(subject),)).fetchone() + if a is None: + return None + ev = conn.execute("SELECT seq, hash FROM events WHERE subject = ? AND kind =" + " 'approval.granted' ORDER BY seq LIMIT 1", + ("approval:%d" % a["id"],)).fetchone() + return {"approval": a["id"], "approver": a["approver"], "via": a["via"], + "approved_at": _iso(a["created_ms"]), + "revoked_at": _iso(a["revoked_ms"]) if a["revoked_ms"] else "", + "event": {"seq": ev["seq"], "hash": ev["hash"]} if ev else None} + + +def _iso(ms): + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(ms / 1000)) if ms else "" + + +def pipeline(conn, pid): + """A pipeline as `bits trace` shows it: what, where, who requested, approval.""" + try: + pid = int(pid) + except (TypeError, ValueError): + return None + p = conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + if p is None: + return None + variables = json.loads(p["variables"] or "{}") + q = conn.execute("SELECT requested_by, batch FROM build_queue WHERE pipeline_id = ?" + " ORDER BY id LIMIT 1", (pid,)).fetchone() + # The queue's requester is the backend's own record; BITS_CERTIFIER is a pipeline + # variable, which the backend sets on the pipelines it starts, but anyone + # starting one in GitLab could set too. + who, how = ((q["requested_by"], "queue") if q and q["requested_by"] else + (variables.get("BITS_CERTIFIER", ""), "pipeline variable")) + return {"pipeline": pid, "web_url": p["web_url"], "community": p["community"], + "package": p["package"], "version": p["version"], "platform": p["platform"], + "created_at": p["created_at"], "outcome": p["outcome"], "source": p["source"], + "requested_by": who, "requested_by_from": how if who else "", + "queue_batch": q["batch"] if q else "", + "approval": _approval(conn, pid)} + + +def lookup(conn, repo, path, build_id="", visible=None): + """What is known of *path* (repo-relative) and of the build *build_id* that + compiled its package. *visible*: the communities whose pipelines may be + shown (None: all); others are named by id only.""" + parts = [p for p in str(path).strip("/").split("/") if p] + above = ["/".join(parts[:i]) for i in range(len(parts), 0, -1)] + rows, published = [], "" + if above: + rows = conn.execute( + "SELECT * FROM published_paths WHERE repo = ? AND path IN (%s)" + " ORDER BY length(path) DESC, ts_ms" % ",".join("?" * len(above)), + [repo] + above).fetchall() + published = rows[0]["path"] if rows else "" + rows = [r for r in rows if r["path"] == published] + + def shown(pid): + p = pipeline(conn, pid) + if p is None: + return {"pipeline": pid, "known": False} + if visible is not None and p["community"].lower() not in visible: + return {"pipeline": p["pipeline"], "known": True, "visible": False} + return dict(p, known=True, visible=True) + + def publish(r): + p = shown(r["build_id"]) + if p.get("visible") is False: + return p # who, when and where: only for the pipeline's community + return dict({"published_at": _iso(r["ts_ms"]), "job": r["job_id"], + "replaced": bool(r["replaced"]), "prepub": r["prepub"]}, **p) + + built = [] + if build_id: + for b in conn.execute("SELECT pipeline_id FROM builds WHERE build_id = ?" + " AND pipeline_id != 0 ORDER BY pipeline_id", (build_id,)): + built.append(shown(b["pipeline_id"])) + return {"repo": repo, "path": "/".join(parts), "published_path": published, + "publishes": [publish(r) for r in rows], + "build_id": build_id, "built_by_pipelines": built, + "indexed": bool(conn.execute("SELECT 1 FROM traced_builds LIMIT 1").fetchone())} diff --git a/console-backend/console_backend/orch/vuln_triage.py b/console-backend/console_backend/orch/vuln_triage.py new file mode 100644 index 00000000..54e8cfe7 --- /dev/null +++ b/console-backend/console_backend/orch/vuln_triage.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The admins' triage of vulnerability findings (VEX). + +A rule of a community says a vulnerability does not matter there, for one +version of a package, for every version of it, or for every package (package +and version empty); or that no vulnerability of a package does (vuln_id ANY: +one taken out of the analysis, a host tool say; licence findings stay): +not_affected (with a CycloneDX justification), +false_positive, or will_not_fix (an accepted risk). It has a reason, an author +and, optionally, an expiry; it is revoked rather than deleted. A finding a rule +covers is suppressed: left out of the effective counts, and served in the SBOM +with the rule as its CycloneDX `analysis`. Adding and revoking are events. +""" + +import re + +from . import db, events + +STATES = ("not_affected", "false_positive", "will_not_fix") +JUSTIFICATIONS = ("code_not_present", "code_not_reachable", "requires_configuration", + "requires_dependency", "requires_environment", "protected_by_compiler", + "protected_at_runtime", "protected_at_perimeter", + "protected_by_mitigating_control") +MAX_DAYS = 730 +_VULN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{1,63}$") +ANY = "*" # every vulnerability of the rule's package +_DAY_MS = 86400 * 1000 + + +class Invalid(ValueError): + pass + + +def _text(value, name, limit, required=False): + v = str(value or "").strip() + if len(v) > limit or ("\n" in v and name != "reason"): + raise Invalid("%s: at most %d characters, on one line" % (name, limit)) + if required and not v: + raise Invalid("%s is required" % name) + return v + + +def add(conn, community, vuln_id, state, reason, by, package="", version="", + justification="", days=None, now=None): + """Add a rule; returns its id. Raises Invalid.""" + community = _text(community, "community", 128, True).lower() + vuln_id = str(vuln_id or "").strip() + if not (_VULN.match(vuln_id) or vuln_id == ANY): + raise Invalid("vuln_id: a vulnerability id such as CVE-2024-1234, or * for all") + if state not in STATES: + raise Invalid("state is one of: %s" % ", ".join(STATES)) + justification = str(justification or "").strip() + if state == "not_affected" and justification not in JUSTIFICATIONS: + raise Invalid("not_affected needs a justification, one of: %s" % ", ".join(JUSTIFICATIONS)) + if state != "not_affected": + justification = "" + package = _text(package, "package", 128) + version = _text(version, "version", 128) + if version and not package: + raise Invalid("a version needs its package") + if vuln_id == ANY and not package: + raise Invalid("every vulnerability (*) needs a package") + reason = _text(reason, "reason", 2000, True) + now = db.now_ms() if now is None else now + expires = None + if days not in (None, "", 0): + try: + days = int(days) + except (TypeError, ValueError): + raise Invalid("days: a number of days") + if not 1 <= days <= MAX_DAYS: + raise Invalid("days: from 1 to %d" % MAX_DAYS) + expires = now + days * _DAY_MS + with db.transaction(conn): + rid = conn.execute( + "INSERT INTO vuln_triage (community, vuln_id, package, version, state, justification," + " reason, author, created_ms, expires_ms) VALUES (?,?,?,?,?,?,?,?,?,?)", + (community, vuln_id, package, version, state, justification, reason, by, now, + expires)).lastrowid + events.append(conn, "sbom.triaged", "vuln:%s" % vuln_id, events.Actor("person", by), + {"rule": rid, "community": community, "package": package, + "version": version, "state": state, "justification": justification, + "reason": reason, "expires_ms": expires}) + return rid + + +def get(conn, rule_id): + r = conn.execute("SELECT * FROM vuln_triage WHERE id = ?", (int(rule_id),)).fetchone() + return dict(r) if r else None + + +def revoke(conn, rule_id, by, now=None): + """Revoke a rule; False when it already was.""" + now = db.now_ms() if now is None else now + with db.transaction(conn): + cur = conn.execute("UPDATE vuln_triage SET revoked_ms = ?, revoked_by = ? WHERE id = ?" + " AND revoked_ms IS NULL", (now, by, int(rule_id))) + if cur.rowcount != 1: + return False + r = get(conn, rule_id) + events.append(conn, "sbom.triage_revoked", "vuln:%s" % r["vuln_id"], + events.Actor("person", by), {"rule": r["id"], "community": r["community"]}) + return True + + +def listing(conn, communities=None, revoked=False, now=None, limit=500): + """The rules of *communities* (None: all), newest first; the active ones + only unless *revoked* (then expired and revoked ones too).""" + now = db.now_ms() if now is None else now + q, where, args = "SELECT * FROM vuln_triage", [], [] + if communities is not None: + if not communities: + return [] + where.append("community IN (%s)" % ",".join("?" * len(communities))) + args += sorted(communities) + if not revoked: + where.append("revoked_ms IS NULL AND (expires_ms IS NULL OR expires_ms > ?)") + args.append(now) + if where: + q += " WHERE " + " AND ".join(where) + q += " ORDER BY id DESC LIMIT ?" + return [dict(r) for r in conn.execute(q, args + [int(limit)])] + + +def _covers(rule, vuln_id, names, version): + return (rule["vuln_id"].lower() in (vuln_id.lower(), ANY) + and (not rule["package"] or rule["package"].lower() in names) + and (not rule["version"] or rule["version"] == version)) + + +class Rules: + """The active rules by community, to look findings up in.""" + + def __init__(self, rules): + self._by = {} + for r in rules: + self._by.setdefault((r["community"], r["vuln_id"].lower()), []).append(r) + for rs in self._by.values(): # the most specific rule first, then the newest + rs.sort(key=lambda r: (not r["version"], not r["package"], -r["id"])) + + @classmethod + def load(cls, conn, communities=None, now=None): + return cls(listing(conn, communities, now=now, limit=1000000)) + + def rule(self, community, vuln_id, name, version, alias=""): + """The rule covering the finding, or None: one for this vulnerability + first, else one for every vulnerability of the package (not a licence's).""" + names, vid = {n.lower() for n in (name, alias) if n}, str(vuln_id).lower() + rules = list(self._by.get((community, vid), ())) + if not vid.startswith("licence:"): + rules += self._by.get((community, ANY), ()) + return next((r for r in rules if _covers(r, str(vuln_id), names, version)), None) + + +def analysis(rule): + """A rule as a CycloneDX vulnerability `analysis`.""" + out = {"state": "exploitable" if rule["state"] == "will_not_fix" else rule["state"], + "detail": "%s (%s)" % (rule["reason"], rule["author"])} + if rule["justification"]: + out["justification"] = rule["justification"] + if rule["state"] == "will_not_fix": + out["response"] = ["will_not_fix"] + return out + + +NOT_SHIPPED = {"tool": "a host tool, run but not shipped with the stack (sbom: tool)", + "tool-only": "used only by host tools, not shipped with the stack", + "build-only": "a system package only the build needs, not shipped with the stack"} + + +def annotate(vulns, components, rules, community, not_shipped=None): + """CycloneDX *vulns* with the rules of *community*: a vulnerability's affected + components a rule covers move to a copy of it with that rule's analysis (and + bits:triage:* properties); those *not_shipped* ({ref: why}) to one marked + not_affected (bits:not_shipped). *components* maps a bom-ref to its component.""" + out, not_shipped = [], not_shipped or {} + for v in vulns or []: + if not isinstance(v, dict): + continue + groups = {} + for a in v.get("affects") or []: + ref = a.get("ref") if isinstance(a, dict) else None + if isinstance(ref, str) and not_shipped.get(ref): + groups.setdefault(("not_shipped", not_shipped[ref]), (None, []))[1].append(a) + continue + c = components.get(ref) if isinstance(ref, str) else None + c = c if isinstance(c, dict) else {} + props = c.get("properties") + alias = next((p.get("value") for p in props if isinstance(p, dict) + and p.get("name") == "bits:package"), "") if isinstance(props, list) else "" + r = rules.rule(community, v.get("id"), str(c.get("name") or ""), + str(c.get("version") or ""), str(alias or "")) + groups.setdefault(r["id"] if r else None, (r, []))[1].append(a) + if list(groups) == [None] or not groups: + out.append(v) + continue + for key, (r, affects) in sorted(groups.items(), key=lambda g: ( + isinstance(g[0], tuple), g[0] if isinstance(g[0], tuple) else g[0] or 0)): + entry = dict(v, affects=affects) + if isinstance(key, tuple): + entry["bom-ref"] = "%s-%s" % (v.get("bom-ref") or v.get("id"), key[1]) + entry["analysis"] = {"state": "not_affected", + "detail": NOT_SHIPPED.get(key[1], "not shipped with the stack")} + entry["properties"] = [{"name": "bits:not_shipped", "value": key[1]}] + elif r is not None: + entry["bom-ref"] = "%s-triage-%d" % (v.get("bom-ref") or v.get("id"), key) + entry["analysis"] = analysis(r) + entry["properties"] = [{"name": "bits:triage:" + k, "value": str(r[k])} for k in ( + "id", "vuln_id", "package", "version", "author", "created_ms", "expires_ms") + if r[k]] + out.append(entry) + return out diff --git a/console-backend/console_backend/orch/watches.py b/console-backend/console_backend/orch/watches.py new file mode 100644 index 00000000..5073dfd2 --- /dev/null +++ b/console-backend/console_backend/orch/watches.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Commit CI (ADR bits 0009): a schedule may follow branches, its watch list +(url@branch;...). Every WATCH_S the worker reads each branch's head with git's +own protocol over HTTPS (protocol v2 ls-refs: no git binary, no API quota), +keeping the heads it read and those it last built (schedule_watches). A +schedule whose heads changed fires a commit run of every cell once they have +held still for SETTLE_MS (or have waited MAX_WAIT_MS) and none of its commit +runs is still queued or running; nothing is cancelled. A branch first seen (or +seen again) is not built: only a change is. Commit runs certify and publish +nothing, and record the heads when they fired: a build reads its branches then. +""" + +import datetime as dt +import ipaddress +import json +import logging +import socket +import time +import urllib.parse + +import requests + +from . import db, events, nightlies, schedules + +log = logging.getLogger("bits_orchestrator.watches") + +SETTLE_MS = 5 * 60 * 1000 # a head unchanged this long is a burst of pushes ended +MAX_WAIT_MS = 30 * 60 * 1000 # a branch pushed to all the time still builds this often +_CHECKED, _FAILED = "watches.checked_ms", "watches.failed" +_PASS_S = 120 # a pass reads no longer: the rest wait for the next +_DEADLINE_S = 15 # a whole answer, at most this long +_MAX_BYTES = 64 * 1024 # ... and this big: refs under one branch name +_HEX = frozenset("0123456789abcdef") + + +class WatchError(Exception): + """A repository whose heads cannot be read now (it is tried again).""" + + +def _pkt(text): + b = text.encode() + return b"%04x" % (len(b) + 4) + b + + +def _no_auth(r): # not the worker's ~/.netrc: these are anyone's repositories + return r + + +def _public(url): + """The upload-pack URL of *url*; WatchError unless its host is public (DNS is + read again on connect: this keeps out a mistyped or internal host, not a + rebinding one, which only sees an ls-refs request).""" + u = urllib.parse.urlsplit(url) + try: + port = u.port or 443 + except ValueError: + raise WatchError("%s: not a port" % url) + if u.scheme != "https" or not u.hostname: + raise WatchError("%s: not an https URL" % url) + try: + addrs = {a[4][0] for a in socket.getaddrinfo(u.hostname, port, proto=socket.IPPROTO_TCP)} + except (OSError, UnicodeError) as e: + raise WatchError("%s: %s" % (url, e)) + for a in addrs: + if not ipaddress.ip_address(a.split("%")[0]).is_global: + raise WatchError("%s: %s is not a public address" % (url, a)) + path = u.path.rstrip("/") + return urllib.parse.urlunsplit(("https", u.netloc, (path if path.endswith(".git") else path + ".git") + + "/git-upload-pack", "", "")) + + +def _body(r, url): + data, until = b"", time.monotonic() + _DEADLINE_S + try: + for chunk in r.iter_content(8192): + data += chunk + if len(data) > _MAX_BYTES: + raise WatchError("%s: an answer over %d bytes" % (url, _MAX_BYTES)) + if time.monotonic() > until: + raise WatchError("%s: no whole answer in %ds" % (url, _DEADLINE_S)) + except requests.RequestException as e: + raise WatchError("%s: %s" % (url, str(e)[:200])) + finally: + r.close() + return data + + +def _refs(data, url): + """{ref: sha} of an ls-refs answer; WatchError for anything else.""" + out, i = {}, 0 + while True: + size = data[i:i + 4] + n = int(size, 16) if len(size) == 4 and set(size) <= set(b"0123456789abcdef") else -1 + if n == 0 and i + 4 == len(data): # the flush that ends the list + return out + if n < 5 or i + n > len(data): + raise WatchError("%s: not a whole git protocol v2 answer" % url) + line = data[i + 4:i + n].decode("utf-8", "replace").rstrip("\n") + if line.startswith("ERR "): + raise WatchError("%s: %s" % (url, line[4:200])) + words = line.split(" ") + if len(words) < 2 or len(words[0]) not in (40, 64) or not set(words[0]) <= _HEX: + raise WatchError("%s: not a git protocol v2 answer" % url) + out[words[1]] = words[0] + i += n + + +def head(url, branch): + """The commit at the head of *branch* of the repository at *url* (https, a + public host), or '' when it has no such branch.""" + target, ref = _public(url), "refs/heads/" + branch + body = _pkt("command=ls-refs\n") + b"0001" + _pkt("ref-prefix %s\n" % ref) + b"0000" + try: + r = requests.post(target, data=body, timeout=(5, 10), stream=True, allow_redirects=False, + auth=_no_auth, headers={ + "Content-Type": "application/x-git-upload-pack-request", + "Accept": "application/x-git-upload-pack-result", "Git-Protocol": "version=2", + "Accept-Encoding": "identity", "User-Agent": "git/2 bits-orchestrator"}) + except requests.RequestException as e: + raise WatchError("%s: %s" % (url, str(e)[:200])) + if r.status_code != 200: + r.close() + raise WatchError("%s: HTTP %s" % (url, r.status_code)) + return _refs(_body(r, url), url).get(ref, "") + + +def _kv(conn, key): + r = conn.execute("SELECT value FROM kv WHERE key = ?", (key,)).fetchone() + return r["value"] if r else None + + +def read(conn, now=None, every_s=120, get_head=head): + """Read the heads of every enabled schedule's watch list, at most every + *every_s*. Returns how many heads changed; WatchError after the others were + read when some could not be (and on each pass until a read is due).""" + now = now or db.now_ms() + last = _kv(conn, _CHECKED) + if last is not None and now - int(last) < every_s * 1000: + if _kv(conn, _FAILED): # a step failing until the next read, not on and off + raise WatchError(_kv(conn, _FAILED)) + return 0 + seen, failed, changed, want, bad = {}, [], 0, {}, set() + until = time.monotonic() + _PASS_S + for s in schedules.listing(conn): + if s["enabled"] and schedules.watches(s): + want[s["id"]] = (s, schedules.watches(s)) + for _s, ws in want.values(): + for w in ws: + if w in seen: + continue + seen[w] = None # not read now: as it was + if w[0] in bad or time.monotonic() > until: + continue # a repository that failed, or no time left + try: + seen[w] = get_head(*w) + except Exception as e: # anything: the other repositories are read + bad.add(w[0]) + failed.append(str(e) if isinstance(e, WatchError) else "%s: %r" % (w[0], e)) + if time.monotonic() > until: + failed.append("not all branches read in %ds: the rest on the next read" % _PASS_S) + with db.transaction(conn): + for sid in {r[0] for r in conn.execute("SELECT DISTINCT schedule_id FROM schedule_watches")} - set(want): + conn.execute("DELETE FROM schedule_watches WHERE schedule_id = ?", (sid,)) + for sid, (s, ws) in want.items(): + have = {(r["repo"], r["branch"]): r for r in conn.execute( + "SELECT * FROM schedule_watches WHERE schedule_id = ?", (sid,))} + for w in set(have) - set(ws): + conn.execute("DELETE FROM schedule_watches WHERE schedule_id = ? AND repo = ? AND branch = ?", + (sid,) + w) + for w in ws: + sha, old = seen[w], have.get(w) + if sha is None or (old is not None and old["sha"] == sha): + continue + if old is None: # first seen: its head is the one built + conn.execute("INSERT INTO schedule_watches (schedule_id, repo, branch, sha, built_sha," + " changed_ms) VALUES (?, ?, ?, ?, ?, ?)", (sid,) + w + (sha, sha, now)) + continue + changed += 1 + again = not old["sha"] # gone and back: seen again, not built + moved = old["moved_ms"] if old["sha"] != old["built_sha"] and old["moved_ms"] else now + conn.execute("UPDATE schedule_watches SET sha = ?, built_sha = ?, changed_ms = ?, moved_ms = ?" + " WHERE schedule_id = ? AND repo = ? AND branch = ?", + (sha, sha if again else old["built_sha"], now, moved, sid) + w) + events.append(conn, "schedule.head_moved", nightlies.subject(sid), events.WORKER, + {"repo": w[0], "branch": w[1], "from": old["sha"], "to": sha}) + for k, v in ((_CHECKED, str(now)), (_FAILED, "; ".join(failed[:5]))): + conn.execute("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE" + " SET value = excluded.value", (k, v)) + if failed: + raise WatchError("; ".join(failed[:5])) + return changed + + +def ready(conn, s, now): + """The heads to build (JSON) when *s* should fire a commit run now, else None.""" + want = set(schedules.watches(s)) + rows = [r for r in conn.execute("SELECT * FROM schedule_watches WHERE schedule_id = ?", (s["id"],)) + if (r["repo"], r["branch"]) in want and r["sha"]] # a branch gone holds nothing + moved = [r for r in rows if r["sha"] != r["built_sha"]] + if not moved: + return None + if (any(now - r["changed_ms"] < SETTLE_MS for r in moved) + and now - min(r["moved_ms"] for r in moved) < MAX_WAIT_MS): + return None # still moving + if conn.execute("SELECT 1 FROM schedule_runs WHERE schedule_id = ? AND fired_by = 'commit'" + " AND state IN ('queued', 'running') LIMIT 1", (s["id"],)).fetchone(): + return None # one goes on: after it + return json.dumps({"%s@%s" % (r["repo"], r["branch"]): r["sha"] for r in rows}, sort_keys=True) + + +def step(conn, reader, ref, tz, now=None, every_s=120, get_head=head): + """The worker step: read the heads, then fire the commit runs that are ready.""" + now = now or db.now_ms() + errors = [] + try: + read(conn, now, every_s, get_head) + except WatchError as e: + errors.append(str(e)) # fire on what was read; say so after + cfgs, fired = {}, 0 + for s in schedules.listing(conn): + if not s["enabled"] or not schedules.watches(s) or ready(conn, s, now) is None: + continue + if s["community_dir"] not in cfgs: # read before the write lock + try: + cfgs[s["community_dir"]] = nightlies.config(reader, s["community_dir"], ref) + except Exception as e: # an unreadable config: the next pass + errors.append("schedule %s: the console's configuration: %s" % (s["id"], e)) + continue + with db.transaction(conn): + fresh = schedules.get(conn, s["id"]) + heads = ready(conn, fresh, now) if fresh and fresh["enabled"] else None + if heads is None: + continue + night = dt.datetime.fromtimestamp(now / 1000, tz).date().isoformat() + nightlies.fire_night(conn, fresh, night, schedules.cells(fresh), cfgs[s["community_dir"]], ref, + "commit", now, fired_by="commit", commits=heads) + for w, sha in json.loads(heads).items(): + repo, branch = w.rsplit("@", 1) + conn.execute("UPDATE schedule_watches SET built_sha = ?, moved_ms = 0 WHERE schedule_id = ?" + " AND repo = ? AND branch = ?", (sha, s["id"], repo, branch)) + fired += 1 + if errors: + raise WatchError("; ".join(errors[:5])) + return fired + + +def heads(conn, sid): + """What a schedule follows, as last read: [{repo, branch, sha, built_sha, changed_ms}].""" + return [{k: r[k] for k in ("repo", "branch", "sha", "built_sha", "changed_ms")} for r in conn.execute( + "SELECT * FROM schedule_watches WHERE schedule_id = ? ORDER BY repo, branch", (int(sid),))] diff --git a/console-backend/console_backend/orch/worker.py b/console-backend/console_backend/orch/worker.py new file mode 100644 index 00000000..43939b3e --- /dev/null +++ b/console-backend/console_backend/orch/worker.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator worker: one process, one loop. + +Each pass runs the steps in order. A step reads the database and the outside +world, then writes what it decided; a step that fails is logged, recorded as an +event when its error changes, and tried again on the next pass. The steps: +running due actions, then hints, discover, settle, index and index_cli (reconcile.py: what +is true of builds, from GitLab and S3; with BITS_ORCH_QUEUE, after settle, dispatch +starts queued builds on free hosts, build_queue.py), with BITS_ORCH_CVMFS_STATUS the console's +CVMFS status document (cvmfs_status.py), then collect and backfill (failures.py: +every failed job's record, deduplicated into issues), phases (phases.py: +what each finished build's job spent its time on), trace (trace.py: which +build published each CVMFS path), then ship (outside.py: the +events to the outside copy of the audit chain) and the daily backup. Certification +in shadow (shadow.py) runs on a thread of its own, since a first comparison +hashes a whole architecture of the store; so do the worker's certifications +(certifier.py, M8a: the "certify" step only requests them). Schedules are added by +later milestones. + + python -m console_backend.orch.worker # BITS_ORCH_DB=/orch/orch.db +""" + +import json +import logging +import os +import signal +import threading +import time + +from .. import config +from . import (actions, approvals, authority, build_queue, certifier, cvmfs_status, db, events, + failures, kinds, mirror, nightlies, outside, phases, reconcile, sbom_history, + sbom_scan, schedules, shadow, trace, watches) + +log = logging.getLogger("bits_orchestrator") + + +class Worker: + def __init__(self, conn, settings, interval=30.0, heartbeat=None, path=None): + self.conn = conn + self.settings = settings + self.interval = interval + self.heartbeat = heartbeat + self.path = path + self.outside = outside.Client.from_settings(settings) + self.shadow = None + self.certifier = None + self.deps = kinds.Deps(settings) + self.authorize = authority.checker(settings) + # Actions first, so an approved cancel never waits behind a long catch-up. + self.steps = [ + ("actions", self._run_actions), + ("hints", lambda: reconcile.apply_hints(self.conn, self.deps.reader())), + ("discover", lambda: reconcile.discover(self.conn, self.deps.reader(), + settings.orch_lookback_days)), + ("settle", lambda: reconcile.settle(self.conn, self.deps.reader())), + ("index", lambda: reconcile.index(self.conn, self.deps.store())), + ("index_cli", lambda: reconcile.index_cli(self.conn, self.deps.store())), + ("collect", lambda: failures.collect(self.conn, self.deps.reader())), + ("backfill", lambda: failures.backfill(self.conn, self.deps.reader(), + settings.orch_backfill_days, + settings.orch_lookback_days)), + # What each finished build's job spent its time on (the timeline). + ("phases", lambda: phases.step(self.conn, self.deps.reader())), + # Which build published each CVMFS path (bits trace). + ("trace", lambda: trace.step(self.conn, settings.orch_prepub_urls)), + ("ship", lambda: outside.ship(self.conn, self.outside)), + ("backup", self._backup), + ] + if settings.orch_certify: + # Request the certifications of new builds, after they are + # recorded; the certifier thread runs them. + at = [n for n, _ in self.steps].index("index") + 1 + self.steps.insert(at, ("certify", lambda: certifier.trigger( + self.conn, settings, self.deps.reader()))) + # Its own step: a paused or failing certify step never holds back a + # revocation. + self.steps.insert(at, ("withdraw", lambda: certifier.withdrawals(self.conn, settings))) + if settings.orch_cvmfs_status: + # The console's CVMFS status, from the builds just recorded. + at = [n for n, _ in self.steps].index("index") + 1 + self.steps.insert(at, ("cvmfs_status", self._cvmfs_status)) + if settings.orch_queue: + # Queued builds start on free hosts, once the running ones are settled. + at = [n for n, _ in self.steps].index("settle") + 1 + self.steps.insert(at, ("dispatch", self._dispatch)) + # The nightly schedules that are due queue their builds, for this dispatch, + # and those whose followed branches moved their commit builds. + self.steps.insert(at, ("watches", self._watches)) + self.steps.insert(at, ("schedules", self._schedules)) + # What became of their runs, once pipelines and failures are recorded. + at = [n for n, _ in self.steps].index("collect") + 1 + self.steps.insert(at, ("nightly_results", lambda: nightlies.results(self.conn))) + if settings.orch_scanner_url: + # The published builds' SBOMs against the scanner's vulnerability database. + at = [n for n, _ in self.steps].index("trace") + 1 + self.steps.insert(at, ("scan", lambda: sbom_scan.step( + self.conn, self.deps.store(), settings.orch_scanner_url, settings.orch_scans_dir))) + # Each release's risk, once a day (the Vulnerabilities page's chart). + self.steps.insert(at + 1, ("history", lambda: sbom_history.snapshot(self.conn))) + if settings.orch_owns_set: + # M8b: the set is the worker's; bits-manifests mirrors it, and the + # store says whose manifests these are. + self.steps.insert(len(self.steps) - 2, ("mirror", self._mirror)) + self.steps.insert(len(self.steps) - 2, ("marker", self._marker)) + self._errors = {} # step -> last error text + self._stop = threading.Event() + + def _cvmfs_status(self): + from .. import forge_ops + forge, store = forge_ops.GitLabForge.from_settings(self.settings), self.deps.store() + if forge is None or store is None: + raise RuntimeError("BITS_ORCH_CVMFS_STATUS needs the ops token (BITS_FORGE_OPS_TOKEN)" + " and the store (BITS_ORCH_STORE)") + cvmfs_status.step(self.conn, forge, store) + + def _dispatch(self): + forge = self.deps.forge() + hs = build_queue.hosts(forge) + if hs is None: # GitLab cannot tell: the hosts it last gave + hs = build_queue.last_hosts(self.conn) + try: + # A certifying build's batch approval becomes its pipeline's pre-approval. + build_queue.dispatch(self.conn, forge, hs, approvals.preapprove_queued) + except build_queue.Stop as e: + raise RuntimeError("builds wait: %s" % e) + + def _schedules(self): + refs = self.settings.forge_refs + nightlies.step(self.conn, self.deps.reader(), "main" if "main" in refs else sorted(refs)[0], + schedules.time_zone()) + + def _watches(self): + refs = self.settings.forge_refs + watches.step(self.conn, self.deps.reader(), "main" if "main" in refs else sorted(refs)[0], + schedules.time_zone(), every_s=self.settings.orch_watch_s) + + def _mirror(self): + from .. import forge_ops + forge = forge_ops.GitLabForge.from_settings(self.settings, "mirror", + project=self.settings.orch_manifests_project) + mirror.push(self.conn, self.settings, forge) + + def _marker(self): + if self._errors.get("marker") is None and getattr(self, "_marked", False): + return + from .. import credproxy + store = self.deps.store() + if store is None: + raise RuntimeError("BITS_ORCH_STORE is not set: the worker that owns the set" + " writes its manifests there") + mirror.write_marker(self.conn, self.settings, store, credproxy.s3_writer(self.settings)[0], + certifier.issuer(store), certifier.write_prefix(self.settings)) + self._marked = True + + def _backup(self): + if self.settings.orch_backup_dir: + path = db.backup(self.conn, self.settings.orch_backup_dir) + if path: + log.info("database copied to %s", path) + + def _run_actions(self): + # Certifications run on the certifier thread: hashing a stack must + # never delay a cancel. + actions.run_due(self.conn, self.deps, self.authorize, exclude={certifier.KIND, certifier.WITHDRAW, + certifier.DECERTIFY, + certifier.DECERTIFY_PACKAGE}) + + def _note(self, step, error, conn=None): + """Record a step's failure or recovery as an event, but only when it + changes, so a step failing every pass does not flood the log. *conn*: + the connection of the thread the step runs on.""" + if self._errors.get(step) == error: + return + self._errors[step] = error + conn = conn or self.conn + try: + with db.transaction(conn): + events.append(conn, "worker.step_failed" if error else "worker.step_recovered", + "worker", events.WORKER, {"step": step, "error": error}) + except Exception: + log.exception("cannot record step state for %s", step) + + def run_once(self): + for name, step in self.steps: + if self._stop.is_set(): + break + try: + step() + except Exception as exc: + log.exception("step %s failed", name) + self._note(name, "%s: %s" % (type(exc).__name__, exc)) + else: + if self._errors.get(name): + self._note(name, None) + if self.heartbeat: + with open(self.heartbeat, "w") as fh: + fh.write(str(int(time.time()))) + + def _check_outside(self): + """Refuse to start on a database restored from a copy older than the + outside one: the first event appended would fork the chain.""" + if self.outside is None: + return + # Signing waits for this run's first comparison, not one the database + # (perhaps a restored copy) remembers. + with db.transaction(self.conn): + r = self.conn.execute("SELECT value FROM kv WHERE key = 'outside.head'").fetchone() + if r: + self.conn.execute("UPDATE kv SET value = ? WHERE key = 'outside.head'", + (json.dumps(dict(json.loads(r["value"]), ms=0)),)) + try: + gap = outside.behind(self.conn, self.outside) + except outside.OutsideError as e: + log.warning("cannot compare with the outside audit copy (%s); signing waits until" + " it is in step", e) + return + if gap: + with db.transaction(self.conn): + self.conn.execute("INSERT INTO kv (key, value) VALUES ('outside.behind', ?)" + " ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (json.dumps({"seq": gap[0], "outside": gap[1]}),)) + raise outside.OutsideError( + "this database ends at event %d, the outside audit copy at %d: it was restored" + " from an older copy. With the worker stopped, run `python -m" + " console_backend.orch.outside recover --by `" % gap) + + def start(self): + self._check_outside() + with db.transaction(self.conn): + events.append(self.conn, "worker.started", "worker", events.WORKER, + {"schema": db.SCHEMA_VERSION}) + failures.seed_signatures(self.conn) + n = actions.recover(self.conn) + if n: + log.warning("recovered %d interrupted action(s)", n) + if self.path and self.settings.orch_manifests_project and not self.settings.orch_owns_set: + # Until M8b: the set is read from bits-manifests and its certifications + # compared. Once the worker owns the set, it mirrors it there instead. + self.shadow = shadow.Runner(self.path, self.settings, self.deps, self.interval, + self._stop, lambda c, step, err: self._note(step, err, c)) + self.shadow.start() + if self.path and self.settings.orch_certify: + self.certifier = certifier.Certifier( + self.path, self.settings, lambda: kinds.Deps(self.settings), self.authorize, + self.interval, self._stop, lambda c, step, err: self._note(step, err, c)) + self.certifier.start() + + def loop(self): + self.start() + while not self._stop.is_set(): + began = time.monotonic() + self.run_once() + self._stop.wait(max(1.0, self.interval - (time.monotonic() - began))) + for t in (self.shadow, self.certifier): + if t is not None: + t.join(timeout=5) + with db.transaction(self.conn): + events.append(self.conn, "worker.stopped", "worker", events.WORKER, {}) + + def stop(self, *_): + self._stop.set() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + settings = config.Settings() + if not settings.orch_db: + raise SystemExit("BITS_ORCH_DB is not set") + problem = authority.unresolvable(settings) + if problem: + raise SystemExit(problem) + path = settings.orch_db + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + try: + with db.worker_lock(path): + conn = db.connect(path) + db.migrate(conn) + w = Worker(conn, settings, interval=settings.orch_interval, + heartbeat=path + ".heartbeat", path=path) + signal.signal(signal.SIGTERM, w.stop) + signal.signal(signal.SIGINT, w.stop) + log.info("orchestrator worker on %s (schema %d, every %ss)", + path, db.SCHEMA_VERSION, settings.orch_interval) + w.loop() + except db.LockedError as e: + raise SystemExit("bits-orchestrator: %s; not starting a second worker" % e) + except outside.OutsideError as e: + raise SystemExit("bits-orchestrator: %s" % e) + + +if __name__ == "__main__": + main() diff --git a/console-backend/console_backend/session.py b/console-backend/console_backend/session.py index 08307f4c..6b012324 100644 --- a/console-backend/console_backend/session.py +++ b/console-backend/console_backend/session.py @@ -113,6 +113,35 @@ def get(self, build_id): return None return entry + # The operations the sign path uses (the database-backed store in + # orch/approvals.py has the same ones). Synchronous, so atomic against the + # event loop in this single process. + def approve(self, build_id, data, assertion=None): + """Record *data* as the approved pre-approval of *build_id*; False when one + is already approved.""" + existing = self.get(build_id) + if existing and existing.get("status") == "approved": + return False + self.put(build_id, dict(data, status="approved")) + return True + + def reserve_sign(self, build_id, cap, expect=None): + """Take one of the approval's signs. Returns the record it was taken on + (give it to release_sign), or None when none is left or there is no + approval.""" + pre = self.get(build_id) + if not pre or pre.get("status") != "approved" or pre.get("signs", 0) >= cap: + return None + pre["signs"] = pre.get("signs", 0) + 1 + return pre + + def release_sign(self, build_id, slot=None): + """Give back a sign to the record it was taken on, if that is still the + approval of the build.""" + pre = self.get(build_id) + if pre is not None and (slot is None or pre is slot): + pre["signs"] = max(0, pre.get("signs", 1) - 1) + def _make_room(self): # Evict oldest PENDING (or expired) first; an approved-but-not-yet-consumed # record is still needed by the CI sign step, so keep it unless the store is diff --git a/console-backend/pyproject.toml b/console-backend/pyproject.toml index ef5ed089..f5cc93a0 100644 --- a/console-backend/pyproject.toml +++ b/console-backend/pyproject.toml @@ -14,9 +14,15 @@ dependencies = [ "httpx==0.28.1", "pyjwt[crypto]==2.15.1", "webauthn==2.5.2", + # GitLab (forge_ops) and the store through the credentials proxy (credproxy); + # both were installed only by the image until now. + "requests>=2.31", + "boto3>=1.34", + # bits_helpers' site configuration (site.py) reads YAML. + "pyyaml", ] -# bits_helpers is provided by the surrounding bits repo (on PYTHONPATH / copied -# alongside in the image), not a PyPI dependency. +# bits_helpers comes from the bits checkout this lives in (the commit bits-console +# pins as a submodule): the backend is part of bits, not released on its own. [tool.setuptools] -packages = ["console_backend"] +packages = ["console_backend", "console_backend.orch"] diff --git a/console-backend/tests/test_credproxy.py b/console-backend/tests/test_credproxy.py new file mode 100644 index 00000000..4abfc7d0 --- /dev/null +++ b/console-backend/tests/test_credproxy.py @@ -0,0 +1,228 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""M6: the backend and the worker reach GitLab and the store through the +credentials proxy, holding only its gate tokens.""" + +import importlib.util +import json +import os +import shutil +import socket +import tempfile +import threading +import unittest +from unittest.mock import patch + +from console_backend import authz, config, credproxy, forge_ops +from console_backend.orch import kinds + + +class FakeAgent: + """The proxy's agent socket: a line with a route name in, its port and gate + token out.""" + + def __init__(self, path, routes=("gitlab-ops", "gitlab-read", "gitlab-groups", "s3"), port=4711): + self.path, self.routes, self.port, self.asked = path, set(routes), port, [] + self.srv = socket.socket(socket.AF_UNIX) + self.srv.bind(path) + self.srv.listen(8) + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + c, _ = self.srv.accept() + except OSError: + return + with c: + route = c.makefile().readline().strip() + self.asked.append(route) + reply = ({"port": self.port, "host": "0.0.0.0", "service": route, + "token": "gate-" + route} if route in self.routes + else {"error": "unknown service %r" % route}) + c.sendall((json.dumps(reply) + "\n").encode()) + + def close(self): + self.srv.close() + + +class _Proxy(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.sock = os.path.join(self.dir, "agent.sock") + self.agent = FakeAgent(self.sock) + self.addCleanup(self.agent.close) + credproxy._cache.clear() + self.addCleanup(credproxy._cache.clear) + self.addCleanup(credproxy._clients.clear) + self.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gitlab.cern.ch/api/v4", "BITS_FORGE_PROJECT": "42", + "BITS_FORGE_OPS_TOKEN": "env-ops", "BITS_CRED_PROXY_AGENT_SOCKET": self.sock, + "BITS_ORCH_STORE": "b3://lcgapp-bits-testing::rw"}) + + +class CredProxyTest(_Proxy): + def test_gitlab_through_the_proxy(self): + self.assertEqual(credproxy.gitlab(self.settings, "ops"), + ("http://credentials-proxy:4711/api/v4", "gate-gitlab-ops")) + self.assertEqual(credproxy.gitlab(self.settings, "read")[1], "gate-gitlab-read") + credproxy.gitlab(self.settings, "ops") + self.assertEqual(self.agent.asked, ["gitlab-ops", "gitlab-read"]) # cached + self.agent.port = 4712 # a restart + self.assertEqual(credproxy.endpoint(self.settings, "gitlab-ops", refresh=True)[0], + "http://credentials-proxy:4712") + + def test_without_the_proxy_the_environment(self): + s = config.Settings(env={"GITLAB_API_URL": "https://gl/api/v4", + "BITS_FORGE_OPS_TOKEN": "env-ops"}) + self.assertEqual(credproxy.gitlab(s, "ops"), ("https://gl/api/v4", "env-ops")) + self.assertEqual(credproxy.gitlab(s, "read"), ("https://gl/api/v4", "env-ops")) + self.assertEqual(credproxy.gitlab(s, "groups"), ("", "")) + self.assertFalse(credproxy.has(s, "groups")) + self.assertTrue(credproxy.has(self.settings, "groups")) + + def test_a_missing_proxy_or_route_is_an_outage(self): + self.agent.routes.discard("gitlab-groups") + with self.assertRaises(credproxy.Unavailable) as e: + credproxy.gitlab(self.settings, "groups") + self.assertIn("credentials proxy", str(e.exception)) + self.settings.cred_proxy_agent_socket = os.path.join(self.dir, "none.sock") + with self.assertRaises(OSError): + credproxy.gitlab(self.settings, "ops") + + def test_the_forge_sends_the_gate_token_to_the_proxy(self): + seen = [] + + class R: + status_code = 200 + + def json(self): + return {"id": 7} + with patch.object(forge_ops.requests, "get", + lambda url, headers, timeout: seen.append((url, headers)) or R()): + f = forge_ops.GitLabForge.from_settings(self.settings) + self.assertEqual(f.get_pipeline(7), {"id": 7}) + self.assertEqual(seen, [("http://credentials-proxy:4711/api/v4/projects/42/pipelines/7", + {"PRIVATE-TOKEN": "gate-gitlab-ops"})]) + self.agent.close() + os.unlink(self.sock) + credproxy._cache.clear() + with self.assertRaises(forge_ops.ForgeError) as e: # the proxy went away + f.get_pipeline(7) + self.assertEqual(e.exception.status, 503) + + @unittest.skipUnless(importlib.util.find_spec("boto3"), "needs boto3 (in the image)") + def test_the_worker_reads_and_its_store_go_through_the_proxy(self): + checked = [] + p = patch.object(kinds, "_check_bucket", lambda c, b: checked.append(b)) + p.start() + self.addCleanup(p.stop) + credproxy._clients.clear() + d = kinds.Deps(self.settings) + r = d.reader() + self.assertEqual((r._api, r._token), ("http://credentials-proxy:4711/api/v4", "gate-gitlab-read")) + s3, bucket = d.store() + self.assertEqual((s3.meta.endpoint_url, bucket), + ("http://credentials-proxy:4711", "lcgapp-bits-testing")) + self.assertEqual(s3._request_signer._credentials.access_key, "gate-s3") + self.assertIs(d.store()[0], s3) # kept + credproxy._cache.clear() + self.agent.port = 4800 + self.assertEqual(d.store()[0].meta.endpoint_url, "http://credentials-proxy:4800") + self.assertEqual(checked, ["lcgapp-bits-testing"] * 2) # once per client + + @unittest.skipUnless(importlib.util.find_spec("boto3"), "needs boto3 (in the image)") + def test_a_public_store_is_read_with_no_credentials(self): + checked = [] + p = patch.object(kinds, "_check_bucket", + lambda c, b, anonymous=False: checked.append((b, anonymous))) + p.start() + self.addCleanup(p.stop) + self.settings.orch_store_anonymous = True # wins over the proxy + d = kinds.Deps(self.settings) + s3, bucket = d.store() + from botocore import UNSIGNED + self.assertEqual((s3.meta.endpoint_url, bucket), ("https://s3.cern.ch", "lcgapp-bits-testing")) + self.assertIs(s3.meta.config.signature_version, UNSIGNED) + self.assertNotIn("s3", self.agent.asked) + self.assertIs(d.store()[0], s3) # kept + self.assertEqual(checked, [("lcgapp-bits-testing", True)]) + self.settings.orch_s3_endpoint = "http://s3.cern.ch" + with self.assertRaises(kinds.NotConfigured): + kinds.Deps(self.settings).store() + + def test_a_bucket_that_is_not_public_is_not_configured(self): + class E(Exception): + response = {"ResponseMetadata": {"HTTPStatusCode": 403}} + + class C: + def list_objects_v2(self, **kw): + raise E() + with self.assertRaises(kinds.NotConfigured) as e: + kinds._check_bucket(C(), "b", anonymous=True) + self.assertIn("anonymously", str(e.exception)) + + def test_paths_are_encoded_once_more_for_the_proxy(self): + seen = [] + + class R: + status_code = 200 + + def json(self): + return {} + with patch.object(forge_ops.requests, "get", + lambda url, headers, timeout: seen.append(url) or R()): + f = forge_ops.GitLabForge.from_settings(self.settings, "read", + project="buncic/bits-manifests") + f.finished_pipelines("main") + f._file_exists("communities/lcg/pipelines/a.json", "main") + direct = forge_ops.GitLabForge("https://gl/api/v4", "t", "buncic/bits-manifests") + direct.commit("abc") + direct.runner(7) + self.assertEqual(seen[-1], "https://gl/api/v4/runners/7") # not under the project + seen.pop() + self.assertEqual(seen, [ + "http://credentials-proxy:4711/api/v4/projects/buncic%252Fbits-manifests/pipelines" + "?ref=main&scope=finished&order_by=id&sort=desc&per_page=50", + "http://credentials-proxy:4711/api/v4/projects/buncic%252Fbits-manifests/repository/" + "files/communities%252Flcg%252Fpipelines%252Fa.json?ref=main", + "https://gl/api/v4/projects/buncic%2Fbits-manifests/repository/commits/abc"]) + self.assertEqual(credproxy.segment(self.settings, "a/b"), "a%2Fb") + self.assertEqual(credproxy.segment(config.Settings(env={}), "a/b"), "a/b") + + def test_a_bucket_the_proxy_cannot_read_is_not_configured(self): + class E(Exception): + response = {"ResponseMetadata": {"HTTPStatusCode": 404}} + + class C: + def list_objects_v2(self, **kw): + raise E() + with self.assertRaises(kinds.NotConfigured): + kinds._check_bucket(C(), "b") + + def test_a_route_missing_from_the_config_is_not_a_credential(self): + self.agent.routes.discard("gitlab-groups") + self.assertFalse(credproxy.has(self.settings, "groups")) + self.assertTrue(credproxy.has(self.settings, "ops")) + self.settings.cred_proxy_agent_socket = os.path.join(self.dir, "none.sock") + self.assertTrue(credproxy.has(self.settings, "ops")) # down, not unconfigured + + def test_the_admin_policy_reads_groups_through_the_proxy(self): + self.settings.admins_group = "bits/admins" + calls = [] + + def members(api, token, group, inherited=True): + calls.append((api, token, group)) + return {"carol"} + with patch.object(authz.forge, "gitlab_group_members", members), \ + patch.object(authz.forge, "gitlab_subgroups", lambda api, token, root: []): + pol = authz.tree_policy(self.settings) + # The proxy decodes the path once: the group path goes encoded once more. + self.assertEqual(calls, [("http://credentials-proxy:4711/api/v4", "gate-gitlab-groups", + "bits%2Fadmins")]) + self.assertIn("carol", pol["*"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_ops_admin.py b/console-backend/tests/test_ops_admin.py index 475d7a90..adffaa2d 100644 --- a/console-backend/tests/test_ops_admin.py +++ b/console-backend/tests/test_ops_admin.py @@ -75,6 +75,22 @@ def test_ci_var_key_not_allowlisted(self): json={"key": "BITS_FORGE_OPS_TOKEN", "value": "steal"}) self.assertEqual(r.status_code, 400) + @patch("console_backend.forge_ops.requests.get") + def test_ci_var_read_by_any_admin(self, get): + url = "/ops/admin/ci-var?key=BITS_DESIRED_REF" + self.assertEqual(self.client.get(url).status_code, 401) + self.assertEqual(self.client.get(url, headers=self._hdr("dave")).status_code, 403) + self.assertEqual(self.client.get("/ops/admin/ci-var?key=BITS_FORGE_OPS_TOKEN", + headers=self._hdr("carol")).status_code, 400) + get.return_value = FakeResp(200, {"key": "BITS_DESIRED_REF", "value": "v1.9.3"}) + r = self.client.get(url, headers=self._hdr("alice")) + self.assertEqual(r.json(), {"key": "BITS_DESIRED_REF", "value": "v1.9.3"}) + self.assertTrue(get.call_args.args[0].endswith("/projects/42/variables/BITS_DESIRED_REF")) + get.return_value = FakeResp(404) + self.assertIsNone(self.client.get(url, headers=self._hdr("carol")).json()["value"]) + get.return_value = FakeResp(500) + self.assertEqual(self.client.get(url, headers=self._hdr("carol")).status_code, 502) + # ---- refresh ------------------------------------------------------------- def test_refresh_group_admin_denied(self): r = self.client.post("/ops/admin/refresh", headers=self._hdr("alice"), json={}) diff --git a/console-backend/tests/test_orch.py b/console-backend/tests/test_orch.py new file mode 100644 index 00000000..54fba9d3 --- /dev/null +++ b/console-backend/tests/test_orch.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M0: database, event chain, actions, worker, API.""" + +import multiprocessing +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, forge_ops, main +from console_backend.orch import actions, db, events, kinds, worker + +ALICE = events.Actor("person", "alice") +ADVISOR = events.Actor("advisor", "triage") + + +class FakeForge: + """Pipelines 1-9 belong to 'testbed', 10+ to no community.""" + + def __init__(self, fail=None): + self.calls = [] + self.fail = fail or {} + + def pipeline_community(self, pid): + return "testbed" if int(pid) < 10 else "" + + def _do(self, what, pid): + self.calls.append((what, pid)) + if what in self.fail: + raise self.fail[what] + + def retry_pipeline(self, pid): + self._do("retry", pid) + + def cancel_pipeline(self, pid): + self._do("cancel", pid) + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + +class DatabaseTest(_DB): + def test_migrate_is_idempotent_and_versioned(self): + self.assertEqual(db.migrate(self.conn), db.SCHEMA_VERSION) + self.assertEqual(db.version(self.conn), db.SCHEMA_VERSION) + db.check(self.conn) + self.assertEqual(self.conn.execute("PRAGMA journal_mode").fetchone()[0], "wal") + + def test_newer_or_unmigrated_schema_is_refused(self): + self.conn.execute("PRAGMA user_version=%d" % (db.SCHEMA_VERSION + 1)) + with self.assertRaises(db.SchemaError): + db.check(self.conn) + with self.assertRaises(db.SchemaError): + db.migrate(self.conn) + fresh = db.connect(os.path.join(self.dir, "new.db")) + self.addCleanup(fresh.close) + with self.assertRaises(db.SchemaError): + db.check(fresh) + + def test_a_failed_transaction_leaves_nothing(self): + with self.assertRaises(ZeroDivisionError): + with db.transaction(self.conn): + events.append(self.conn, "x", "s", ALICE) + 1 / 0 + self.assertEqual(events.since(self.conn), []) + + +def _hold_lock(path, ready, release): + with db.worker_lock(path): + ready.set() + release.wait(10) + + +class LockTest(_DB): + def test_a_second_worker_is_refused(self): + ready, release = multiprocessing.Event(), multiprocessing.Event() + p = multiprocessing.Process(target=_hold_lock, args=(self.path, ready, release)) + p.start() + try: + self.assertTrue(ready.wait(10)) + with self.assertRaises(db.LockedError): + with db.worker_lock(self.path): + pass + finally: + release.set() + p.join(10) + with db.worker_lock(self.path): # free again once the holder exits + pass + + +class EventsTest(_DB): + def _add(self, n): + for i in range(n): + with db.transaction(self.conn): + events.append(self.conn, "test.e", "s%d" % (i % 2), ALICE, {"i": i}) + + def test_append_needs_a_transaction(self): + with self.assertRaises(RuntimeError): + events.append(self.conn, "x", "s", ALICE) + + def test_chain_verifies_and_since_pages(self): + self._add(5) + self.assertEqual(events.verify(self.conn), (5, None)) + got = events.since(self.conn, after=2, limit=2) + self.assertEqual([e["seq"] for e in got], [3, 4]) + self.assertEqual(got[0]["actor"], {"kind": "person", "id": "alice"}) + self.assertEqual([e["seq"] for e in events.since(self.conn, subject="s1")], [2, 4]) + + def test_an_edited_or_removed_event_breaks_the_chain(self): + self._add(5) + self.conn.execute("UPDATE events SET payload = '{\"i\":99}' WHERE seq = 3") + self.assertEqual(events.verify(self.conn), (3, 3)) + self.conn.execute("DELETE FROM events") + self._add(5) + self.conn.execute("DELETE FROM events WHERE seq = 2") + self.assertEqual(events.verify(self.conn)[1], 3) + + def test_actor_and_payload_limits(self): + with self.assertRaises(ValueError): + events.Actor("robot", "x") + with self.assertRaises(ValueError): + events.Actor("person", "") + with self.assertRaises(ValueError): + with db.transaction(self.conn): + events.append(self.conn, "x", "s", ALICE, {"big": "x" * events.MAX_PAYLOAD}) + + +class ValidateTest(unittest.TestCase): + def test_schema_subset(self): + s = {"type": "object", "required": ["a"], "additionalProperties": False, + "properties": {"a": {"type": "string", "pattern": "[0-9]+"}, + "b": {"type": "integer", "minimum": 1}, + "c": {"type": "array", "items": {"enum": ["x", "y"]}}}} + actions.validate(s, {"a": "12", "b": 3, "c": ["x"]}) + for bad in ({}, {"a": "1x"}, {"a": "1", "b": 0}, {"a": "1", "b": True}, + {"a": "1", "c": ["z"]}, {"a": "1", "d": 1}, {"a": "1" * 2000}): + with self.assertRaises(actions.ActionError, msg=bad): + actions.validate(s, bad) + + +class ActionsTest(_DB): + def setUp(self): + super().setUp() + self.forge = FakeForge() + self.deps = kinds.Deps(None) + self.deps.forge = lambda: self.forge + self.allow = lambda actor, community: actor.kind == "person" and actor.id == "alice" + + def _req(self, pid="5", actor=ALICE, kind="pipeline.retry", **kw): + return actions.request(self.conn, kind, {"pipeline": pid}, actor, "testbed", + kw.pop("authorized", True), **kw) + + def _kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + def test_catalogue_lists_kinds_with_schemas(self): + cat = {c["kind"]: c for c in actions.catalogue()} + self.assertIn("pipeline.retry", cat) + self.assertEqual(cat["pipeline.cancel"]["params"]["required"], ["pipeline"]) + + def test_an_authorized_person_is_approved_at_request(self): + row, created = self._req() + self.assertTrue(created) + self.assertEqual((row["state"], row["approved_by_id"], row["subject"]), + ("approved", "alice", "pipeline:5")) + self.assertEqual(self._kinds(), ["action.requested", "action.approved"]) + + def test_the_same_idempotency_key_returns_the_first_action(self): + a, _ = self._req(idempotency_key="k1") + b, created = self._req(idempotency_key="k1") + self.assertEqual((a["id"], created), (b["id"], False)) + c, _ = self._req() + self.assertNotEqual(c["id"], a["id"]) + with self.assertRaises(actions.ActionError): # same key, other params + self._req(pid="6", idempotency_key="k1") + # Keys belong to their requester: another person's "k1" is their own action. + d, created = self._req(actor=events.Actor("person", "carol"), idempotency_key="k1") + self.assertTrue(created) + self.assertNotEqual(d["id"], a["id"]) + + def test_bad_requests(self): + with self.assertRaises(actions.ActionError): + self._req(pid="12a") + with self.assertRaises(actions.ActionError): + self._req(pid="05") # one spelling per pipeline + with self.assertRaises(actions.ActionError): + self._req(kind="pipeline.explode") + with self.assertRaises(actions.ActionError): + self._req(authorized=False) + + def test_an_advisor_only_proposes_and_only_a_person_decides(self): + row, _ = self._req(actor=ADVISOR, authorized=False) + self.assertEqual(row["state"], "proposed") + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 0) + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], ADVISOR, True) + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], events.Actor("person", "mallory"), False) + ok = actions.approve(self.conn, row["id"], ALICE, True) + self.assertEqual((ok["state"], ok["approved_by_id"]), ("approved", "alice")) + with self.assertRaises(actions.ActionError): + actions.reject(self.conn, row["id"], ALICE, True) # no longer proposed + other, _ = self._req(actor=ADVISOR, authorized=False, pid="6") + rej = actions.reject(self.conn, other["id"], ALICE, True, "not now") + self.assertEqual((rej["state"], rej["error"]), ("rejected", "not now")) + + def test_run_due_runs_and_records(self): + self._req() + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 1) + self.assertEqual(self.forge.calls, [("retry", "5")]) + row = actions.listing(self.conn)[0] + self.assertEqual((row["state"], row["result"], row["attempts"]), + ("done", {"status": "retried"}, 1)) + self.assertEqual(self._kinds()[-2:], ["action.started", "action.done"]) + self.assertEqual(events.verify(self.conn)[1], None) + + def test_transient_errors_back_off_then_fail(self): + self.forge.fail["retry"] = forge_ops.ForgeError(503, "busy") + row, _ = self._req() + for attempt in range(1, actions.MAX_ATTEMPTS + 1): + self.conn.execute("UPDATE actions SET not_before_ms = 0") + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 1) + state = actions.get(self.conn, row["id"]) + self.assertEqual(state["attempts"], attempt) + self.assertEqual(state["state"], "failed") + self.assertIn("busy", state["error"]) + self.assertEqual(self._kinds().count("action.retrying"), actions.MAX_ATTEMPTS - 1) + + def test_backoff_delays_the_next_attempt(self): + self.forge.fail["retry"] = forge_ops.ForgeError(0, "timeout") + self._req() + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(actions.run_due(self.conn, self.deps, self.allow), 0) + + def test_permanent_errors_fail_at_once(self): + self.forge.fail["cancel"] = forge_ops.ForgeError(404, "no pipeline") + row, _ = self._req(kind="pipeline.cancel") + actions.run_due(self.conn, self.deps, self.allow) + got = actions.get(self.conn, row["id"]) + self.assertEqual((got["state"], got["attempts"]), ("failed", 1)) + + def test_a_lapsed_authority_fails_the_action_unrun(self): + self._req() + actions.run_due(self.conn, self.deps, lambda a, c: False) + self.assertEqual(self.forge.calls, []) + self.assertIn("no longer authorized", actions.listing(self.conn)[0]["error"]) + + def test_an_unreadable_policy_retries(self): + self._req() + + def broken(actor, community): + raise OSError("gitlab down") + actions.run_due(self.conn, self.deps, broken) + row = actions.listing(self.conn)[0] + self.assertEqual(row["state"], "approved") + self.assertIn("gitlab down", row["error"]) + + def test_a_later_action_waits_for_an_earlier_one_backing_off(self): + self.forge.fail["retry"] = forge_ops.ForgeError(503, "busy") + self._req() + self._req(kind="pipeline.cancel") + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(self.forge.calls, [("retry", "5")]) # the cancel waits + del self.forge.fail["retry"] + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, self.allow) + self.assertEqual(self.forge.calls, [("retry", "5"), ("retry", "5"), ("cancel", "5")]) + + def test_an_approver_kind_needs_someone_else(self): + k = actions.Kind(name="test.second", summary="", params={"type": "object"}, + approval="approver", subject=lambda p: "t", scope=lambda p, d: "", + run=lambda p, d: None) + actions.register(k) + self.addCleanup(actions.REGISTRY.pop, "test.second") + row, _ = actions.request(self.conn, "test.second", {}, ALICE, "testbed", True) + self.assertEqual(row["state"], "proposed") + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, row["id"], ALICE, True) + ok = actions.approve(self.conn, row["id"], events.Actor("person", "carol"), True) + self.assertEqual(ok["state"], "approved") + + def test_listing_filters_communities_before_the_limit(self): + self._req() + for pid in range(10, 15): + actions.request(self.conn, "pipeline.cancel", {"pipeline": str(pid)}, ALICE, "", + True) + got = actions.listing(self.conn, communities={"testbed"}, limit=2) + self.assertEqual([r["community"] for r in got], ["testbed"]) + self.assertEqual(actions.listing(self.conn, communities=set()), []) + + def test_one_running_action_per_subject(self): + a, _ = self._req() + b, _ = self._req() + first = actions.claim_next(self.conn) + self.assertEqual(first["id"], a["id"]) + self.assertIsNone(actions.claim_next(self.conn)) # b waits for a + actions.finish(self.conn, first, result={}) + self.assertEqual(actions.claim_next(self.conn)["id"], b["id"]) + + def test_recover_reruns_or_fails_interrupted_actions(self): + a, _ = self._req() + actions.claim_next(self.conn) + unsafe = actions.Kind(name="test.unsafe", summary="", params={"type": "object"}, + approval="requester", subject=lambda p: "u", + scope=lambda p, d: "", run=lambda p, d: None, + safe_to_rerun=False) + actions.register(unsafe) + self.addCleanup(actions.REGISTRY.pop, "test.unsafe") + b, _ = actions.request(self.conn, "test.unsafe", {}, ALICE, "", True) + actions.claim_next(self.conn) + self.assertEqual(actions.recover(self.conn), 2) + self.assertEqual(actions.get(self.conn, a["id"])["state"], "approved") + self.assertEqual(actions.get(self.conn, b["id"])["state"], "failed") + + def test_an_action_that_keeps_crashing_the_worker_stops(self): + a, _ = self._req() + for _ in range(actions.MAX_ATTEMPTS): + self.assertIsNotNone(actions.claim_next(self.conn)) + actions.recover(self.conn) + self.assertEqual(actions.get(self.conn, a["id"])["state"], "failed") + + +class WorkerTest(_DB): + def setUp(self): + super().setUp() + self.settings = config.Settings(env={}) + self.w = worker.Worker(self.conn, self.settings, + heartbeat=os.path.join(self.dir, "hb")) + self.forge = FakeForge() + self.w.deps.forge = lambda: self.forge + self.w.authorize = lambda actor, community: True + + def test_a_pass_runs_actions_and_beats(self): + self.w.start() + actions.request(self.conn, "pipeline.retry", {"pipeline": "3"}, ALICE, "testbed", True) + self.w.run_once() + self.assertEqual(self.forge.calls, [("retry", "3")]) + self.assertTrue(os.path.exists(os.path.join(self.dir, "hb"))) + + def test_a_failing_step_is_recorded_once_and_its_recovery(self): + calls = [] + + def flaky(): + calls.append(1) + if len(calls) < 3: + raise OSError("disk") + self.w.steps = [("flaky", flaky)] + with self.assertLogs("bits_orchestrator", "ERROR"): + for _ in range(4): + self.w.run_once() + kinds_ = [e["kind"] for e in events.since(self.conn)] + self.assertEqual(kinds_, ["worker.step_failed", "worker.step_recovered"]) + + def test_the_worker_needs_a_resolve_token_for_group_policies(self): + from console_backend.orch import authority + self.assertEqual(authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_POLICY": "testbed @alice"})), "") + self.assertIn("RESOLVE_TOKEN", authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_POLICY": "testbed &cern/bits-admins"}))) + self.assertIn("RESOLVE_TOKEN", authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_GROUP": "bits-admins"}))) + self.assertEqual(authority.unresolvable(config.Settings(env={ + "BITS_ADMINS_GROUP": "bits-admins", "BITS_ADMIN_RESOLVE_TOKEN": "t"})), "") + + def test_the_real_checker_needs_a_person_admin(self): + from console_backend.orch import authority + s = config.Settings(env={"BITS_ADMINS_POLICY": "testbed @alice\n* @carol"}) + ok = authority.checker(s) + self.assertTrue(ok(ALICE, "testbed")) + self.assertFalse(ok(ALICE, "other")) + self.assertFalse(ok(ALICE, "")) + self.assertTrue(ok(events.Actor("person", "carol"), "")) + self.assertFalse(ok(events.Actor("schedule", "nightly"), "testbed")) + self.assertFalse(ok(ADVISOR, "testbed")) + self.assertTrue(ok(events.WORKER, "testbed")) + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "testbed @alice\nother @bob\n* @carol", + "BITS_FORGE_OPS_TOKEN": "optok", "BITS_FORGE_PROJECT": "42", + "BITS_ORCH_DB": self.path}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.forge = FakeForge() + p = patch.object(forge_ops.GitLabForge, "from_settings", lambda s: self.forge) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + + def _h(self, user): + return {"Authorization": "Bearer %s" % user} if user else {} + + def _post(self, user, body, path="/ops/actions"): + return self.client.post(path, headers=self._h(user), json=body) + + def test_catalogue_needs_a_login(self): + self.assertEqual(self.client.get("/ops/actions/catalogue").status_code, 401) + r = self.client.get("/ops/actions/catalogue", headers=self._h("dave")) + self.assertIn("pipeline.retry", [a["kind"] for a in r.json()["actions"]]) + + def test_a_community_admin_requests_and_the_worker_runs(self): + r = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}, + "idempotency_key": "x"}) + self.assertEqual(r.status_code, 201, r.text) + self.assertEqual((r.json()["state"], r.json()["community"]), ("approved", "testbed")) + again = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}, + "idempotency_key": "x"}) + self.assertEqual((again.status_code, again.json()["id"]), (200, r.json()["id"])) + actions.run_due(self.conn, kinds.Deps(main.settings), lambda a, c: True) + self.assertEqual(self.forge.calls, [("retry", "5")]) + got = self.client.get("/ops/actions/%d" % r.json()["id"], headers=self._h("alice")) + self.assertEqual(got.json()["state"], "done") + + def test_authority_is_the_target_community(self): + body = {"kind": "pipeline.retry", "params": {"pipeline": "5"}} + self.assertEqual(self._post(None, body).status_code, 401) + self.assertEqual(self._post("bob", body).status_code, 403) + unscoped = {"kind": "pipeline.cancel", "params": {"pipeline": "12"}} + self.assertEqual(self._post("alice", unscoped).status_code, 403) + self.assertEqual(self._post("carol", unscoped).status_code, 201) + self.assertEqual(self._post("alice", {"kind": "pipeline.retry", + "params": {"pipeline": "x"}}).status_code, 400) + self.assertEqual(self._post("alice", {"kind": "nope", "params": {}}).status_code, 400) + + def test_lists_and_reads_are_limited_to_ones_communities(self): + mine = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}).json() + self._post("carol", {"kind": "pipeline.cancel", "params": {"pipeline": "12"}}) + seen = lambda u: [a["id"] for a in self.client.get( + "/ops/actions", headers=self._h(u)).json()["actions"]] + self.assertEqual(seen("alice"), [mine["id"]]) + self.assertEqual(seen("bob"), []) + self.assertEqual(len(seen("carol")), 2) + self.assertEqual(self.client.get("/ops/actions/%d" % mine["id"], + headers=self._h("bob")).status_code, 404) + + def test_a_proposal_is_approved_by_an_admin_of_its_community(self): + row, _ = actions.request(self.conn, "pipeline.retry", {"pipeline": "5"}, ADVISOR, + "testbed", False) + url = "/ops/actions/%d/approve" % row["id"] + self.assertEqual(self._post("bob", {}, url).status_code, 404) + r = self._post("alice", {}, url) + self.assertEqual((r.status_code, r.json()["state"]), (200, "approved")) + self.assertEqual(self._post("alice", {}, url).status_code, 409) + other, _ = actions.request(self.conn, "pipeline.retry", {"pipeline": "6"}, ADVISOR, + "testbed", False) + r = self._post("alice", {"reason": "flaky runner"}, + "/ops/actions/%d/reject" % other["id"]) + self.assertEqual((r.json()["state"], r.json()["error"]), ("rejected", "flaky runner")) + + def test_events_are_for_overall_admins(self): + self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}) + self.assertEqual(self.client.get("/ops/events", headers=self._h("alice")).status_code, 403) + r = self.client.get("/ops/events?after=1", headers=self._h("carol")) + self.assertEqual([e["kind"] for e in r.json()["events"]], ["action.approved"]) + r = self.client.get("/ops/events?after=99&wait=1", headers=self._h("carol")) + self.assertEqual(r.json()["events"], []) + + def test_an_unconfigured_forge_is_503_to_the_caller(self): + with patch.object(forge_ops.GitLabForge, "from_settings", lambda s: None): + r = self._post("alice", {"kind": "pipeline.retry", "params": {"pipeline": "5"}}) + self.assertEqual(r.status_code, 503) + + def test_another_users_key_is_not_theirs(self): + body = {"kind": "pipeline.cancel", "params": {"pipeline": "12"}, "idempotency_key": "k"} + first = self._post("carol", body).json() + mine = self._post("alice", dict(body, params={"pipeline": "5"})) + self.assertEqual(mine.status_code, 201) + self.assertNotEqual(mine.json()["id"], first["id"]) + self.assertEqual(self._post("carol", dict(body, params={"pipeline": "13"})).status_code, + 400) + + def test_unconfigured_or_unmigrated_is_503(self): + main.settings.orch_db = "" + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertEqual(r.status_code, 503) + main.settings.orch_db = os.path.join(self.dir, "empty.db") + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertEqual(r.status_code, 503) + self.assertIn("has the worker started", r.json()["detail"]) + self.assertFalse(os.path.exists(main.settings.orch_db)) # not created + db.connect(main.settings.orch_db).close() + r = self.client.get("/ops/actions", headers=self._h("alice")) + self.assertIn("not migrated", r.json()["detail"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_approvals.py b/console-backend/tests/test_orch_approvals.py new file mode 100644 index 00000000..1b847cb7 --- /dev/null +++ b/console-backend/tests/test_orch_approvals.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M3: approvals in the database. The sign path's tests run +again against the database-backed store, and its own properties are tested: +surviving a restart, expiry, revocation, the sign cap across processes, standing +approvals.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, main +from console_backend.orch import approvals, db, events + +import test_cli_preapprove as tcp +import test_preapprove as tp +import test_sign_preapproved as tsp + + +class _WithDB: + """Run a sign-path test class with approved pre-approvals in a database: + every Settings the tests build gets BITS_ORCH_DB.""" + + def setUp(self): + self._dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self._dir, True) + self.db_path = os.path.join(self._dir, "orch.db") + c = db.connect(self.db_path) + db.migrate(c) + c.close() + real_init = config.Settings.__init__ + path = self.db_path + + def init(s, env=None): + real_init(s, env) + s.orch_db = path + p = patch.object(config.Settings, "__init__", init) + p.start() + self.addCleanup(p.stop) + # The sign-path tests replace these module functions and never put them + # back; keep them from leaking into the tests that run after these. + for mod, names in ((main.ci_auth, ("verify_ci_token", "load_ci_signers", + "is_ci_authorized")), + (main.identity, ("verify_gitlab_token",))): + for name in names: + self.addCleanup(setattr, mod, name, getattr(mod, name)) + super().setUp() + main.settings.orch_db = path + main.preapprovals = approvals.Preapprovals(lambda: main.settings) + + def _rows(self): + c = db.connect(self.db_path) + try: + return c.execute("SELECT * FROM approvals").fetchall() + finally: + c.close() + + +class TestSignPreapprovedDB(_WithDB, tsp.TestSignPreapproved): + @unittest.skip("sets the in-memory sign counter by hand; the cap is tested below") + def test_preapproval_status_lookup(self): + pass + + def test_cli_preapproval_refusals_do_not_consume(self): + # As the in-memory test, where an unused record has signs 0 rather than none. + with patch.object(self, "assertNotIn", + lambda key, rec: self.assertEqual(rec.get(key, 0), 0)): + super().test_cli_preapproval_refusals_do_not_consume() + + def test_exhausted_after_the_cap(self): + self._preapprove("p1", ["lcg"]) + for _ in range(main._PREAPPROVAL_SIGN_CAP): + self.assertEqual(self._sign("p1", "lcg").status_code, 200) + self.assertEqual(self._sign("p1", "lcg").status_code, 429) + self.assertEqual(self.client.get("/preapproval/p1").json()["status"], "exhausted") + + def test_an_unusable_store_fails_the_sign_closed(self): + self._preapprove("p1", ["lcg"]) + main.settings.orch_db = self.db_path + ".gone" + self.assertEqual(self._sign("p1", "lcg").status_code, 503) + + def test_a_sign_is_an_event_and_the_record_outlives_the_process(self): + self._preapprove("p1", ["lcg"]) + self.assertEqual(self._sign("p1", "lcg").status_code, 200) + main.preapprovals = approvals.Preapprovals(lambda: main.settings) # a restart + rec = main.preapprovals.get("p1") + self.assertEqual((rec["status"], rec["user"], rec["signs"]), ("approved", "alice", 1)) + c = db.connect(self.db_path) + kinds = [e["kind"] for e in events.since(c)] + self.assertEqual(kinds, ["approval.granted", "approval.used"]) + self.assertEqual(events.verify(c)[1], None) + c.close() + + +class TestPreapproveDB(_WithDB, tp.TestPreapprove): + @unittest.skip("memory eviction does not apply to the database") + def test_eviction_keeps_approved(self): + pass + + def test_the_approval_keeps_its_assertion(self): + self.test_flow_records_preapproval() + (r,) = self._rows() + self.assertEqual((r["kind"], r["via"], r["approver"]), ("preapproval", "console", "alice")) + self.assertIn("clientDataJSON", r["assertion"]) + + +class TestCliPreapproveDB(_WithDB, tcp.TestCliPreapprove): + def test_the_bound_packages_are_stored(self): + self.test_full_flow_records_bound_preapproval() + (r,) = self._rows() + self.assertEqual(r["via"], "cli") + self.assertTrue(json.loads(r["packages"])) + self.assertTrue(r["challenge"]) + + +class StoreTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.settings = config.Settings(env={"BITS_ORCH_DB": self.path}) + + def _store(self, cap=16): + return approvals.Preapprovals(lambda: self.settings, sign_cap=cap) + + def _approve(self, s, bid="77", user="alice"): + return s.approve(bid, {"groups": ["lcg"], "user": user, "challenge": b"c"}, + assertion={"id": "cred"}) + + def test_pending_in_memory_approved_in_the_database(self): + s = self._store() + s.put("77", {"groups": ["lcg"], "user": "alice", "status": "pending", "challenge": b"c"}) + self.assertEqual(s.get("77")["status"], "pending") + self.assertIsNone(self._store().get("77")) # another process: not there + self.assertTrue(self._approve(s)) + self.assertEqual(self._store().get("77")["status"], "approved") + self.assertFalse(self._approve(s)) # already approved + + def test_the_sign_cap_holds_across_processes(self): + a, b = self._store(cap=3), self._store(cap=3) + self._approve(a) + got = [bool(x) for x in (a.reserve_sign("77", 16), b.reserve_sign("77", 16), + a.reserve_sign("77", 16), b.reserve_sign("77", 16))] + self.assertEqual(got, [True, True, True, False]) + b.release_sign("77", a.get("77")["id"]) + self.assertTrue(a.reserve_sign("77", 16)) + self.assertFalse(a.reserve_sign("nope", 16)) + + def test_a_sign_is_given_back_to_the_approval_it_was_taken_on(self): + s = self._store() + self._approve(s) + first = s.get("77")["id"] + slot = s.reserve_sign("77", 16, first) + self.assertEqual(slot, first) + approvals.revoke(self.conn, first, "carol") # meanwhile: revoked and re-granted + self._approve(s, user="bob") + second = s.get("77")["id"] + self.assertIsNone(s.reserve_sign("77", 16, first)) # not the expected one any more + s.release_sign("77", slot) # the failed sign of the first + signs = dict(self.conn.execute("SELECT id, signs FROM approvals").fetchall()) + self.assertEqual(signs, {first: 0, second: 0}) + + def test_expiry_and_revocation(self): + s = self._store() + self._approve(s) + self.conn.execute("UPDATE approvals SET expires_ms = 1") + self.assertIsNone(s.get("77")) + self.assertTrue(self._approve(s, user="bob")) # a fresh one may be given + (r,) = approvals.listing(self.conn) + self.assertTrue(approvals.revoke(self.conn, r["id"], "carol")) + self.assertFalse(approvals.revoke(self.conn, r["id"], "carol")) + self.assertIsNone(s.get("77")) + self.assertFalse(s.reserve_sign("77", 16)) + self.assertEqual(approvals.listing(self.conn), []) + self.assertEqual(len(approvals.listing(self.conn, active_only=False)), 2) + + def test_an_unusable_database_fails_closed(self): + self.settings.orch_db = os.path.join(self.dir, "missing.db") + with self.assertRaises(approvals.StoreError): + self._store().get("77") + + def test_standing_runs_and_days(self): + sha = "ab" * 32 + sched = "communities/lcg/pipelines/nightly.json" + aid = approvals.grant_standing(self.conn, sched, sha, ["lcg"], "alice", 2, 30, + {"id": "c"}, b"chal") + self.assertIsNotNone(aid) + self.assertIsNone(approvals.grant_standing(self.conn, sched, sha, ["lcg"], "alice", 2, 30, + {}, b"x")) # one per schedule content + self.assertIsNone(approvals.use_standing(self.conn, sched, "cd" * 32, ["lcg"], 1)) + self.assertIsNone(approvals.use_standing(self.conn, sched, sha, ["lcg", "atlas"], 1)) + self.assertIsNotNone(approvals.use_standing(self.conn, sched, sha, ["lcg"], 1)) + self.assertIsNotNone(approvals.use_standing(self.conn, sched, sha, ["lcg"], 2)) + self.assertIsNone(approvals.use_standing(self.conn, sched, sha, ["lcg"], 3)) # runs spent + for runs, days in ((0, 1), (101, 1), (1, 31), (1, 0)): + with self.assertRaises(ValueError): + approvals.grant_standing(self.conn, sched, "ee" * 32, ["lcg"], "a", runs, days, + {}, b"x") + kinds = [e["kind"] for e in events.since(self.conn)] + self.assertEqual(kinds, ["approval.granted", "approval.used", "approval.used"]) + self.assertIsNone(approvals.use_standing(self.conn, sched, sha, [], 4)) + (row,) = approvals.listing(self.conn, active_only=False) + self.assertTrue(approvals.verify_standing(row)) + self.conn.execute("UPDATE approvals SET groups = '[\"lcg\", \"atlas\"]'") + self.assertFalse(approvals.verify_standing( + self.conn.execute("SELECT * FROM approvals").fetchone())) # tampered + + def test_listing_by_community_before_the_limit(self): + for i in range(5): + self._approve(self._store(), bid="o%d" % i, user="bob") + self.conn.execute("UPDATE approvals SET groups = '[\"atlas\"]'") + self._approve(self._store(), bid="mine") + got = approvals.listing(self.conn, limit=2, communities={"lcg"}, user="nobody") + self.assertEqual([r["subject"] for r in got], ["mine"]) + self.assertEqual(len(approvals.listing(self.conn, communities=set(), user="bob")), 5) + + +class BackupTest(unittest.TestCase): + def test_a_daily_consistent_copy_keeping_a_week(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(conn.close) + db.migrate(conn) + with db.transaction(conn): + events.append(conn, "x", "s", events.WORKER) + out = os.path.join(d, "copies") + for day in range(10): + db.backup(conn, out, keep=7, now=86400 * (20000 + day)) + self.assertIsNone(db.backup(conn, out, keep=7, now=86400 * 20009)) # today's exists + files = sorted(os.listdir(out)) + self.assertEqual(len(files), 7) + copy = db.connect(os.path.join(out, files[-1])) + self.assertEqual(events.verify(copy), (1, None)) + copy.close() + + +class ApiTest(unittest.TestCase): + def setUp(self): + for name in ("settings", "credstore", "preapprovals"): + self.addCleanup(setattr, main, name, getattr(main, name)) + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + c = db.connect(self.path) + db.migrate(c) + self.conn = c + self.addCleanup(c.close) + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol", + "BITS_ORCH_DB": self.path, "BITS_WEBAUTHN_RP_ID": tp.RP_ID, + "BITS_WEBAUTHN_ORIGIN": tp.ORIGIN, + "BITS_WEBAUTHN_CREDENTIALS": os.path.join(self.dir, "creds.json")}) + main.credstore = main.credentials.CredentialStore(main.settings.credentials_path) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + self.sched = "communities/lcg/pipelines/nightly.json" + + def _post(self, user, path, body): + return self.client.post(path, json=body, headers={"Authorization": "Bearer " + user}) + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer " + user}) + + def test_a_standing_request_is_checked_before_any_passkey(self): + body = {"schedule": self.sched, "sha256": "ab" * 32, "groups": ["lcg"], "runs": 5, "days": 30} + self.assertEqual(self._post("bob", "/approvals/standing/request", body).status_code, 403) + for bad in ({"days": 31}, {"runs": 0}, {"runs": True}, {"sha256": "xyz"}, + {"schedule": "communities/lcg/../x.json"}, {"groups": []}): + self.assertEqual(self._post("alice", "/approvals/standing/request", + dict(body, **bad)).status_code, 400, bad) + r = self._post("alice", "/approvals/standing/request", body) + self.assertEqual(r.status_code, 400) # no passkey enrolled + self.assertIn("passkey", r.json()["detail"]) + + def test_the_standing_flow_with_a_passkey(self): + from soft_webauthn import SoftWebauthnDevice + from console_backend import credentials, webauthn_rp + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol", "BITS_ORCH_DB": self.path, + "BITS_WEBAUTHN_RP_ID": tp.RP_ID, "BITS_WEBAUTHN_ORIGIN": tp.ORIGIN, + "BITS_WEBAUTHN_REQUIRE_UV": "0", + "BITS_WEBAUTHN_CREDENTIALS": os.path.join(self.dir, "creds.json")}) + main.credstore = credentials.CredentialStore(main.settings.credentials_path) + dev = SoftWebauthnDevice() + opts_json, chal = webauthn_rp.registration_options(main.settings, "alice", []) + main.credstore.add("alice", webauthn_rp.verify_registration( + main.settings, tp._attestation(dev, opts_json), chal)) + body = {"schedule": self.sched, "sha256": "ab" * 32, "groups": ["lcg"], "runs": 5, "days": 30} + rq = self._post("alice", "/approvals/standing/request", body) + self.assertEqual(rq.status_code, 200, rq.text) + rq = rq.json() + asr = tp._assertion(dev, rq["publicKey"]) + self.assertEqual(self._post("bob", "/approvals/standing/approve", + {"request_id": rq["request_id"], "assertion": asr}).status_code, 403) + ok = self._post("alice", "/approvals/standing/approve", + {"request_id": rq["request_id"], "assertion": asr}) + self.assertEqual(ok.status_code, 200, ok.text) + again = self._post("alice", "/approvals/standing/approve", + {"request_id": rq["request_id"], "assertion": asr}) + self.assertEqual(again.status_code, 400) # single use + listed = self._get("alice", "/approvals?kind=standing").json()["approvals"] + self.assertEqual([(a["runs_left"], a["binding"]["schedule"]) for a in listed], + [(5, self.sched)]) + self.assertEqual(self._get("bob", "/approvals").json()["approvals"], []) + aid = listed[0]["id"] + self.assertEqual(self._post("bob", "/approvals/%d/revoke" % aid, {}).status_code, 403) + self.assertEqual(self._post("alice", "/approvals/%d/revoke" % aid, {}).status_code, 200) + self.assertEqual(self._post("carol", "/approvals/%d/revoke" % aid, {}).status_code, 409) + self.assertEqual(self._get("alice", "/approvals").json()["approvals"], []) + self.assertEqual(len(self._get("carol", "/approvals?all=1").json()["approvals"]), 1) + (row,) = approvals.listing(self.conn, active_only=False) + self.assertIn("clientDataJSON", row["assertion"]) + + def test_an_unusable_store_is_503_on_the_sign_path(self): + main.preapprovals = approvals.Preapprovals(lambda: main.settings) + main.settings.orch_db = os.path.join(self.dir, "missing.db") + r = self.client.get("/preapproval/123") + self.assertEqual(r.status_code, 503) + + def test_a_groupless_preapproval_is_revoked_by_its_approver_or_overall_admins(self): + s = approvals.Preapprovals(lambda: main.settings) + s.approve("p9", {"groups": [], "user": "alice", "challenge": b"c"}) + aid = s.get("p9")["id"] + self.assertEqual(self._post("bob", "/approvals/%d/revoke" % aid, {}).status_code, 403) + self.assertNotIn("lcg", self._post("bob", "/approvals/%d/revoke" % aid, {}).text) + self.assertEqual([a["id"] for a in self._get("alice", "/approvals").json()["approvals"]], + [aid]) # its approver sees it + self.assertEqual(self._get("bob", "/approvals").json()["approvals"], []) + self.assertEqual(self._post("alice", "/approvals/%d/revoke" % aid, {}).status_code, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_build_queue.py b/console-backend/tests/test_orch_build_queue.py new file mode 100644 index 00000000..70fcac5d --- /dev/null +++ b/console-backend/tests/test_orch_build_queue.py @@ -0,0 +1,492 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The build queue: builds wait in the orchestrator and start one per build host.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, forge_ops, main +from console_backend.orch import approvals, build_queue as bq, db, replay + +import test_preapprove as tp + +HOSTS = [{"name": "arm", "tag": "bits-host-arm", "arch": ""}, + {"name": "bits", "tag": "bits-host-bits", "arch": "x86_64"}, + {"name": "pc", "tag": "bits-host-pc", "arch": "x86_64"}] + + +def _build(tag="bits-build-x86_64", pkg="ROOT", **more): + v = {"PACKAGE": pkg, "COMMUNITY": "lcg", "BITS_RUNNER_TAG": tag} + v.update(more) + return ("%s build" % pkg, [{"key": k, "value": x} for k, x in v.items()]) + + +class FakeForge: + def __init__(self): + self.started, self.next_id, self.fail, self.deleted = [], 100, None, [] + self.runners = [{"id": i, "description": d, "status": "online", "paused": False, "tags": t, + "runner_type": "project_type", "is_shared": False} + for i, d, t in ((1, "bits.cern.ch", ["self-hosted", "bits-host-bits", + "bits-build-x86_64"]), + (2, "pc-runner", ["bits-host-pc", "bits-build-x86_64"]), + (3, "", ["bits-host-arm"]))] + self.details = 0 + + def list_runners(self, tag=""): + if isinstance(self.runners, Exception): + raise self.runners + return [{k: v for k, v in r.items() if k != "tags"} # GitLab's list has no tags + for r in self.runners if not tag or tag in r["tags"]] + + def runner(self, rid): + self.details += 1 + return {"id": rid, "tag_list": next(r["tags"] for r in self.runners if r["id"] == rid)} + + def delete_pipeline(self, pid): + self.deleted.append(pid) + + def trigger_pipeline(self, ref, variables, name=None): + if self.fail: + e, self.fail = self.fail, None + if isinstance(e, Exception): + raise e + return e + self.next_id += 1 + self.started.append((self.next_id, {v["key"]: v["value"] for v in variables})) + return {"id": self.next_id, "status": "created"} + + +class _Queue(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.hosts = [dict(h) for h in HOSTS] + self.forge = FakeForge() + bq._named.clear() + bq._hosts_cache.clear() + bq._missing.clear() + + def _queue(self, *builds): + return bq.add(self.conn, "lcg", "alice", "main", list(builds))[1] + + def _pipeline(self, pid, outcome="running", tag="", jobs=(), token=None): + v = {"PACKAGE": "X", "BITS_RUNNER_TAG": tag} + if token is not None: + v[bq.ENTRY_VAR] = token + with db.transaction(self.conn): + self.conn.execute("INSERT OR REPLACE INTO pipelines (id, is_build, outcome, variables," + " jobs, first_seen_ms) VALUES (?, 1, ?, ?, ?, 0)", + (pid, outcome, json.dumps(v), json.dumps(list(jobs)))) + + def _unpause(self): + with db.transaction(self.conn): + self.conn.execute("DELETE FROM kv WHERE key = 'queue.pause'") + + def _states(self): + return [(r["id"], r["state"], r["host"], r["pipeline_id"]) + for r in self.conn.execute("SELECT * FROM build_queue ORDER BY id")] + + + +class QueueTest(_Queue): + def test_the_hosts_are_the_projects_runners_in_gitlab(self): + from console_backend import forge_ops + f = self.forge + f.runners += [{"id": 4, "description": "", "status": "online", "paused": False, + "tags": ["bits-host-shared"], "runner_type": "instance_type", "is_shared": True}, + {"id": 5, "description": "", "status": "online", "paused": False, + "tags": ["bits-host-", "docker"], "runner_type": "group_type", "is_shared": False}] + self.assertEqual(bq.hosts(f), HOSTS) # no shared runner, no bare tag + self.assertEqual(f.details, 4) + self.assertEqual(bq.hosts(f), HOSTS) # read again only after HOSTS_TTL_S + self.assertEqual(f.details, 4) + bq._hosts_cache.clear() + runner = f.runner + f.runner = lambda rid: (_ for _ in ()).throw(forge_ops.ForgeError(403, "x")) if rid == 2 \ + else runner(rid) + self.assertEqual([h["name"] for h in bq.hosts(f)], ["arm", "bits"]) # one it may not see + bq._hosts_cache.clear() + f.runner = lambda rid: (_ for _ in ()).throw(forge_ops.ForgeError(502, "x")) + self.assertIsNone(bq.hosts(f)) # GitLab down: cannot tell + bq.dispatch(self.conn, FakeForge(), self.hosts) + self.assertEqual(bq.last_hosts(self.conn), HOSTS) # what the worker then takes + bq.dispatch(self.conn, FakeForge(), []) # none now: none shown or kept + self.assertEqual(bq.last_hosts(self.conn), []) + + def test_a_host_whose_runners_disagree_takes_one_arch_and_says_so(self): + self.forge.runners.append({"id": 6, "description": "", "status": "online", "paused": False, + "tags": ["bits-host-pc", "bits-build-aarch64"], + "runner_type": "project_type", "is_shared": False}) + bq._warned.clear() + with self.assertLogs("bits_orchestrator.queue", "WARNING") as logs: + pc = [h for h in bq.hosts(self.forge) if h["name"] == "pc"][0] + self.assertEqual(pc["arch"], "x86_64") + self.assertIn("build host pc: its runners have the architectures aarch64, x86_64; it takes x86_64", + "\n".join(logs.output)) + + def test_one_build_per_host_oldest_first(self): + a, b, c = self._queue(_build(pkg="A"), _build(pkg="B"), _build(pkg="C")) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 2) + self.assertEqual(self._states(), [(a, "dispatched", "bits", 101), (b, "dispatched", "pc", 102), + (c, "queued", "", None)]) + started = dict(self.forge.started) + self.assertEqual((started[101]["BITS_RUNNER_TAG"], started[101][bq.ENTRY_VAR]), + ("bits-host-bits", bq.get(self.conn, a)["token"])) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) # not recorded yet + self._pipeline(101, "running") + self._pipeline(102, "running") + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self._pipeline(102, "failed") # pc is free + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self.assertEqual(self._states()[2], (c, "dispatched", "pc", 103)) + self.assertEqual([h["busy"] for h in bq.host_status(self.conn)], [False, True, False]) + + def test_a_pinned_build_waits_for_its_host_only(self): + a, b = self._queue(_build("bits-host-bits", pkg="A"), _build("bits-host-bits", pkg="B")) + c, = self._queue(_build(pkg="C")) + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertEqual([s[1:3] for s in self._states()], + [("dispatched", "bits"), ("queued", ""), ("dispatched", "pc")]) + + def test_a_pipeline_waiting_for_a_runner_keeps_a_host_free(self): + self._pipeline(1, tag="bits-build-x86_64") # it will take bits or pc + self._queue(_build("bits-host-bits", pkg="A"), _build("bits-host-pc", pkg="B")) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self._queue(_build("bits-host-arm", pkg="C")) # another arch (none): starts + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + + def test_pipelines_from_outside_the_queue_hold_hosts(self): + self._pipeline(1, tag="bits-host-bits") # pinned + self._pipeline(2, tag="bits-build-x86_64", + jobs=[{"status": "running", "runner": "pc-runner"}]) # on pc's runner + self._pipeline(3, outcome="success", tag="bits-host-bits") + self._queue(_build()) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self._pipeline(2, outcome="success") + self._pipeline(4, tag="bits-build-x86_64") # not on a runner yet + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self._pipeline(1, outcome="canceled") + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + + def test_a_job_on_a_runner_described_otherwise_holds_its_host(self): + from console_backend import forge_ops + self._pipeline(1, tag="bits-build-x86_64", + jobs=[{"status": "running", "runner": "bits.cern.ch"}]) # bits' runner + self._pipeline(2, jobs=[{"status": "running", "runner_id": 2}]) # pc's, by its id alone + self._queue(_build()) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self.forge.runners = forge_ops.ForgeError(403, "forbidden") # the last names stay + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self._pipeline(1, outcome="success") + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self.assertEqual(self._states()[0][1:3], ("dispatched", "bits")) + self.assertTrue([h["busy"] for h in bq.host_status(self.conn)][2]) # pc + + def test_a_host_whose_runner_is_offline_is_not_free(self): + from console_backend import forge_ops + self.forge.runners[1]["status"] = "offline" # pc's runner + a, b = self._queue(_build(pkg="A"), _build(pkg="B")) + c, = self._queue(_build("bits-host-pc", pkg="C")) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self.assertEqual([s[1:3] for s in self._states()], + [("dispatched", "bits"), ("queued", ""), ("queued", "")]) + self.assertEqual([(h["name"], h["offline"]) for h in bq.host_status(self.conn)], + [("arm", False), ("bits", False), ("pc", True)]) # bits.cern.ch: by its tag + self.forge.runners[1].update(status="online", paused=True) # paused: not free either + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self.assertEqual([(h["name"], h["offline"], h["paused"]) for h in bq.host_status(self.conn)], + [("arm", False, False), ("bits", False, False), ("pc", False, True)]) + self.forge.runners = forge_ops.ForgeError(403, "forbidden") # cannot tell: as before + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self.assertEqual(self._states()[1][1:3], ("dispatched", "pc")) + + def test_a_build_gitlab_dropped_unstarted_is_queued_again(self): + a, b = self._queue(_build("bits-host-bits", pkg="A"), _build("bits-host-pc", pkg="B")) + bq.dispatch(self.conn, self.forge, self.hosts) + stuck = [{"name": "build", "status": "failed", "failure_reason": "stuck_or_timeout_failure"}, + {"name": "publish", "status": "skipped"}] + self._pipeline(101, "failed", jobs=stuck) + self._pipeline(102, "failed", jobs=[{"status": "failed", "failure_reason": "script_failure", + "runner": "pc-runner", "started_at": "t"}]) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) # A again, on bits + self.assertEqual(self._states(), [(a, "dispatched", "bits", 103), (b, "dispatched", "pc", 102)]) + self.assertTrue(bq.dropped(stuck) and not bq.dropped([dict(stuck[0], runner="bits")])) + + def test_builds_no_host_can_take_wait_then_are_refused(self): + a, b, c = self._queue(_build("bits-host-gone"), _build("bits-build-aarch64"), + _build("bits-host-arm")) + with db.transaction(self.conn): # long queued: no matter + self.conn.execute("UPDATE build_queue SET created_ms = created_ms - ?", (3 * bq.HOST_GONE_MS,)) + bq.dispatch(self.conn, self.forge, self.hosts) # a runner may be coming back + self.assertEqual([s[1] for s in self._states()], ["queued", "queued", "dispatched"]) + self.assertIn("bits-host-gone is not a build host", bq.get(self.conn, a)["reason"]) + bq.dispatch(self.conn, self.forge, self.hosts) # waiting for a host is not missing one + for k in bq._missing: + bq._missing[k] -= bq.HOST_GONE_MS + 1 # an hour since they went missing + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertEqual([s[1] for s in self._states()], ["refused", "refused", "dispatched"]) + with self.assertRaises(ValueError): + bq.check_target([{"key": "BITS_RUNNER_TAG", "value": "self-hosted"}]) + with self.assertRaises(ValueError): + bq.check_target([{"key": "BITS_RUNNER_TAG", "value": "bits-host-"}]) + self.assertEqual(bq.check_target([{"key": "BITS_RUNNER_TAG", "value": "bits-host-a"}]), + "bits-host-a") + with self.assertRaises(RuntimeError): + self._queue(_build()) + bq.dispatch(self.conn, self.forge, []) + + def test_gitlab_refusing(self): + a, b = self._queue(_build(pkg="A"), _build(pkg="B")) + # The job limit: back in its place, not counted, the pass ends. + self.forge.fail = forge_ops.ForgeError(400, "Project exceeded the allowed number of jobs") + with self.assertRaises(bq.Stop): + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertEqual([(s[1], s[2]) for s in self._states()], [("queued", ""), ("queued", "")]) + self.assertEqual(bq.get(self.conn, a)["attempts"], 0) + self.assertIn("allowed number of jobs", bq.paused(self.conn)["why"]) + # Paused: nothing is tried for a while. + with self.assertRaises(bq.Stop): + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertEqual(self.forge.started, []) + self._unpause() + # Created failed, with no jobs: the same, and the dead pipeline goes. + self.forge.fail = {"id": 7, "status": "failed"} + with self.assertRaises(bq.Stop): + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertEqual((bq.get(self.conn, a)["state"], self.forge.deleted), ("queued", [7])) + self._unpause() + # Refused for this build: counted, the others go on; refused in the end. + for n in range(bq.MAX_ATTEMPTS): + self.forge.fail = forge_ops.ForgeError(400, "variables are invalid") + bq.dispatch(self.conn, self.forge, self.hosts) + with db.transaction(self.conn): + self.conn.execute("UPDATE build_queue SET state = 'canceled' WHERE id = ?" + " AND state = 'dispatched'", (b,)) + r = bq.get(self.conn, a) + self.assertEqual((r["state"], r["attempts"]), ("refused", bq.MAX_ATTEMPTS)) + self.assertIn("variables are invalid", r["reason"]) + + def test_gitlabs_answer_lost(self): + a, = self._queue(_build(pkg="A")) + self.forge.fail = forge_ops.ForgeError(0, "timed out") + with self.assertRaises(bq.Stop): + bq.dispatch(self.conn, self.forge, self.hosts) + r = bq.get(self.conn, a) + self.assertEqual((r["state"], r["host"]), ("dispatching", "bits")) + # It was created after all: recorded with this attempt's token, adopted. + self._pipeline(7, token="an older attempt's") + self._pipeline(8, token=r["token"]) + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 0) + self.assertEqual(self._states(), [(a, "dispatched", "bits", 8)]) + self.assertEqual(self.forge.started, []) + + def test_a_start_the_worker_did_not_finish(self): + a, b = self._queue(_build(pkg="A"), _build(pkg="B")) + with db.transaction(self.conn): + self.conn.execute("UPDATE build_queue SET state = 'dispatching', host = 'bits'," + " updated_ms = 0, token = 't' || id") + self._pipeline(55, token="t%d" % a) + # A's pipeline is adopted (on bits); B never started: queued again, then on pc. + self.assertEqual(bq.dispatch(self.conn, self.forge, self.hosts), 1) + self.assertEqual([s[1:] for s in self._states()], + [("dispatched", "bits", 55), ("dispatched", "pc", 101)]) + self.assertEqual(bq.get(self.conn, b)["attempts"], 1) + + def test_cancel_and_listing(self): + a, b = self._queue(_build(pkg="A"), _build(pkg="B")) + self.assertTrue(bq.cancel(self.conn, b, "alice")) + self.assertFalse(bq.cancel(self.conn, b, "alice")) + bq.dispatch(self.conn, self.forge, self.hosts) + self.assertFalse(bq.cancel(self.conn, a, "alice")) + c, d = self._queue(_build("bits-host-bits", pkg="C"), _build("bits-host-bits", pkg="D")) + rows = {r["id"]: r for r in bq.listing(self.conn)} + self.assertEqual((rows[c]["ahead"], rows[d]["ahead"], rows[a]["state"], rows[a]["package"]), + (0, 1, "dispatched", "A")) + self.assertEqual(bq.listing(self.conn, communities={"atlas"}), []) + + +class BatchApprovalTest(_Queue): + """A passkey approves a batch's certifying builds; each, as it starts, gets + the pre-approval of its pipeline the certifier reads.""" + + def _grant(self, ids, groups=("lcg",)): + rows = [bq.get(self.conn, i) for i in ids] + digests = [r["digest"] for r in rows if r["certify"]] + with db.transaction(self.conn): + got = approvals.grant_queue(self.conn, rows[0]["batch"], "lcg", list(groups), digests, + "alice", {"id": "cred"}, b"n" * 16) + return got and got[0] + + def test_each_started_build_is_preapproved(self): + ids = self._queue(_build(pkg="A", CERTIFY="true"), _build(pkg="B"), + _build(pkg="C", CERTIFY="true")) + aid = self._grant(ids) + self.assertIsNone(self._grant(ids)) # one per batch + self.assertTrue(approvals.verify_standing(approvals.get(self.conn, aid))) + then = approvals.preapprove_queued + bq.dispatch(self.conn, self.forge, self.hosts, then) # A on bits, B on pc + pre = replay.preapproval_at(self.conn, 101, db.now_ms()) + self.assertEqual((pre["approver"], json.loads(pre["groups"]), json.loads(pre["binding"])), + ("alice", ["lcg"], {"queue": bq.get(self.conn, ids[0])["batch"], + "entry": ids[0], "approval": aid})) + self.assertIsNone(replay.preapproval_at(self.conn, 102, db.now_ms())) # B certifies not + self.assertEqual(approvals.get(self.conn, aid)["runs_left"], 1) + self._pipeline(101, "success") + bq.dispatch(self.conn, self.forge, self.hosts, then) # C on bits + self.assertIsNotNone(replay.preapproval_at(self.conn, 103, db.now_ms())) + self.assertEqual(approvals.get(self.conn, aid)["runs_left"], 0) + + def test_a_dropped_build_gives_its_run_back(self): + a, = self._queue(_build("bits-host-bits", pkg="A", CERTIFY="true")) + aid = self._grant([a]) + then = approvals.preapprove_queued + bq.dispatch(self.conn, self.forge, self.hosts, then) + self.assertEqual(approvals.get(self.conn, aid)["runs_left"], 0) + self._pipeline(101, "failed", jobs=[{"status": "failed", + "failure_reason": "stuck_or_timeout_failure"}]) + bq.dispatch(self.conn, self.forge, self.hosts, then) # again, as 102 + self.assertIsNone(replay.preapproval_at(self.conn, 101, db.now_ms())) # a retry certifies not + self.assertIsNotNone(replay.preapproval_at(self.conn, 102, db.now_ms())) + self.assertEqual(approvals.get(self.conn, aid)["runs_left"], 0) + + def test_a_revoked_batch_preapproves_nothing_more(self): + ids = self._queue(_build(pkg="A", CERTIFY="true"), _build("bits-host-bits", pkg="B", + CERTIFY="true")) + aid = self._grant(ids) + bq.dispatch(self.conn, self.forge, self.hosts, approvals.preapprove_queued) # A + self.assertIsNotNone(replay.preapproval_at(self.conn, 101, db.now_ms())) + approvals.revoke(self.conn, aid, "alice") # A's goes with it + self.assertIsNone(replay.preapproval_at(self.conn, 101, db.now_ms())) + self._pipeline(101, "success") + bq.dispatch(self.conn, self.forge, self.hosts, approvals.preapprove_queued) # B + self.assertIsNone(replay.preapproval_at(self.conn, 102, db.now_ms())) + self.assertIsNone(bq.get(self.conn, ids[1])["approval_id"]) + + +class ApiTest(unittest.TestCase): + def setUp(self): + self.addCleanup(setattr, main, "settings", main.settings) + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + path = os.path.join(d, "orch.db") + c = db.connect(path) + db.migrate(c) + self.conn = c + self.addCleanup(c.close) + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": path, "BITS_ORCH_QUEUE": "1", + "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + + def _post(self, user, path, body): + return self.client.post(path, json=body, headers={"Authorization": "Bearer " + user}) + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer " + user}) + + def _body(self, **more): + name, variables = _build(**more) + return {"community": "lcg", "builds": [{"name": name, "variables": variables}]} + + def test_queue_list_cancel(self): + self.assertEqual(self._post("bob", "/ops/queue", self._body()).status_code, 403) + r = self._post("alice", "/ops/queue", self._body(BITS_CERTIFIER="mallory")) + self.assertEqual(r.status_code, 200, r.text) + eid, = r.json()["entries"] + v = {x["key"]: x["value"] for x in json.loads(bq.get(self.conn, eid)["variables"])} + self.assertEqual(v["BITS_CERTIFIER"], "alice") + listed = self._get("alice", "/ops/queue").json() + self.assertEqual([(e["id"], e["ahead"]) for e in listed["entries"]], [(eid, 0)]) + self.assertIsNone(listed["paused"]) + self.assertEqual(self._get("bob", "/ops/queue").json()["entries"], []) + with db.transaction(self.conn): # the hosts, for admins only + self.conn.execute("INSERT INTO kv (key, value) VALUES ('queue.hosts', '[{\"name\": \"h1\"}]')") + self.assertEqual(self._get("alice", "/ops/queue").json()["hosts"], [{"name": "h1"}]) + read = self._get("dave", "/ops/queue?community=lcg").json() # any signed-in user + self.assertEqual(([e["id"] for e in read["entries"]], read["hosts"]), ([eid], [])) + self.assertNotIn("requested_by", read["entries"][0]) + self.assertEqual(listed["entries"][0]["requested_by"], "alice") + self.assertEqual(self._get("dave", "/ops/queue").json()["entries"], []) + self.assertEqual(self._post("bob", "/ops/queue/%d/cancel" % eid, {}).status_code, 403) + self.assertEqual(self._post("carol", "/ops/queue/%d/cancel" % eid, {}).status_code, 200) + self.assertEqual(self._post("alice", "/ops/queue/%d/cancel" % eid, {}).status_code, 409) + self.assertTrue(self.client.get("/healthz").json()["build_queue"]) + dup = self._body() + dup["builds"][0]["variables"].append({"key": "PACKAGE", "value": "other"}) + self.assertEqual(self._post("alice", "/ops/queue", dup).status_code, 400) + + def test_ops_build_leaves_builds_to_the_queue(self): + main.settings.forge_ops_token, main.settings.forge_project = "t", "1" + name, variables = _build() + r = self._post("alice", "/ops/build", {"community": "lcg", "variables": variables}) + self.assertEqual(r.status_code, 409, r.text) + + def test_a_batch_approved_with_a_passkey(self): + from soft_webauthn import SoftWebauthnDevice + from console_backend import credentials, webauthn_rp + self.addCleanup(setattr, main, "credstore", main.credstore) + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": main.settings.orch_db, + "BITS_ORCH_QUEUE": "1", "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol", + "BITS_WEBAUTHN_RP_ID": tp.RP_ID, "BITS_WEBAUTHN_ORIGIN": tp.ORIGIN, + "BITS_WEBAUTHN_REQUIRE_UV": "0", + "BITS_WEBAUTHN_CREDENTIALS": os.path.join(d, "creds.json")}) + main.credstore = credentials.CredentialStore(main.settings.credentials_path) + body = dict(self._body(CERTIFY="true"), certify_groups=["lcg"]) + r = self._post("alice", "/ops/queue", body).json() # no passkey: queued all the same + self.assertEqual((len(r["entries"]), r["approval"], r["approval_error"]), + (1, None, "no passkey enrolled; enrol first")) + self.assertEqual(self._post("alice", "/ops/queue", dict(body, certify_groups=["atlas"]) + ).status_code, 403) + dev = SoftWebauthnDevice() + opts_json, chal = webauthn_rp.registration_options(main.settings, "alice", []) + main.credstore.add("alice", webauthn_rp.verify_registration( + main.settings, tp._attestation(dev, opts_json), chal)) + r = self._post("alice", "/ops/queue", body).json() + self.assertEqual((r["approval"]["builds"], r["approval"]["groups"]), (1, ["lcg"])) + eid, = r["entries"] + # Started before the passkey answered: pre-approved on approval. + with db.transaction(self.conn): + self.conn.execute("UPDATE build_queue SET state = 'dispatched', pipeline_id = 77" + " WHERE id = ?", (eid,)) + asr = tp._assertion(dev, r["approval"]["publicKey"]) + ok = {"request_id": r["approval"]["request_id"], "assertion": asr} + self.assertEqual(self._post("bob", "/ops/queue/approve", ok).status_code, 403) + got = self._post("alice", "/ops/queue/approve", ok) + self.assertEqual(got.status_code, 200, got.text) + self.assertEqual(got.json()["preapproved"], [77]) + self.assertEqual(self._post("alice", "/ops/queue/approve", ok).status_code, 400) + self.assertEqual(replay.preapproval_at(self.conn, 77, db.now_ms())["approver"], "alice") + # A batch's request is not a schedule's. + r2 = self._post("alice", "/ops/queue", body).json() + self.assertEqual(self._post("alice", "/approvals/standing/approve", + {"request_id": r2["approval"]["request_id"], + "assertion": asr}).status_code, 400) + # Without certify_groups, or no build that certifies: no challenge. + self.assertIsNone(self._post("alice", "/ops/queue", self._body()).json()["approval"]) + + def test_requests_are_checked(self): + for body in (self._body(tag="self-hosted"), self._body(pkg=""), + {"community": "lcg", "builds": []}, dict(self._body(), ref="evil"), + dict(self._body(), community="atlas")): + self.assertEqual(self._post("carol", "/ops/queue", body).status_code, 400, body) + main.settings.orch_queue = False + self.assertEqual(self._post("alice", "/ops/queue", self._body()).status_code, 503) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_canary.py b/console-backend/tests/test_orch_canary.py new file mode 100644 index 00000000..b2052082 --- /dev/null +++ b/console-backend/tests/test_orch_canary.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""M8a until M8b: the canary — the worker's certifications compared with the CI's.""" + +import json +import os +import tempfile + +from console_backend import config +from console_backend.orch import canary, certified, certifier, shadow + +from test_orch_certifier import CertifierBase +from test_orch_shadow import SETTINGS, _bom + + +class CanaryTest(CertifierBase): + def compared(self): + return {r["arch"]: r for r in canary.annotate(self.conn, certifier.listing(self.conn, 7))} + + def mine(self): + return canary.content_digest(json.loads(bytes(self.rows()["el9"]["manifest"]))) + + def test_the_canary_matches_the_ci(self): + """The CI certifies the same build on the commit the set was at: the + canary's manifest and the CI's are the same, timestamps aside.""" + self.approve() + self.certify() + self.gl.trees["aaaa000"] = {} + self.gl.trees["aaaa111"] = {"manifests/lcg/b2.el9.host-1.json": + dict(self.bom, certified_by=["bot"])} + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + shadow.discover(self.conn, self.gl, self.settings) + while shadow.compare_one(self.conn, self.gl, (self.s3, "bucket"), self.settings, + self.cache, trusted=self.trusted, + probe_factory=lambda c: self.probe, console=self.console): + pass + self.assertEqual(canary.match(self.conn, self.settings), 1) + row = self.compared()["el9"] + self.assertEqual((row["compare"], row["ci_certification"]), ("match", 5)) + row = self.rows()["el9"] + self.assertIsNone(row["manifest"]) # kept until compared + self.assertEqual(canary.summary(self.conn), {"match": 1}) + + def set_at(self, commit, pid, files): + """The set following one more bits-manifests commit, with *files*.""" + root = os.path.join(tempfile.mkdtemp(dir=self.dir), "manifests") + for path, doc in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, path)), exist_ok=True) + with open(os.path.join(root, path), "w") as fh: + json.dump(doc, fh) + os.makedirs(root, exist_ok=True) + certified.apply_tree(self.conn, root, pid, commit) + + def ci_output(self, content, before, cert=5, sha="aaaa111"): + self.conn.execute("INSERT OR IGNORE INTO certifications (pipeline_id, sha, created_ms," + " updated_ms) VALUES (?, ?, 0, 0)", (cert, sha)) + self.conn.execute("INSERT INTO ci_outputs (certification, pipeline_id, build_id, arch," + " content, sha, before_sha, created_ms) VALUES (?, 7, 'b2', 'el9', ?," + " ?, ?, 0)", (cert, content, sha, before)) + + def test_same_inputs_on_another_commit(self): + """Another certification merged first, for another architecture: the + CI's base is not the worker's set version, but its el9 inputs are.""" + self.approve() + self.certify() + self.set_at("bbbb000", 5, {"lcg/b9.el8.host-1.json": _bom("b9", "el8", ("q", "h9"))}) + self.ci_output(self.mine(), "bbbb000") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "match") + + def test_waits_for_the_set_to_take_in_the_cis_base(self): + self.approve() + self.certify() + self.ci_output(self.mine(), "bbbb000") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "") # not yet + self.set_at("bbbb000", 5, {}) + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "match") + + def test_a_base_the_set_never_takes_in(self): + self.approve() + self.certify() + self.ci_output(self.mine(), "bbbb000") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "") # may still come + certified._set_followed(self.conn, 5) # moved past it + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "unbased") + + def test_the_cis_merge_is_checked_too(self): + """The worker computed after the set took in the CI's merge, which also + replaced another el9 BOM: not the CI's inputs.""" + self.set_at("bbbb000", 5, {"lcg/b9.el9.host-1.json": _bom("b9", "el9", ("q", "h9"))}) + self.set_at("aaaa111", 6, {"lcg/b2.el9.host-1.json": dict(self.bom)}) + self.approve() + self.certify() + self.ci_output(self.mine(), "bbbb000", sha="aaaa111") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "rebased") + + def test_provenance_is_not_compared_and_old_digests_are_stale(self): + self.approve() + self.certify() + doc = json.loads(bytes(self.rows()["el9"]["manifest"])) + other = dict(doc, packages=[dict(e, built_by="ci@elsewhere", completed_at="2026-01-01T00:00:00Z") + for e in doc["packages"]]) + self.assertEqual(canary.content_digest(other), self.mine()) + self.ci_output("0" * 64, "aaaa000") # a v1 digest + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "stale") + self.assertIsNotNone(self.rows()["el9"]["manifest"]) # kept: not a match + + def test_rebased_unmatched_and_missing(self): + self.approve() + self.certify() + # The CI's base holds another el9 BOM the worker's set version did not. + self.set_at("bbbb000", 5, {"lcg/b9.el9.host-1.json": _bom("b9", "el9", ("q", "h9"))}) + self.conn.execute("INSERT INTO certifications (pipeline_id, sha, created_ms, updated_ms)" + " VALUES (5, 'aaaa111', 0, 0)") + self.conn.execute("INSERT INTO ci_outputs (certification, pipeline_id, build_id, arch," + " content, sha, before_sha, created_ms) VALUES (5, 7, 'b2', 'el9', 'x'," + " 'aaaa111', 'bbbb000', 0)") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "rebased") # another set version + self.conn.execute("INSERT INTO ci_outputs (certification, pipeline_id, build_id, arch," + " content, sha, before_sha, created_ms) VALUES (5, 8, 'b3', 'el9', 'y'," + " 'aaaa111', 'aaaa000', 0)") + self.conn.execute("DELETE FROM canary_compare") + self.conn.execute("UPDATE certify_runs SET created_ms = 0 WHERE arch = 'el9'") + self.conn.execute("UPDATE ci_outputs SET compare = 'rebased' WHERE build_id = 'b2'") + canary.match(self.conn, self.settings) + self.assertEqual(self.compared()["el9"]["compare"], "unmatched") + self.assertEqual(self.conn.execute("SELECT compare FROM ci_outputs WHERE build_id = 'b3'") + .fetchone()[0], "missing") + self.assertIn("canary.missing", self.kinds()) + + def test_the_ci_first_then_the_canary(self): + self.conn.execute("INSERT INTO certifications (pipeline_id, sha, created_ms, updated_ms)" + " VALUES (5, 'aaaa111', 0, 0)") + self.approve() + self.certify() + mine = self.mine() + self.conn.execute("INSERT INTO ci_outputs (certification, pipeline_id, build_id, arch," + " content, sha, before_sha, created_ms) VALUES (5, 7, 'b2', 'el9', ?," + " 'aaaa111', 'aaaa000', 0)", (mine,)) + self.assertEqual(canary.match(self.conn, self.settings), 1) + self.assertEqual(self.compared()["el9"]["compare"], "match") + + def test_two_ci_outputs_in_one_pass(self): + self.conn.execute("INSERT INTO certifications (pipeline_id, sha, created_ms, updated_ms)" + " VALUES (5, 'aaaa111', 0, 0), (6, 'aaaa111', 0, 0)") + self.approve() + self.certify() + mine = self.mine() + for c in (5, 6): + self.conn.execute("INSERT INTO ci_outputs (certification, pipeline_id, build_id, arch," + " content, sha, before_sha, created_ms) VALUES (?, 7, 'b2', 'el9', ?," + " 'aaaa111', 'aaaa000', 0)", (c, mine)) + self.assertEqual(canary.match(self.conn, self.settings), 1) + self.assertEqual([r[0] for r in self.conn.execute( + "SELECT compare FROM ci_outputs ORDER BY certification")], ["match", "again"]) + self.assertEqual(self.compared()["el9"]["ci_certification"], 5) + + def test_the_builds_listing_carries_the_workers_certification(self): + from console_backend.orch import api, reconcile + self.approve() + self.certify() + self.ci_output(self.mine(), "aaaa000") + canary.match(self.conn, self.settings) + rows = [{"pipeline": 7, "builds": [{"build_id": "b2"}, {"build_id": "b3"}]}] + got = api._with_certify(self.conn, rows, reasons=True)[0]["builds"] + self.assertEqual(got[0]["certify"]["state"], "signed") + self.assertEqual(got[0]["certify"]["approver"], "alice") + self.assertEqual([(a["arch"], a["state"], a["compare"], a["ci_certification"], a["written"]) + for a in got[0]["certify"]["archs"]], + [("el9", "signed", "match", 5, True)]) + self.assertNotIn("certify", got[1]) # never considered + self.assertEqual(api._with_certify(self.conn, []), []) + + def test_only_overall_admins_see_the_workers_reasons(self): + from unittest import mock + from console_backend.orch import api + self.approve() + self.certify() + rows = lambda: [{"pipeline": 7, "builds": [{"build_id": "b2"}]}] + c = api._with_certify(self.conn, rows())[0]["builds"][0]["certify"] + self.assertNotIn("reason", c) + self.assertTrue(all("reason" not in a for a in c["archs"])) + c = api._with_certify(self.conn, rows(), reasons=True)[0]["builds"][0]["certify"] + self.assertIn("reason", c) + with mock.patch.object(api.certifier, "for_builds", side_effect=RuntimeError("x")): + self.assertEqual(api._with_certify(self.conn, rows()), rows()) # rows still listed + + def test_the_cis_side_is_kept_only_while_the_worker_certifies(self): + off = config.Settings(env=dict(SETTINGS)) + self.assertEqual(canary.record_ci(self.conn, off, 5, "s", "b", self.dir, None, {}), 0) + diff --git a/console-backend/tests/test_orch_certified.py b/console-backend/tests/test_orch_certified.py new file mode 100644 index 00000000..60e46154 --- /dev/null +++ b/console-backend/tests/test_orch_certified.py @@ -0,0 +1,236 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M5: the certified set.""" + +import io +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, main +from console_backend.orch import certified, db, events, shadow + +from test_orch_shadow import FakeManifests, FakeS3, _Base, _bom + + +def _write_tree(root, files): + for path, doc in files.items(): + p = os.path.join(root, "manifests", path) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(doc if isinstance(doc, bytes) else json.dumps(doc).encode()) + return os.path.join(root, "manifests") + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.conn = db.connect(os.path.join(self.dir, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + def tree(self, name, files): + return _write_tree(os.path.join(self.dir, name), files) + + def kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + def live(self): + return {e["path"]: e for e in certified.listing(self.conn)["entries"]} + + +A = _bom("b1", "el9", ("p", "h1")) +B = _bom("b2", "el10", ("q", "h2")) +C = _bom("b3", "el9", ("r", "h3")) + + +class SetTest(_DB): + def test_import_then_follow(self): + v1 = self.tree("v1", {"lcg/a.json": A, "alice/b.json": B}) + self.assertEqual(certified.apply_tree(self.conn, v1, 10, "c1"), (1, 2, 0)) + live = self.live() + self.assertEqual(set(live), {"lcg/a.json", "alice/b.json"}) + self.assertEqual((live["lcg/a.json"]["group"], live["lcg/a.json"]["arch"], + live["lcg/a.json"]["build_id"]), ("lcg", "el9", "b1")) + added = [e for e in events.since(self.conn) if e["kind"] == "certified.added"] + self.assertTrue(all(e["payload"]["imported"] for e in added)) + self.assertEqual(added[0]["subject"], "certified:alice/el10") + self.assertIn("certified.imported", self.kinds()) + + changed = dict(A, extra=1) + v2 = self.tree("v2", {"lcg/a.json": changed, "lcg/c.json": C}) + self.assertEqual(certified.apply_tree(self.conn, v2, 11, "c2"), (2, 2, 2)) + self.assertEqual(set(self.live()), {"lcg/a.json", "lcg/c.json"}) + gone = certified.listing(self.conn, include_removed=True)["entries"] + self.assertEqual(len(gone), 4) # a and b removed, a again and c + removed = [e["payload"] for e in events.since(self.conn, limit=100) + if e["kind"] == "certified.removed"] + self.assertEqual(sorted(p["path"] for p in removed), ["alice/b.json", "lcg/a.json"]) + self.assertNotIn("imported", [e for e in events.since(self.conn, limit=100) + if e["kind"] == "certified.added"][-1]["payload"]) + self.assertEqual(certified.apply_tree(self.conn, v2, 12, "c3"), (3, 0, 0)) # no change + self.assertEqual(certified.current(self.conn), (3, 12, "c3")) + self.assertEqual(events.verify(self.conn)[1], None) + + def test_materialize_gives_each_versions_bytes_back(self): + v1 = self.tree("v1", {"lcg/a.json": A, "alice/b.json": B}) + certified.apply_tree(self.conn, v1, 10, "c1") + v2 = self.tree("v2", {"lcg/a.json": dict(A, extra=1)}) + certified.apply_tree(self.conn, v2, 11, "c2") + for version, src in ((1, v1), (2, v2)): + out = certified.materialize(self.conn, version, os.path.join(self.dir, "m%d" % version)) + self.assertEqual({k: shadow._read(p) for k, p in shadow._files(out).items()}, + {k: shadow._read(p) for k, p in shadow._files(src).items()}) + + def test_an_unsafe_path_is_refused(self): + v1 = self.tree("v1", {"lcg/a.json": A}) + certified.apply_tree(self.conn, v1, 10, "c1") + self.conn.execute("UPDATE certified SET path = '../../evil.json'") + with self.assertRaises(ValueError): + certified.materialize(self.conn, 1, os.path.join(self.dir, "m")) + + def test_a_bom_bits_cannot_certify_is_kept_and_flagged(self): + mixed = {"build_id": "bx", "packages": [ + {"hash": "h1", "tarball_sha256": "s", "effective_architecture": "el9"}, + {"hash": "h2", "tarball_sha256": "s", "effective_architecture": "el10"}]} + v1 = self.tree("v1", {"lcg/m.json": mixed, "lcg/x.json": b"{nope"}) + certified.apply_tree(self.conn, v1, 10, "c1") + live = self.live() + self.assertIn("mixes architectures", live["lcg/m.json"]["problem"]) + self.assertEqual(live["lcg/x.json"]["problem"], "not JSON") + + def test_the_store_copy_is_matched_ignoring_certified_by(self): + store_a = dict(A) + git_a = dict(A, certified_by=["alice"]) + s3 = FakeS3({"MANIFESTS/b1/host.el9.json": json.dumps(store_a).encode(), + "MANIFESTS/b1/host.el9.json.sig": b"{}", + "MANIFESTS/b2/host.el10.json": json.dumps(dict(B, extra=1)).encode()}) + v1 = self.tree("v1", {"lcg/a.json": git_a, "alice/b.json": B, "lcg/c.json": C}) + certified.apply_tree(self.conn, v1, 10, "c1") + self.assertEqual(certified.match_store(self.conn, (s3, "bucket")), 3) + live = self.live() + self.assertEqual((live["lcg/a.json"]["store_match"], live["lcg/a.json"]["store_key"]), + ("same", "MANIFESTS/b1/host.el9.json")) + self.assertEqual(live["alice/b.json"]["store_match"], "differs") + self.assertEqual(live["lcg/c.json"]["store_match"], "absent") + self.assertEqual(certified.match_store(self.conn, (s3, "bucket")), 0) + self.assertEqual(certified.listing(self.conn)["groups"], + {"lcg": {"same": 1, "absent": 1}, "alice": {"differs": 1}}) + + +class SyncTest(_Base): + def setUp(self): + super().setUp() + self.gl.trees["aaaa000"] = {"manifests/lcg/a.json": _bom("b1", "el9", ("p", "h1"))} + self.gl.trees["aaaa111"] = dict(self.gl.trees["aaaa000"], **{ + "manifests/lcg/b.json": _bom("b2", "el9", ("q", "h2"))}) + self.store = {("el9", "h1"): "sh1", ("el9", "h2"): "sh2"} + + def test_the_set_follows_pipelines_in_order(self): + self.gl.add(4, "aaaa000", "aaaa000") + self.gl.add(5, "aaaa111", "aaaa000") + self.gl.add(6, "deadbeef", "aaaa111") + shadow.discover(self.conn, self.gl, self.settings) + self.assertEqual(certified.sync(self.conn, self.gl, self.cache, limit=1), 1) + self.assertEqual(certified.current(self.conn)[1], 4) + self.gl.running = [6] # not past one still running + self.assertEqual(certified.sync(self.conn, self.gl, self.cache, branch="main"), 1) + self.assertEqual(certified.followed(self.conn), 5) + self.gl.running = [] + with self.assertLogs("bits_orchestrator.certified", "WARNING"): + for _ in range(certified.MAX_ATTEMPTS): # 6's commit is gone + self.assertEqual(certified.sync(self.conn, self.gl, self.cache), 0) + self.assertEqual((certified.followed(self.conn), certified.current(self.conn)[:2]), + (6, (2, 5))) + self.assertIsNone(certified.version_of(self.conn, 6)) + self.assertIn("certified.gap", self.kinds()) + self.assertEqual(set(self.live_paths()), {"lcg/a.json", "lcg/b.json"}) + v = certified.listing(self.conn) + self.assertEqual(v["digest"], certified.digest_of(certified.members(self.conn, 2))) + + def live_paths(self): + return [e["path"] for e in certified.listing(self.conn)["entries"]] + + def test_the_set_computes_what_the_tree_computes(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + self.run_shadow() + self.assertEqual(self.run_rows(5, "set")["el9"]["state"], "match") + self.assertEqual(self.run_rows(5, "set")["el9"]["manifest_sha256"], + self.run_rows(5)["el9"]["manifest_sha256"]) + self.assertEqual(shadow.summary(self.conn, mode="set"), {"match": 2}) # el9 and (tree) + self.assertEqual(self.run_rows(5, "set")["(tree)"]["state"], "match") + self.assertEqual(self.conn.execute("SELECT job_status FROM set_versions WHERE" + " pipeline_id = 5").fetchone()[0], "success") + + def test_a_set_that_differs_from_the_tree_is_a_mismatch(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + shadow.discover(self.conn, self.gl, self.settings) + certified.sync(self.conn, self.gl, self.cache) + self.conn.execute("DELETE FROM certified WHERE path = 'lcg/b.json'") # the set lost one + self.run_shadow(sync=False) + self.assertEqual(self.run_rows(5)["el9"]["state"], "match") + self.assertEqual(self.run_rows(5, "set")["el9"]["state"], "mismatch") + self.assertEqual(json.loads(self.run_rows(5, "set")["(tree)"]["diff"])["only_tree"], + ["lcg/b.json"]) + self.assertIn("shadow.set_mismatch", self.kinds()) + + def test_a_pipeline_the_set_skipped_says_so(self): + self.gl.add(6, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + shadow.discover(self.conn, self.gl, self.settings) + certified.sync(self.conn, self.gl, self.cache) + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) # found after 6 was synced + self.run_shadow() + self.assertEqual(self.run_rows(5, "set")["*"]["state"], "unsynced") + self.assertEqual(self.run_rows(6, "set")["el9"]["state"], "match") + + def test_comparisons_wait_for_the_set(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + shadow.discover(self.conn, self.gl, self.settings) + self.assertIsNone(shadow.compare_one(self.conn, self.gl, ("s3", "b"), self.settings, + self.cache, trusted=self.trusted, + probe_factory=lambda c: self.probe, upto=0)) + self.assertEqual(self.cert(5)["state"], "pending") + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": os.path.join(self.dir, "orch.db"), + "BITS_ADMINS_POLICY": "lcg @alice\n* @carol"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + v1 = self.tree("v1", {"lcg/a.json": A, "alice/b.json": B}) + certified.apply_tree(self.conn, v1, 10, "c1") + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer %s" % user}) + + def test_the_set_by_group(self): + paths = lambda u, q="": [e["path"] for e in self._get(u, "/ops/certified" + q).json()["entries"]] # noqa: E731 + self.assertEqual(sorted(paths("carol")), ["alice/b.json", "lcg/a.json"]) + self.assertEqual(paths("alice"), ["lcg/a.json"]) + self.assertEqual(paths("carol", "?group=alice"), ["alice/b.json"]) + self.assertEqual(paths("dave"), []) + r = self._get("carol", "/ops/certified") + self.assertEqual((r.json()["version"], r.json()["commit"]), (1, "c1")) + self.assertEqual(self.client.get("/ops/certified").status_code, 401) + + def test_one_bom_by_sha256(self): + sha = self._get("carol", "/ops/certified?group=alice").json()["entries"][0]["bom_sha256"] + r = self._get("carol", "/ops/certified/bom/" + sha) + self.assertEqual((r.status_code, r.json()["build_id"]), (200, "b2")) + self.assertEqual(self._get("alice", "/ops/certified/bom/" + sha).status_code, 404) + self.assertEqual(self._get("carol", "/ops/certified/bom/xyz").status_code, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_certifier.py b/console-backend/tests/test_orch_certifier.py new file mode 100644 index 00000000..57025a69 --- /dev/null +++ b/console-backend/tests/test_orch_certifier.py @@ -0,0 +1,1097 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""M8a: the worker certifies, signs and writes builds itself.""" + +import hashlib +import json +import os +import tempfile +from unittest.mock import patch + +from bits_helpers import trust +from console_backend import config +from console_backend.orch import actions, certified, certifier, db, events + +from test_orch_replay import FakeConsole, FakeS3 +from test_orch_shadow import SETTINGS, _Base, _bom, _pkg + + +class FakeWriter: + def __init__(self, fail=None): + self.puts, self.fail = [], fail + + def put_object(self, Bucket, Key, Body, ContentType=None): + if self.fail: + e, self.fail = self.fail, None + raise e + self.puts.append((Key, Body)) + + +class Deps: + def __init__(self, test): + self.t = test + self.conn, self.settings = test.conn, test.settings + self.sign = lambda body: trust.sign_bytes(body, trust.load_private_key(test.key)) + self.writer, self.probe, self.key_id = test.writer, test.probe, test.kid + + def reader(self): + return self.t.console + + def store(self): + return (self.t.s3, "bucket") + + +class CertifierBase(_Base): + def setUp(self): + super().setUp() + self.settings = config.Settings(env=dict( + SETTINGS, BITS_ADMINS_POLICY="lcg @alice\n* @carol", BITS_ORCH_CERTIFY="1", + BITS_ORCH_MANIFESTS_PROJECT="buncic/bits-manifests")) + self.s3, self.console, self.writer = FakeS3(), FakeConsole(), FakeWriter() + self.kid = next(iter(self.trusted)) + p = patch.object(trust, "load_trusted_keys", lambda dirs=None: self.trusted) + p.start() + self.addCleanup(p.stop) + self.bom = _bom("b2", "el9", ("q", "h2")) # source_pipeline_id 7 + self.store = {("el9", "h2"): "sh2"} + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = dict(self.bom) + # The set, imported from bits-manifests at aaaa000 (version 1). + empty = tempfile.mkdtemp(dir=self.dir) + os.makedirs(os.path.join(empty, "manifests")) + certified.apply_tree(self.conn, os.path.join(empty, "manifests"), 4, "aaaa000") + now = db.now_ms() + self.conn.execute("INSERT INTO kv (key, value) VALUES ('certify.since', ?)", (str(now - 1),)) + self.conn.execute( + "INSERT INTO pipelines (id, is_build, community, sha, variables, outcome," + " first_seen_ms) VALUES (7, 1, 'lcg', 'c0ffee', ?, 'success', ?)", + (json.dumps({"COMMUNITY": "LCG", "CERTIFY": "true"}), now)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, recorded_ms)" + " VALUES (7, 'b2', '[\"MANIFESTS/b2/host-1.el9.json\"]', ?)", (now,)) + self.deps = Deps(self) + + def approve(self, user="alice", subject="7", **kw): + now = db.now_ms() + cur = self.conn.execute( + "INSERT INTO approvals (kind, subject, groups, approver, created_ms, expires_ms," + " revoked_ms) VALUES ('preapproval', ?, '[\"lcg\"]', ?, ?, ?, ?)", + (subject, user, kw.get("created", now - 1000), kw.get("expires", now + 3600_000), + kw.get("revoked"))) + return cur.lastrowid + + def certify(self): + certifier.trigger(self.conn, self.settings, self.console) + return actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + + def rows(self): + return {r["arch"]: r for r in self.conn.execute( + "SELECT * FROM certify_runs WHERE pipeline_id = 7 AND build_id = 'b2'")} + + +class CertifierTest(CertifierBase): + def test_a_preapproved_build_is_certified_signed_and_written(self): + self.approve() + self.assertEqual(self.certify(), 1) + rows = self.rows() + self.assertEqual({a: r["state"] for a, r in rows.items()}, {"*": "signed", "el9": "signed"}) + keys = [k for k, _b in self.writer.puts] + self.assertEqual(keys, ["CANARY/common-manifest-el9.json", + "CANARY/common-manifest-el9.json.sig"]) # .json first + body, sig = self.writer.puts[0][1], json.loads(self.writer.puts[1][1]) + self.assertEqual(trust.verify_bytes(body, sig, self.trusted), self.kid) + m = json.loads(body) + self.assertEqual((m["certified_by"], [p["hash"] for p in m["packages"]]), (["alice"], ["h2"])) + self.assertIn("expires", m) # the CI's 30 days, by default + self.assertNotIn("source_commit", m) + a = self.conn.execute("SELECT * FROM actions WHERE kind = 'certify'").fetchone() + self.assertEqual((a["state"], a["approved_by_id"], a["requested_by_kind"]), + ("done", "alice", "worker")) + self.assertIn("certify.signed", self.kinds()) + + def test_a_build_in_the_store_is_certified_while_its_pipeline_runs(self): + self.conn.execute("UPDATE pipelines SET outcome = 'running'") # publishing to CVMFS + self.approve() + self.assertEqual(self.certify(), 1) + self.assertEqual(self.rows()["*"]["state"], "signed") + self.conn.execute("DELETE FROM certify_runs") + self.conn.execute("DELETE FROM actions") + for outcome in ("failed", "canceled"): # not taken up then + self.conn.execute("UPDATE pipelines SET outcome = ?", (outcome,)) + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows(), {}) + + def test_a_waiting_build_is_refused_in_the_end_whatever_its_pipeline_did(self): + self.conn.execute("UPDATE pipelines SET outcome = 'running'") + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows()["*"]["state"], "waiting") + self.conn.execute("UPDATE pipelines SET outcome = 'failed'") # its CVMFS publish + self.conn.execute("UPDATE builds SET recorded_ms = recorded_ms - 2 * 86400000") + self.conn.execute("UPDATE kv SET value = '0' WHERE key = 'certify.since'") + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows()["*"]["state"], "refused") + + def test_nobody_approves_a_certification_in_the_console(self): + with self.assertRaises(actions.ActionError): + actions.request(self.conn, certifier.KIND, {"pipeline": "7", "build_id": "b2"}, + events.Actor("person", "carol"), "lcg", True) + # No pre-approval: the build waits, and no action exists to approve. + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows()["*"]["state"], "waiting") + self.assertIsNone(self.conn.execute("SELECT 1 FROM actions").fetchone()) + self.approve() + certifier.trigger(self.conn, self.settings, self.console) + a = self.conn.execute("SELECT * FROM actions").fetchone() + self.assertEqual(a["state"], "approved") + self.conn.execute("UPDATE actions SET state = 'proposed'") # even if it were + with self.assertRaises(actions.ActionError): + actions.approve(self.conn, a["id"], events.Actor("person", "carol"), True) + certifier.trigger(self.conn, self.settings, self.console) # no second action + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM actions").fetchone()[0], 1) + + def test_a_build_waits_a_day_for_its_preapproval(self): + self.conn.execute("UPDATE builds SET recorded_ms = recorded_ms - 2 * 86400000") + self.conn.execute("UPDATE kv SET value = '0' WHERE key = 'certify.since'") + certifier.trigger(self.conn, self.settings, self.console) + row = self.rows()["*"] + self.assertEqual((row["state"], row["reason"]), + ("refused", "no console pre-approval of pipeline 7")) + + def test_what_is_not_certified(self): + self.conn.execute("UPDATE pipelines SET variables = ?", + (json.dumps({"COMMUNITY": "LCG"}),)) + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows()["*"]["state"], "skipped") + self.conn.execute("DELETE FROM certify_runs") + self.conn.execute("UPDATE pipelines SET variables = ?", + (json.dumps({"COMMUNITY": "LCG", "CERTIFY": "true"}),)) + self.console.files[("communities/LCG/ui-config.yaml", "c0ffee")] = \ + b"certify_manifests_remote: https://gitlab.cern.ch/other/manifests\n" + self.approve() + certifier.trigger(self.conn, self.settings, self.console) + self.assertIn("other/manifests", self.rows()["*"]["reason"]) + off = config.Settings(env=dict(SETTINGS)) + self.assertEqual(certifier.trigger(self.conn, off, self.console), 0) + + def test_the_canary_prefix_stays_out_of_the_production_manifests(self): + bad = config.Settings(env=dict(SETTINGS, BITS_ORCH_CERTIFY="1", + BITS_ORCH_CANARY_PREFIX="MANIFESTS/canary")) + with self.assertRaises(RuntimeError): + certifier.trigger(self.conn, bad, self.console) + + def test_a_rerun_signs_the_same_bytes(self): + self.approve() + self.writer.fail = OSError("store writer gone") + self.certify() + a = self.conn.execute("SELECT * FROM actions").fetchone() + self.assertEqual(a["state"], "approved") # Retry: backing off + first = bytes(self.rows()["el9"]["manifest"]) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(self.rows()["el9"]["state"], "signed") + self.assertEqual(self.writer.puts[0][1], first) # computed once + + def test_refusals(self): + cases = [ + (lambda: self.approve("bob"), "bob (who pre-approved pipeline 7) is not an admin"), + (lambda: (self.approve(), self.s3.docs.__setitem__( + "MANIFESTS/b2/host-1.el9.json", dict(self.bom, source_pipeline_id=9))), + "is of pipeline 9, not 7"), + ] + for setup, why in cases: + with self.subTest(why=why): + for t in ("certify_runs", "actions", "approvals"): + self.conn.execute("DELETE FROM %s" % t) + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = dict(self.bom) + setup() + self.certify() + self.assertEqual(self.rows()["*"]["state"], "refused") + self.assertIn(why, self.rows()["*"]["reason"]) + self.assertEqual(self.writer.puts, []) + with patch.object(trust, "load_key_policy", lambda dirs=None: None): + for t in ("certify_runs", "actions", "approvals"): + self.conn.execute("DELETE FROM %s" % t) + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = dict(self.bom) + self.approve() + self.certify() + self.assertIn("fails closed", self.rows()["*"]["reason"]) + + def test_a_revoked_approval_stops_the_run(self): + aid = self.approve() + certifier.trigger(self.conn, self.settings, self.console) + self.conn.execute("UPDATE approvals SET revoked_ms = ? WHERE id = ?", (db.now_ms(), aid)) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(self.rows()["*"]["state"], "refused") + self.assertEqual(self.writer.puts, []) + + def test_the_daily_limit_pauses_certifying(self): + self.settings.orch_certify_daily_signs = 0 + self.approve() + with self.assertRaises(RuntimeError): + certifier.trigger(self.conn, self.settings, self.console) + + def test_a_rerun_after_the_approval_is_revoked_signs_nothing(self): + aid = self.approve() + self.writer.fail = OSError("store writer gone") + self.certify() # computed, signed + self.assertEqual(self.rows()["el9"]["state"], "signing") + self.assertIsNotNone(self.rows()["el9"]["signature"]) # kept at once + self.conn.execute("UPDATE approvals SET revoked_ms = ? WHERE id = ?", (db.now_ms(), aid)) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + rows = self.rows() + self.assertEqual((rows["*"]["state"], rows["el9"]["state"]), ("refused", "refused")) + self.assertIsNone(rows["el9"]["manifest"]) + self.assertEqual(self.writer.puts, []) + + def test_a_retried_write_reuses_the_signature(self): + self.approve() + signed = [] + sign = self.deps.sign + self.deps.sign = lambda body: signed.append(1) or sign(body) + self.writer.fail = OSError("store writer gone") + self.certify() + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual((len(signed), self.rows()["el9"]["state"]), (1, "signed")) + + def test_the_daily_limit_defers_a_run_without_counting_an_attempt(self): + self.approve() + certifier.trigger(self.conn, self.settings, self.console) + self.settings.orch_certify_daily_signs = 0 + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + a = self.conn.execute("SELECT * FROM actions").fetchone() + self.assertEqual((a["state"], a["attempts"]), ("approved", 0)) + self.assertGreater(a["not_before_ms"], db.now_ms() + 3000_000) + self.assertNotIn("*", {k for k, r in self.rows().items() if r["state"] == "refused"}) + self.assertIn("action.deferred", self.kinds()) + + def test_nothing_is_signed_while_the_outside_copy_is_not_in_step(self): + self.approve() + self.settings.orch_audit_url = "http://audit-log:8443" + signed = [] + sign = self.deps.sign + self.deps.sign = lambda body: signed.append(1) or sign(body) + self.certify() # never checked yet + a = self.conn.execute("SELECT * FROM actions").fetchone() + self.assertEqual((a["state"], a["attempts"], signed), ("approved", 0, [])) + self.assertEqual(self.rows()["el9"]["state"], "signing") + self.conn.execute("INSERT INTO kv (key, value) VALUES ('outside.head', ?)", + (json.dumps({"seq": 1, "hash": "x", "ms": db.now_ms()}),)) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual((len(signed), self.rows()["el9"]["state"]), (1, "signed")) + + def test_each_architectures_manifests_are_numbered(self): + self.approve() + self.certify() + m = json.loads(self.writer.puts[0][1]) + self.assertEqual((m["sequence"], m["issuer"]), (1, "bits-orchestrator/bucket")) + self.assertEqual(self.rows()["el9"]["sequence"], 1) + store = self.deps.store() + self.assertEqual(certifier.next_sequence(self.conn, self.settings, store, "el9"), 2) + self.assertEqual(certifier.next_sequence(self.conn, self.settings, store, "el10"), 1) + # A restored database continues after the one in the store. + self.s3.docs["CANARY/common-manifest-el9.json"] = {"sequence": 41} + self.assertEqual(certifier.next_sequence(self.conn, self.settings, store, "el9"), 42) + + def test_the_limit_counts_signatures_not_updates(self): + self.approve() + self.certify() + self.assertEqual(certifier.signed_today(self.conn), 1) + self.conn.execute("UPDATE certify_runs SET updated_ms = ?", (db.now_ms(),)) + self.assertEqual(certifier.signed_today(self.conn), 1) + + def test_an_existing_action_keeps_its_approval(self): + first = self.approve() + certifier.trigger(self.conn, self.settings, self.console) + self.conn.execute("UPDATE certify_runs SET state = 'waiting'") # looked at again + second = self.approve() + certifier.trigger(self.conn, self.settings, self.console) + rows = self.conn.execute("SELECT approval_id FROM actions").fetchall() + self.assertEqual([r["approval_id"] for r in rows], [first]) + self.assertNotEqual(first, second) + + def test_an_architecture_is_a_plain_name(self): + self.approve() + bad = dict(self.bom, effective_architecture="el9/../../../../tmp/zz", + packages=[dict(p, effective_architecture="el9/../../../../tmp/zz") + for p in self.bom["packages"]]) + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = bad + self.certify() + self.assertIn("not a plain name", self.rows()["*"]["reason"]) + self.assertEqual(self.writer.puts, []) + + def test_gitlab_failing_refuses_no_build(self): + from console_backend import forge_ops + self.approve() + raw = self.console.file_raw + self.console.file_raw = lambda *a, **k: (_ for _ in ()).throw(forge_ops.ForgeError(401, "x")) + with self.assertRaises(forge_ops.ForgeError): # the step fails + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows(), {}) + self.console.file_raw = lambda *a, **k: (_ for _ in ()).throw(forge_ops.ForgeError(404, "x")) + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.rows()["*"]["state"], "waiting") # looked at again + self.assertIn("404", self.rows()["*"]["reason"]) + self.console.file_raw = raw + self.assertEqual(self.certify(), 1) + self.assertEqual(self.rows()["*"]["state"], "signed") + + def test_a_preapproved_build_is_not_hidden_by_waiting_ones(self): + now = db.now_ms() + self.conn.execute("UPDATE kv SET value = '0' WHERE key = 'certify.since'") + self.conn.execute( + "INSERT INTO pipelines (id, is_build, community, sha, variables, outcome," + " first_seen_ms) VALUES (8, 1, 'lcg', 'c0ffee', ?, 'success', ?)", + (json.dumps({"COMMUNITY": "LCG", "CERTIFY": "true"}), now)) + for i in range(3): # older, never pre-approved + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, recorded_ms)" + " VALUES (8, ?, '[]', ?)", ("a%d" % i, now - 1000 + i)) + with patch.object(certifier, "PER_PASS", 1): + for _ in range(4): + certifier.trigger(self.conn, self.settings, self.console) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM certify_runs WHERE" + " state = 'waiting'").fetchone()[0], 4) + self.approve() + self.assertEqual(certifier.trigger(self.conn, self.settings, self.console), 1) + self.assertEqual(self.rows()["*"]["state"], "signing") + + def test_certified_manifests_expire(self): + for v in ("0", "-1", "nan", "inf", "1e9"): + with self.assertRaises(ValueError): + config.Settings(env=dict(SETTINGS, BITS_ORCH_CERTIFY_VALID_DAYS=v)) + + def test_the_console_cannot_request_a_certification(self): + from fastapi.testclient import TestClient + from console_backend import main + main.settings = config.Settings(env={"GITLAB_API_URL": "https://gl/api/v4", + "BITS_ORCH_DB": self.path, + "BITS_ADMINS_POLICY": "* @carol"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + r = TestClient(main.app).post("/ops/actions", headers={"Authorization": "Bearer carol"}, + json={"kind": "certify", + "params": {"pipeline": "7", "build_id": "b2"}}) + self.assertEqual(r.status_code, 400) + self.assertIn("not requested by a person", r.json()["detail"]) + + +class RawS3(FakeS3): + """The store, keeping what the worker writes as the exact bytes.""" + + def get_object(self, Bucket, Key): + import io + if isinstance(self.docs.get(Key), bytes): + return {"Body": io.BytesIO(self.docs[Key])} + return super().get_object(Bucket, Key) + + +class StoreWriter(FakeWriter): + def __init__(self, s3): + super().__init__() + self.s3 = s3 + + def put_object(self, Bucket, Key, Body, ContentType=None): + super().put_object(Bucket, Key, Body, ContentType) + self.s3.docs[Key] = Body + + +class _StoreBase(CertifierBase): + def setUp(self): + super().setUp() + docs = self.s3.docs + self.s3 = RawS3() + self.s3.docs.update(docs) + self.writer = self.deps.writer = StoreWriter(self.s3) + + def set_at(self, commit, pid, files): + root = os.path.join(tempfile.mkdtemp(dir=self.dir), "manifests") + os.makedirs(root, exist_ok=True) + for path, doc in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, path)), exist_ok=True) + with open(os.path.join(root, path), "w") as fh: + json.dump(doc, fh) + certified.apply_tree(self.conn, root, pid, commit) + + def withdraw(self): + n = certifier.withdrawals(self.conn, self.settings) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + return n + + def written(self): + return json.loads(self.s3.docs["CANARY/common-manifest-el9.json"]) + + +class RecoveredTest(CertifierBase): + def test_a_build_decertified_after_the_restored_copy_needs_a_new_approval(self): + self.approve() + self.conn.execute("INSERT INTO kv (key, value) VALUES ('certify.decertified', ?)", + (json.dumps({"b2": db.now_ms() + 1000}),)) + self.certify() + self.assertIn("a new build certifies it", self.rows()["*"]["reason"]) + self.assertEqual(self.writer.puts, []) + + +class NumberInPlaceTest(_StoreBase): + """What the store holds is never replaced by a lower number.""" + + def test_a_half_written_manifest_is_finished(self): + self.approve() + key = "%s-el9.json" % certifier.write_prefix(self.settings) + put = self.writer.put_object + + def body_only(Bucket, Key, Body, ContentType=None): # the .sig put fails once + if Key.endswith(".sig") and not getattr(self, "_failed", False): + self._failed = True + raise OSError("down") + return put(Bucket=Bucket, Key=Key, Body=Body, ContentType=ContentType) + self.writer.put_object = body_only + self.certify() + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + sig = json.loads(self.s3.docs[key + ".sig"]) + self.assertEqual(trust.verify_bytes(self.s3.docs[key], sig, self.trusted), self.kid) + self.assertIsNotNone(self.rows()["el9"]["written_ms"]) + + def test_a_manifest_numbered_higher_in_the_store_is_not_overwritten(self): + self.approve() + self.writer.fail = OSError("down") # signed, not written + self.certify() + key = "%s-el9.json" % certifier.write_prefix(self.settings) + self.s3.docs[key] = json.dumps({"packages": [], "sequence": 50}).encode() + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(json.loads(self.s3.docs[key])["sequence"], 50) + self.assertIn("newer", self.rows()["el9"]["reason"]) + + def test_an_overtaken_certification_is_not_signed_in_vain(self): + self.settings.orch_owns_set = True + self.approve() + signs = [] + sign = self.deps.sign + self.deps.sign = lambda body: signs.append(1) or sign(body) + key = "%s-el9.json" % certifier.write_prefix(self.settings) + real_compute = certifier.compute + + def compute(*a, **kw): # another manifest lands meanwhile + out = real_compute(*a, **kw) + if not signs: + self.s3.docs[key] = json.dumps({"packages": [], "sequence": 7}).encode() + return out + with patch.object(certifier, "compute", compute): + self.certify() + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(len(signs), 1) # only the one written + self.assertEqual(json.loads(self.s3.docs[key])["sequence"], 8) + self.assertEqual(certifier.signed_today(self.conn), 1) + + + def test_a_half_written_certification_whose_approval_is_revoked_is_put_back(self): + self.approve() + self.certify() # b2 written + before = self.written() + now = db.now_ms() + self.s3.docs["MANIFESTS/b3/host-1.el9.json"] = dict(_bom("b3", "el9", ("r", "h3")), + source_pipeline_id=8) + self.store[("el9", "h3")] = "sh3" + self.conn.execute("INSERT INTO pipelines (id, is_build, community, sha, variables," + " outcome, first_seen_ms) VALUES (8, 1, 'lcg', 'c0ffee', ?," + " 'success', ?)", (json.dumps({"CERTIFY": "true"}), now)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, recorded_ms)" + " VALUES (8, 'b3', '[\"MANIFESTS/b3/host-1.el9.json\"]', ?)", (now,)) + aid = self.approve(subject="8") + put = self.writer.put_object + + def body_only(Bucket, Key, Body, ContentType=None): + if Key.endswith(".sig"): + raise OSError("down") + return put(Bucket, Key, Body, ContentType) + self.writer.put_object = body_only + self.certify() # b3's bytes, not its .sig + self.writer.put_object = put + self.conn.execute("UPDATE approvals SET revoked_ms = ? WHERE id = ?", (db.now_ms(), aid)) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + key = "%s-el9.json" % certifier.write_prefix(self.settings) + self.assertEqual(trust.verify_bytes(self.s3.docs[key], json.loads( + self.s3.docs[key + ".sig"]), self.trusted), self.kid) + self.assertEqual(self.written(), before) # without b3 + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 0) + + + def test_a_manifest_written_in_full_is_recorded_not_rolled_back(self): + self.approve() + self.certify() # sequence 1, written + row = self.rows()["el9"] + body = json.dumps(dict(json.loads(bytes(row["manifest"])), sequence=2)).encode() + sig = trust.sign_bytes(body, trust.load_private_key(self.key)) + key = "%s-el9.json" % certifier.write_prefix(self.settings) + self.s3.docs[key], self.s3.docs[key + ".sig"] = body, json.dumps(sig).encode() + self.conn.execute( # both put, its row not updated + "INSERT INTO certify_runs (pipeline_id, build_id, arch, state, signature, sequence," + " sha256, created_ms, updated_ms) VALUES (5, 'bc', 'el9', 'refused', ?, 2, ?, 1, 1)", + (json.dumps(sig), hashlib.sha256(body).hexdigest())) + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + self.assertEqual(self.s3.docs[key], body) + self.assertIsNotNone(self.conn.execute("SELECT written_ms FROM certify_runs WHERE" + " build_id = 'bc'").fetchone()[0]) + + + def test_only_the_manifest_whose_signature_is_in_place_is_put_back(self): + self.approve() + self.certify() # sequence 1, lists h2 + row = self.rows()["el9"] + m = json.dumps(dict(json.loads(bytes(row["manifest"])), sequence=2, packages=[])).encode() + key = "%s-el9.json" % certifier.write_prefix(self.settings) + self.s3.docs[key + ".sig"] = json.dumps(trust.sign_bytes( # its row lost in a restore + m, trust.load_private_key(self.key))).encode() + c = json.dumps({"packages": [], "sequence": 3}).encode() + self.s3.docs[key] = c # a certification's bytes over it + self.conn.execute( + "INSERT INTO certify_runs (pipeline_id, build_id, arch, state, signature, sequence," + " sha256, created_ms, updated_ms) VALUES (5, 'bc', 'el9', 'refused', '{}', 3, ?, 1, 1)", + (hashlib.sha256(c).hexdigest(),)) + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + self.assertEqual(self.s3.docs[key], c) # not sequence 1 again + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM actions WHERE" + " idempotency_key = 'repair:el9:3'").fetchone()[0], 1) + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 0) + + +class WithdrawTest(_StoreBase): + def test_a_bom_removed_from_the_set_is_withdrawn_without_a_preapproval(self): + self.approve() + self.certify() + self.assertEqual([p["hash"] for p in self.written()["packages"]], ["h2"]) + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 0) # nothing removed + self.set_at("cccc000", 9, {}) # b2 removed + self.assertEqual(self.withdraw(), 1) + m = self.written() + self.assertEqual((m["packages"], m["sequence"], m["withdrawn"]), ([], 2, {"set_version": 3})) + sig = json.loads(self.s3.docs["CANARY/common-manifest-el9.json.sig"]) + self.assertEqual(trust.verify_bytes(self.s3.docs["CANARY/common-manifest-el9.json"], sig, + self.trusted), self.kid) + row = self.conn.execute("SELECT * FROM certify_runs WHERE pipeline_id = 0").fetchone() + self.assertEqual((row["build_id"], row["state"], row["approver"]), ("set@3", "signed", "")) + self.assertEqual(self.withdraw(), 0) # done once + + def _withdraw_with_failing_puts(self, suffix): + self.approve() + self.certify() + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.set_at("cccc000", 9, {}) # b2 removed + put = self.writer.put_object + + def body_only(Bucket, Key, Body, ContentType=None): + if Key.endswith(suffix): + raise OSError("down") + return put(Bucket, Key, Body, ContentType) + self.writer.put_object = body_only + certifier.withdrawals(self.conn, self.settings) + for _ in range(actions.MAX_ATTEMPTS + 1): # retries spent + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + certifier.failed_actions(self.conn) + self.writer.put_object = put + + def test_a_half_written_manifest_whose_action_failed_is_repaired(self): + self._withdraw_with_failing_puts(".sig") + key = "CANARY/common-manifest-el9.json" + self.assertIsNone(trust.verify_bytes(self.s3.docs[key], json.loads( + self.s3.docs[key + ".sig"]), self.trusted)) # broken + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + self.assertEqual(trust.verify_bytes(self.s3.docs[key], json.loads( + self.s3.docs[key + ".sig"]), self.trusted), self.kid) + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 0) # done + + def test_a_failed_withdrawal_is_asked_for_again(self): + self._withdraw_with_failing_puts(".json") # nothing written + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 1) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertEqual(self.written()["packages"], []) + + def _overtaken_by_a_half_written_certification(self): + self.approve() + self.certify() # sequence 1, lists h2 + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.set_at("cccc000", 9, {}) # b2 removed + self.writer.fail = OSError("down") + self.withdraw() # sequence 2, not written + self.writer.fail = None + body = json.dumps({"packages": [], "sequence": 3}).encode() + self.s3.docs["CANARY/common-manifest-el9.json"] = body # a certification's bytes only + self.conn.execute( + "INSERT INTO certify_runs (pipeline_id, build_id, arch, state, signature, sequence," + " sha256, created_ms, updated_ms) VALUES (5, 'bc', 'el9', 'signing', '{}', 3, ?, ?, ?)", + (hashlib.sha256(body).hexdigest(), db.now_ms(), db.now_ms())) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.conn.execute("UPDATE certify_runs SET state = 'refused' WHERE build_id = 'bc'") + + def test_a_withdrawal_overtaken_by_a_half_written_certification_is_made_again(self): + self._overtaken_by_a_half_written_certification() + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + self.assertEqual(self.written()["sequence"], 1) # put back + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertEqual(self.written()["packages"], []) # and re-signed from the set + self.assertGreater(self.written()["sequence"], 3) + + def test_a_withdrawal_waits_for_a_half_written_manifest_to_be_repaired(self): + self._overtaken_by_a_half_written_certification() + self.writer.fail = OSError("down") # the put-back fails once + certifier.repair(self.conn, self.settings, self.deps.store(), self.writer) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertEqual(self.conn.execute("SELECT state FROM actions WHERE idempotency_key =" + " 'repair:el9:3'").fetchone()[0], "approved") + certifier.repair(self.conn, self.settings, self.deps.store(), self.writer) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertEqual(self.written()["packages"], []) + + def test_the_half_written_manifest_in_place_is_repaired_not_the_last_signed(self): + self.approve() + self.certify() # sequence 1, lists h2 + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.set_at("cccc000", 9, {}) # b2 removed + r1 = json.dumps({"packages": [], "sequence": 2}).encode() + self.s3.docs["CANARY/common-manifest-el9.json"] = r1 # its bytes only + for bid, seq, sha in (("r1", 2, hashlib.sha256(r1).hexdigest()), ("r2", 3, "f" * 64)): + self.conn.execute( # r2: signed, nothing put + "INSERT INTO certify_runs (pipeline_id, build_id, arch, state, signature," + " sequence, sha256, created_ms, updated_ms) VALUES (5, ?, 'el9', 'failed'," + " '{}', ?, ?, 1, 1)", (bid, seq, sha)) + certifier.withdrawals(self.conn, self.settings) + self.assertEqual(certifier.repair(self.conn, self.settings, self.deps.store(), + self.writer), 1) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertEqual(self.written()["packages"], []) + + def test_a_refused_withdrawal_asked_for_again_is_made(self): + self.approve() + self.certify() + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.set_at("cccc000", 9, {}) # b2 removed + with patch.object(trust, "load_key_policy", lambda dirs=None: None): # no policy, for now + self.withdraw() + row = self.conn.execute("SELECT * FROM certify_runs WHERE pipeline_id = 0").fetchone() + self.assertEqual(row["state"], "refused") + self.assertEqual(self.withdraw(), 1) # asked again + self.assertEqual(self.written()["packages"], []) + + def test_a_withdrawal_that_would_add_an_entry_is_refused(self): + self.approve() + self.certify() + other = dict(_bom("b9", "el9", ("z", "h9")), source_pipeline_id=9, certified_by=["alice"]) + self.store[("el9", "h9")] = "sh9" + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"]), + "lcg/b9.el9.host-1.json": other}) + self.set_at("cccc000", 9, {"lcg/b9.el9.host-1.json": other}) # b2 out, b9 stays + before = self.s3.docs["CANARY/common-manifest-el9.json"] + self.withdraw() + row = self.conn.execute("SELECT * FROM certify_runs WHERE pipeline_id = 0").fetchone() + self.assertEqual(row["state"], "refused") + self.assertIn("would add 1 entr", row["reason"]) + self.assertEqual(self.s3.docs["CANARY/common-manifest-el9.json"], before) + self.assertIn("withdraw.refused", self.kinds()) + + def test_a_build_not_yet_in_the_set_is_waited_for(self): + self.approve() + self.certify() # b2 written, not in the set yet + other = dict(_bom("b9", "el9", ("z", "h9")), source_pipeline_id=9, certified_by=["alice"]) + self.set_at("bbbb000", 8, {"lcg/b9.el9.host-1.json": other}) + self.set_at("cccc000", 9, {}) # b9 removed + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 0) + self.assertEqual(certifier.withdrawals(self.conn, self.settings, + now=db.now_ms() + certifier.WAIT_MS + 1), 1) + + def test_a_withdrawal_may_not_change_an_entry(self): + self.approve() + self.certify() + copy = dict(_bom("b8", "el9", ("q", "h2")), source_pipeline_id=8, certified_by=["alice"]) + copy["packages"][0]["redistributable"] = "none" # the same hash, other terms + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"]), + "lcg/b8.el9.host-1.json": copy}) + self.set_at("cccc000", 9, {"lcg/b8.el9.host-1.json": copy}) + self.withdraw() + row = self.conn.execute("SELECT * FROM certify_runs WHERE pipeline_id = 0").fetchone() + self.assertEqual(row["state"], "refused") + + def test_a_certification_in_progress_is_waited_for(self): + self.approve() + self.certify() + self.set_at("bbbb000", 8, {"lcg/b2.el9.host-1.json": dict(self.bom, certified_by=["alice"])}) + self.set_at("cccc000", 9, {}) + self.conn.execute("INSERT INTO certify_runs (pipeline_id, build_id, arch, state," + " created_ms, updated_ms) VALUES (11, 'b11', 'el9', 'signing', 0, 0)") + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 0) + + +class CliBuildTest(CertifierBase): + """A build made with the CLI, of no pipeline: certified by its CLI pre-approval.""" + BID = "key4hep-0123456789ab" + + def setUp(self): + super().setUp() + self.s3.docs["MANIFESTS/%s/host-1.el9.json" % self.BID] = dict( + _bom(self.BID, "el9", ("r", "h3")), source_pipeline_id=None) + self.store[("el9", "h3")] = "sh3" + + def cli_approve(self, packages=None, groups=("lcg",), user="alice"): + now = db.now_ms() + pk = packages if packages is not None else [["el9", "h3", "sh3", "r", "1"]] + self.conn.execute( + "INSERT INTO approvals (kind, subject, groups, approver, via, packages, created_ms," + " expires_ms) VALUES ('preapproval', ?, ?, ?, 'cli', ?, ?, ?)", + (self.BID, json.dumps(sorted(groups)), user, json.dumps(pk), now, now + 3600_000)) + + def cli_rows(self): + return {r["arch"]: r for r in self.conn.execute( + "SELECT * FROM certify_runs WHERE pipeline_id = 0 AND build_id = ?", (self.BID,))} + + def test_a_cli_build_is_certified_by_its_cli_preapproval(self): + self.cli_approve() + self.assertEqual(self.certify(), 1) + rows = self.cli_rows() + self.assertEqual({a: r["state"] for a, r in rows.items()}, {"*": "signed", "el9": "signed"}) + m = json.loads(self.writer.puts[0][1]) + self.assertEqual(([p["hash"] for p in m["packages"]], m["certified_by"]), (["h3"], ["alice"])) + a = self.conn.execute("SELECT * FROM actions WHERE kind = 'certify'").fetchone() + self.assertEqual((a["subject"], a["community"]), ("certify:cli:%s" % self.BID, "lcg")) + + def test_a_new_approval_signs_its_own_bytes(self): + self.cli_approve() + self.writer.fail = OSError("down") # signed, not written + self.certify() + self.conn.execute("UPDATE approvals SET expires_ms = 1") + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(self.cli_rows()["el9"]["state"], "refused") + self.writer.fail = None + self.cli_approve() # approved again + self.certify() + body, sig = self.writer.puts[-2][1], json.loads(self.writer.puts[-1][1]) + self.assertEqual(trust.verify_bytes(body, sig, self.trusted), self.kid) + + def test_a_build_decertified_after_a_restored_copy_needs_a_new_pre_approval(self): + self.cli_approve() + self.conn.execute("INSERT INTO kv (key, value) VALUES ('certify.decertified', ?)", + (json.dumps({self.BID: db.now_ms() + 1000}),)) + self.certify() + self.assertIn("a new pre-approval", self.cli_rows()["*"]["reason"]) + self.conn.execute("UPDATE kv SET value = ? WHERE key = 'certify.decertified'", + (json.dumps({self.BID: db.now_ms() - 1000}),)) + self.cli_approve() # approved again, after + self.certify() + self.assertEqual(self.cli_rows()["el9"]["state"], "signed") + + def test_store_boms_other_than_what_was_approved_wait_then_are_refused(self): + self.cli_approve(packages=[["el9", "h3", "other", "r", "1"]]) + self.certify() + self.assertEqual(self.cli_rows()["*"]["state"], "signing") # deferred: maybe not up yet + self.assertIn("action.deferred", self.kinds()) + self.assertEqual(self.writer.puts, []) + self.conn.execute("UPDATE approvals SET expires_ms = 1") + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + self.assertEqual(self.cli_rows()["*"]["state"], "refused") + + def test_the_approved_run_is_found_among_other_files(self): + d = self.s3.docs + d["MANIFESTS/%s/sbom.cdx.json" % self.BID] = {"bomFormat": "CycloneDX"} + d["MANIFESTS/%s/host-0.el9.json" % self.BID] = dict( # an earlier run + _bom(self.BID, "el9", ("r", "h4")), source_pipeline_id=None) + self.store[("el9", "h4")] = "sh4" + self.cli_approve() + self.certify() + self.assertEqual(self.cli_rows()["el9"]["state"], "signed") + m = json.loads(self.writer.puts[0][1]) + self.assertEqual([p["hash"] for p in m["packages"]], ["h3"]) + + def test_a_package_put_in_another_group_is_refused(self): + doc = self.s3.docs["MANIFESTS/%s/host-1.el9.json" % self.BID] + doc["packages"][0]["group"] = "ship" + self.cli_approve() + self.certify() + self.assertIn("not the approved lcg", self.cli_rows()["*"]["reason"]) + + def test_one_group_only(self): + self.cli_approve(groups=("lcg", "ship")) + self.certify() + self.assertEqual(self.cli_rows()["*"]["state"], "refused") + + +class OwnedSetTest(_StoreBase): + """M8b: the worker owns the set, writes MANIFESTS/ and is the only signer.""" + + def setUp(self): + super().setUp() + self.settings = self.deps.settings = config.Settings(env=dict( + SETTINGS, BITS_ADMINS_POLICY="lcg @alice\n* @carol", BITS_ORCH_CERTIFY="1", + BITS_ORCH_MANIFESTS_PROJECT="buncic/bits-manifests", BITS_ORCH_OWNS_SET="1")) + + def written(self): + return json.loads(self.s3.docs["MANIFESTS/common-manifest-el9.json"]) + + def test_a_certification_writes_manifests_and_adds_the_build_to_the_set(self): + self.approve() + self.certify() + m = self.written() + self.assertIn("expires", m) # bounds a node that remembers nothing + self.assertEqual((m["sequence"], sorted(m["event"])), (1, ["hash", "seq"])) + live = [r["path"] for r in self.conn.execute( + "SELECT path FROM certified WHERE removed_version IS NULL")] + self.assertEqual(live, ["lcg/b2.el9.host-1.json"]) + v = self.conn.execute("SELECT * FROM set_versions ORDER BY version DESC").fetchone() + self.assertEqual((v["source"], v["added"]), ("orchestrator", 1)) + self.assertIsNone(self.rows()["el9"]["set_body"]) + + def test_decertify_takes_the_build_out_and_its_manifest_is_withdrawn(self): + self.approve() + self.certify() + row, _ = actions.request(self.conn, certifier.DECERTIFY, + {"group": "lcg", "build_id": "b2", "reason": "compliance"}, + events.Actor("person", "alice"), "lcg", True) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.DECERTIFY}) + self.assertEqual(actions.get(self.conn, row["id"])["state"], "done") + self.assertEqual(self.withdraw(), 1) + m = self.written() + self.assertEqual((m["packages"], m["sequence"]), ([], 2)) + + def test_decertify_in_another_group_is_refused(self): + self.approve() + self.certify() + row, _ = actions.request(self.conn, certifier.DECERTIFY, + {"group": "ship", "build_id": "b2"}, + events.Actor("person", "carol"), "ship", True) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.DECERTIFY}) + self.assertNotEqual(actions.get(self.conn, row["id"])["state"], "done") + self.assertEqual(len(self.conn.execute("SELECT * FROM certified WHERE removed_version" + " IS NULL").fetchall()), 1) + + def test_decertify_takes_out_only_its_groups_boms(self): + self.approve() + self.certify() + row = self.conn.execute("SELECT * FROM certified WHERE removed_version IS NULL").fetchone() + with db.transaction(self.conn): + certified.change(self.conn, add={"ship/b2.el9.host-1.json": self.conn.execute( + "SELECT body FROM boms WHERE sha256 = ?", (row["bom_sha256"],)).fetchone()[0]}) + self.assertEqual(len(certified.listing(self.conn, build_id="b2")["entries"]), 2) + self.assertEqual(certified.listing(self.conn, build_id="b9")["entries"], []) + res = certifier.run_decertify({"group": "LCG", "build_id": "b2"}, self.deps) + self.assertEqual(res.get("still_certified_by"), ["ship"]) + live = [r["path"] for r in self.conn.execute( + "SELECT path FROM certified WHERE removed_version IS NULL")] + self.assertEqual(live, ["ship/b2.el9.host-1.json"]) + + def test_a_failed_certification_holds_no_architecture_back(self): + self.approve() + self.writer.fail = OSError("down") + certifier.trigger(self.conn, self.settings, self.console) + for _ in range(actions.MAX_ATTEMPTS + 1): # retries spent + self.writer.fail = OSError("down") # (it fails once each time) + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.KIND}) + certifier.failed_actions(self.conn) + self.assertEqual({a: r["state"] for a, r in self.rows().items()}, + {"*": "failed", "el9": "failed"}) + other = dict(_bom("b9", "el9", ("z", "h9")), source_pipeline_id=9) + with db.transaction(self.conn): + certified.change(self.conn, add={"lcg/b9.el9.host-1.json": json.dumps(other).encode()}) + later = db.now_ms() + 11 * certifier.DAY_MS + self.assertEqual(certifier.withdrawals(self.conn, self.settings, now=later), 1) + + def test_a_certification_overtaken_by_a_withdrawal_is_made_again(self): + self.approve() + self.certify() # b2, sequence 1 + now = db.now_ms() + self.conn.execute("INSERT INTO pipelines (id, is_build, community, sha, variables, outcome," + " first_seen_ms) VALUES (8, 1, 'lcg', 'c0ffee', ?, 'success', ?)", + (json.dumps({"COMMUNITY": "LCG", "CERTIFY": "true"}), now)) + self.s3.docs["MANIFESTS/bx/host-1.el9.json"] = json.dumps( + dict(_bom("bx", "el9", ("x", "hx")), source_pipeline_id=8)).encode() + self.store[("el9", "hx")] = "shx" + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, recorded_ms)" + " VALUES (8, 'bx', '[\"MANIFESTS/bx/host-1.el9.json\"]', ?)", (now,)) + self.approve(subject="8") + certifier.trigger(self.conn, self.settings, self.console) # bx requested + actions.request(self.conn, certifier.DECERTIFY, {"group": "lcg", "build_id": "b2"}, + events.Actor("person", "alice"), "lcg", True) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.DECERTIFY}) + certifier.withdrawals(self.conn, self.settings) + self.writer.fail = OSError("down") # bx signs, its write fails + actions.run_due(self.conn, self.deps, lambda a, c: True, + kinds={certifier.KIND, certifier.WITHDRAW}) + self.writer.fail = None + for _ in range(3): # the withdrawal, then bx again + self.conn.execute("UPDATE actions SET not_before_ms = 0") + actions.run_due(self.conn, self.deps, lambda a, c: True, + kinds={certifier.KIND, certifier.WITHDRAW}) + m = self.written() + self.assertEqual([p["hash"] for p in m["packages"]], ["hx"]) + live = [r[0] for r in self.conn.execute( + "SELECT path FROM certified WHERE removed_version IS NULL")] + self.assertEqual(live, ["lcg/bx.el9.host-1.json"]) + later = db.now_ms() + 11 * certifier.DAY_MS + certifier.withdrawals(self.conn, self.settings, now=later) # a refresh + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + self.assertNotIn("withdraw.refused", self.kinds()) + + def test_decertify_package_replaces_each_bom_without_it_and_withdraws(self): + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = _bom("b2", "el9", ("q", "h2"), + ("Secret", "h5")) + self.store[("el9", "h5")] = "sh5" + self.approve() + self.certify() + self.assertEqual(sorted(p["package"] for p in self.written()["packages"]), ["Secret", "q"]) + row, _ = actions.request(self.conn, certifier.DECERTIFY_PACKAGE, + {"group": "lcg", "package": "secret", "reason": "compliance"}, + events.Actor("person", "alice"), "lcg", True) + actions.run_due(self.conn, self.deps, lambda a, c: True, + kinds={certifier.DECERTIFY_PACKAGE}) + done = actions.get(self.conn, row["id"]) + self.assertEqual(done["state"], "done") + live = self.conn.execute("SELECT c.path, b.body FROM certified c JOIN boms b ON" + " b.sha256 = c.bom_sha256 WHERE removed_version IS NULL").fetchall() + self.assertEqual([r["path"] for r in live], ["lcg/b2.el9.host-1.json"]) + bom = json.loads(live[0]["body"]) + self.assertEqual(([p["package"] for p in bom["packages"]], bom["decertified_packages"]), + (["q"], ["secret"])) + self.assertGreaterEqual(self.withdraw(), 1) # removal only: re-signed + m = self.written() + self.assertEqual([p["package"] for p in m["packages"]], ["q"]) + self.assertIn("withdrawn", m) + self.assertNotIn("withdraw.refused", self.kinds()) + again = certifier.run_decertify_package({"group": "lcg", "package": "Secret"}, self.deps) + self.assertEqual((again["version"], again["boms_replaced"]), (None, 0)) + # The same build certified again (a new pre-approval) leaves it out still. + (arch, body, _f), = certifier.compute(self.conn, self.settings, self.console, + self.deps.store(), 7, "b2", "alice", + probe=self.probe, key_id=self.kid) + self.assertEqual([p["package"] for p in json.loads(body)["packages"]], ["q"]) + + def test_decertify_package_says_which_groups_still_certify_it(self): + self.approve() + self.certify() + res = certifier.run_decertify_package({"group": "ship", "package": "q"}, self.deps) + self.assertEqual((res["version"], res["still_certified_by"]), (None, ["lcg"])) + + def test_decertify_package_where_another_group_certifies_the_object_too(self): + self.s3.docs["MANIFESTS/b2/host-1.el9.json"] = _bom("b2", "el9", ("q", "h2"), + ("Secret", "h5")) + self.store[("el9", "h5")] = "sh5" + self.approve() + self.certify() + ship = dict(_bom("b7", "el9", ("Secret", "h5")), certified_by=["carol"]) + with db.transaction(self.conn): # ship certified the object too + certified.change(self.conn, add={"ship/b7.el9.host-1.json": json.dumps(ship).encode()}) + before = {p["package"]: p["group"] for p in self.written()["packages"]} + self.assertEqual(before, {"q": "lcg", "Secret": "lcg"}) # first seen wins + certifier.run_decertify_package({"group": "lcg", "package": "Secret"}, self.deps) + self.withdraw() + self.assertNotIn("withdraw.refused", self.kinds()) + after = {p["package"]: p["group"] for p in self.written()["packages"]} + self.assertEqual(after, {"q": "lcg", "Secret": "ship"}) # ship still vouches + + def test_a_regroup_no_bom_of_that_group_certifies_is_refused(self): + key = "%s-el9.json" % certifier.write_prefix(self.settings) + old = json.dumps({"packages": [dict(_pkg("Secret", "h5"), group="lcg")], + "sequence": 1}).encode() + self.s3.docs[key] = old + self.s3.docs[key + ".sig"] = json.dumps( + trust.sign_bytes(old, trust.load_private_key(self.key))).encode() + new = json.dumps({"packages": [dict(_pkg("Secret", "h5"), group="ship")], + "sequence": 2}).encode() + why = certifier.removal_only(self.settings, self.deps.store(), "el9", new, self.conn) + self.assertIn("would add 1 entr", why) + + def test_decertify_package_takes_out_a_bom_left_with_nothing(self): + self.approve() + self.certify() + res = certifier.run_decertify_package({"group": "lcg", "package": "Q", "arch": "el9"}, + self.deps) + self.assertEqual((res["boms_removed"], res["builds"]), (1, ["b2"])) + self.assertIsNone(self.conn.execute("SELECT 1 FROM certified WHERE removed_version" + " IS NULL").fetchone()) + other = certifier.run_decertify_package({"group": "ship", "package": "q"}, self.deps) + self.assertEqual(other["version"], None) # another group's BOMs: untouched + + def test_decertify_package_waits_for_the_switch(self): + self.settings.orch_owns_set = False + with self.assertRaises(certifier.Refused): + certifier.run_decertify_package({"group": "lcg", "package": "q"}, self.deps) + + def test_the_mirror_commits_each_version_to_bits_manifests(self): + from console_backend.orch import mirror + + class Forge: + def __init__(self): + self.files, self.commits = {}, [] + + def file_exists(self, path, ref): + return path in self.files + + def file_raw(self, path, ref): + return self.files.get(path) + + def create_commit(self, branch, acts, message): + self.commits.append((branch, acts, message)) + for a in acts: + if a["action"] == "delete": + self.files.pop(a["file_path"], None) + else: + self.files[a["file_path"]] = a["content"].encode() + return {"id": "c%d" % len(self.commits)} + forge = Forge() + self.assertEqual(mirror.push(self.conn, self.settings, forge), 0) # nothing yet + self.approve() + self.certify() + self.assertEqual(mirror.push(self.conn, self.settings, forge), 1) + branch, acts, _msg = forge.commits[0] + self.assertEqual((branch, [(a["action"], a["file_path"]) for a in acts]), + ("main", [("create", "manifests/lcg/b2.el9.host-1.json")])) + self.assertEqual(self.conn.execute("SELECT commit_sha FROM set_versions ORDER BY" + " version DESC").fetchone()[0], "c1") + self.assertEqual(mirror.push(self.conn, self.settings, forge), 0) + # A crash after the commit, before it was recorded: pushed again, no commit. + self.conn.execute("UPDATE kv SET value = '1' WHERE key = 'mirror.version'") + self.assertEqual(mirror.push(self.conn, self.settings, forge), 1) + self.assertEqual(len(forge.commits), 1) + + def test_the_store_says_whose_manifests_these_are(self): + from console_backend.orch import mirror + store = self.deps.store() + self.assertTrue(mirror.write_marker(self.conn, self.settings, store, self.writer, + certifier.issuer(store), "MANIFESTS/common-manifest")) + m = json.loads(self.s3.docs["MANIFESTS/orchestrator.json"]) + self.assertEqual((m["issuer"], m["manifests"]), + ("bits-orchestrator/bucket", "MANIFESTS/common-manifest")) + self.assertFalse(mirror.write_marker(self.conn, self.settings, store, self.writer, + certifier.issuer(store), "MANIFESTS/common-manifest")) + + def test_decertify_waits_for_the_switch(self): + self.settings.orch_owns_set = False + with self.assertRaises(certifier.Refused): + certifier.run_decertify({"group": "lcg", "build_id": "b2"}, self.deps) + + def test_a_manifest_is_refreshed_well_before_it_expires(self): + self.assertEqual(certifier.withdrawals(self.conn, self.settings), 0) # the switch + self.approve() + self.certify() + now = db.now_ms() + self.assertEqual(certifier.withdrawals(self.conn, self.settings, now=now), 0) + later = now + 11 * certifier.DAY_MS # past a third of 30 days + self.assertEqual(certifier.withdrawals(self.conn, self.settings, now=later), 1) + actions.run_due(self.conn, self.deps, lambda a, c: True, kinds={certifier.WITHDRAW}) + m = self.written() + self.assertEqual(([p["hash"] for p in m["packages"]], m["sequence"]), (["h2"], 2)) + self.assertEqual(certifier.withdrawals(self.conn, self.settings, now=later), 0) diff --git a/console-backend/tests/test_orch_cvmfs_status.py b/console-backend/tests/test_orch_cvmfs_status.py new file mode 100644 index 00000000..98d3cb8b --- /dev/null +++ b/console-backend/tests/test_orch_cvmfs_status.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The worker's CVMFS status document (cvmfs-status.json in the package registry).""" + +import json +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import cvmfs_status, db +from test_orch_reconcile import FakeS3 + +ARCH = "x86_64-el9-gcc14-opt" + + +class FakeForge: + def __init__(self, doc=None): + self.files = {} if doc is None else {cvmfs_status.FILE: json.dumps(doc).encode()} + self.puts = 0 + + def generic_file(self, package, version, name): + assert (package, version) == (cvmfs_status.PACKAGE, cvmfs_status.VERSION) + return self.files.get(name) + + def put_generic_file(self, package, version, name, data): + self.files[name] = data + self.puts += 1 + + def doc(self): + return json.loads(self.files[cvmfs_status.FILE]) + + +def _bom(arch, pkgs, build_arch=ARCH): + return {"build_id": "b", "effective_architecture": arch, "architecture": build_arch, + "published_at": "2026-10-20T10:00:00Z", "requested_packages": ["LCG"], + "packages": [{"package": n, "version": v, "revision": r, "hash": n} + for n, v, r in pkgs]} + + +class CvmfsStatusTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.s3 = FakeS3({"MANIFESTS/b1/a.json": _bom(ARCH, [("LCG", "110", "2"), ("ROOT", "6", "1")]), + "MANIFESTS/b1/t.json": _bom("x86_64-el9-gcc14", [("GCC-Toolchain", "v14", "1")]), + "MANIFESTS/b2/a.json": _bom(ARCH, [("ROOT", "6", "1")])}) + self.store = (self.s3, "bkt") + + def _build(self, pid, bid, keys, outcome="success", publish="true", package="LCG", at=1000): + v = {"PACKAGE": package, "VERSION": "110", "PLATFORM": "x86_64-el9", "COMMUNITY": "LCG", + "PUBLISH_CVMFS": publish} + with db.transaction(self.conn): + self.conn.execute("INSERT OR IGNORE INTO pipelines (id, is_build, outcome, variables," + " first_seen_ms) VALUES (?, 1, ?, ?, 0)", + (pid, outcome, json.dumps(v))) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, published_at," + " recorded_ms) VALUES (?, ?, ?, ?, ?)", + (pid, bid, json.dumps(keys), "2026-10-20T10:00:00Z", at)) + + def test_a_record_per_published_build_once(self): + old = {"packages": [{"pkg_id": "ROOT-6-1-x86_64-el9-gcc13-opt", "package": "ROOT"}, + {"pkg_id": "LCG-110-2-%s" % ARCH, "package": "LCG", "status": ""}], + "updated_at": "2026-01-01T00:00:00Z"} + forge = FakeForge(old) + self._build(1, "b1", ["MANIFESTS/b1/a.json", "MANIFESTS/b1/t.json"]) + self._build(2, "b2", ["MANIFESTS/b2/a.json"], outcome="failed", at=1001) + self._build(3, "b2", ["MANIFESTS/b2/a.json"], publish="false", at=1002) + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 1) + doc = forge.doc() + self.assertEqual([p["pkg_id"] for p in doc["packages"]], + ["ROOT-6-1-x86_64-el9-gcc13-opt", "LCG-110-2-%s" % ARCH]) # kept, replaced + rec = doc["packages"][1] + self.assertEqual((rec["status"], rec["run_id"], rec["version"], rec["community"]), + ("published", "1", "110", "LCG")) + self.assertEqual(doc["updated_at"], "2026-10-20T10:00:00Z") + # Nothing new: no upload. + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 0) + self.assertEqual(forge.puts, 1) + + def test_the_hosts_arch_older_boms_record_is_not_the_builds(self): + self.s3.objects["MANIFESTS/b3/a.json"] = json.dumps( + _bom(ARCH, [("LCG", "110", "3")], build_arch="x86_64-el9")).encode() + forge = FakeForge() + self._build(1, "b3", ["MANIFESTS/b3/a.json"]) + cvmfs_status.step(self.conn, forge, self.store) + self.assertEqual(forge.doc()["packages"][0]["pkg_id"], "LCG-110-3-%s" % ARCH) + + def test_a_pipeline_retried_to_success_and_a_store_error(self): + forge = FakeForge() + self._build(1, "b1", ["MANIFESTS/b1/a.json"], outcome="failed") + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 0) + with db.transaction(self.conn): # retried: success, indexed again + self.conn.execute("UPDATE pipelines SET outcome = 'success', indexed_ms = 5000") + real = self.s3.get_object + + def flaky(Bucket, Key): + raise OSError("connection reset") + self.s3.get_object = flaky + with self.assertRaises(OSError): # not skipped: looked at again + cvmfs_status.step(self.conn, forge, self.store) + self.s3.get_object = real + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 1) + + def test_an_empty_registry_and_a_package_not_in_the_boms(self): + forge = FakeForge() + self._build(1, "b2", ["MANIFESTS/b2/a.json"]) # LCG is not in b2's BOMs + self._build(2, "b1", ["MANIFESTS/b1/a.json"], at=1001) + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 1) + self.assertEqual([p["pkg_id"] for p in forge.doc()["packages"]], ["LCG-110-2-%s" % ARCH]) + + def test_an_unreadable_document_is_never_overwritten(self): + forge = FakeForge() + forge.files[cvmfs_status.FILE] = b"" + self._build(1, "b1", ["MANIFESTS/b1/a.json"]) + with self.assertRaises(RuntimeError): + cvmfs_status.step(self.conn, forge, self.store) + self.assertEqual((forge.puts, forge.files[cvmfs_status.FILE]), (0, b"")) + # The cursor did not move: the build is written once the document is fixed. + forge.files.clear() + self.assertEqual(cvmfs_status.step(self.conn, forge, self.store), 1) + + +class GenericPackageTest(unittest.TestCase): + def test_upload_and_read_paths(self): + from unittest import mock + from console_backend import forge_ops + f = forge_ops.GitLabForge("https://gl/api/v4", "tok", "232657") + seen = [] + + class R: + status_code, text = 201, "" + with mock.patch.object(forge_ops.requests, "put", + lambda url, **kw: seen.append((url, kw["headers"], kw["data"])) or R()): + f.put_generic_file("cvmfs-status", "latest", "cvmfs-status.json", b"{}") + self.assertEqual(seen, [("https://gl/api/v4/projects/232657/packages/generic/cvmfs-status/" + "latest/cvmfs-status.json", {"PRIVATE-TOKEN": "tok"}, b"{}")]) + R.status_code = 403 + with mock.patch.object(forge_ops.requests, "put", lambda url, **kw: R()): + with self.assertRaises(forge_ops.ForgeError): + f.put_generic_file("cvmfs-status", "latest", "cvmfs-status.json", b"{}") + with mock.patch.object(f, "_raw", lambda path, n: path): + self.assertEqual(f.generic_file("cvmfs-status", "latest", "cvmfs-status.json"), + "/packages/generic/cvmfs-status/latest/cvmfs-status.json") diff --git a/console-backend/tests/test_orch_failures.py b/console-backend/tests/test_orch_failures.py new file mode 100644 index 00000000..d5fe4fe9 --- /dev/null +++ b/console-backend/tests/test_orch_failures.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M2: failure records, issues, signatures, backfill, API.""" + +import os +import shutil +import tempfile +import time +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, forge_ops, main +from console_backend.orch import db, events, failures, reconcile + +from test_orch_reconcile import FakeReader, _job + +ESC = "\x1b" + + +def _day(ago): + """The UTC date *ago* days back: fixtures stay inside the collector's windows.""" + return time.strftime("%Y-%m-%d", time.gmtime(time.time() - ago * 86400)) + + +DAY = _day(1) + + +def bits_failure(pkg="arrow", ver="23.0.1", path="/home/gitlab-runner/jobs/9911/sw/BUILD/a1b2c3d4e5/arrow", + error="flatbuffers/base.h:42:3: error: Non-compatible flatbuffers version included", + line=42): + """A job log as GitLab serves it: section markers, colours, bits' failure.""" + return ( + "section_start:1760000000:step_script\r" + ESC + "[0K" + ESC + "[0;m$ bits build ...\n" + + DAY + "T09:00:01Z Building arrow ...\n" + "export PRIVATE-TOKEN: glpat-AbCdEf0123456789xyz\n" + "cloning https://ci-bot:s3cr3tpass@gitlab.cern.ch/x/y.git\n" + + ESC + "[31m" + ESC + "[1mBUILD FAILED:" + ESC + "[0m %s@%s\n" % (pkg, ver) + + "=" * 70 + "\n\n" + "Log File:\n sw/BUILD/%s/log\n\n" % pkg + + "Error excerpt:\n" + " Matched error lines (last 2):\n" + " %s/src/%s:%d:3: %s\n" % (path, "gen.cc", line, error) + + " make[2]: *** [CMakeFiles/arrow.dir/build.make:%d: gen.o] Error 1\n" % (line * 3) + + " Last 3 lines of log:\n" + " [ 71%%] Built target thing at 10:%02d:11\n" % (line % 60) + + " make: *** [Makefile:146: all] Error 2\n" + " password=hunter2 token: abc123\n" + "\nEnvironment:\n" + " OS: el9\n" + " bits: 0.6 (bits@35b7773abc)\n" + "section_end:1760000100:step_script\r" + ESC + "[0K\n" + "ERROR: Job failed: exit code 1\n") + + +class LogReader(FakeReader): + def __init__(self): + super().__init__() + self.logs, self.artifacts = {}, {} + + def job_log_tail(self, job_id, max_bytes=1 << 20): + self.calls.append(("log", job_id)) + v = self.logs.get(job_id) + if isinstance(v, Exception): + raise v + return v.encode() if isinstance(v, str) else v + + def job_artifact_tail(self, job_id, name, max_bytes=1 << 20): + self.calls.append(("artifact", job_id, name)) + v = self.artifacts.get(job_id) + return v.encode() if isinstance(v, str) else v + + def list_pipelines_before(self, updated_before, status="failed", page=1, per_page=100): + self.calls.append(("before", updated_before, page)) + w = reconcile._when + got = sorted((p for p in self.pipelines.values() + if w(p["updated_at"]) < w(updated_before) and p["status"] == status), + key=lambda p: (w(p["updated_at"]), p["id"]), reverse=True) + return [dict(p) for p in got[(page - 1) * per_page: page * per_page]] + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + failures.seed_signatures(self.conn) + self.gl = LogReader() + + def _fail(self, pid, log, job_id=None, community="lcg", package="arrow", **kw): + jid = job_id or pid * 10 + self.gl.add(pid, status="failed", community=community, package=package, + updated=kw.pop("updated", DAY + "T10:%02d:00Z" % (pid % 60)), + jobs=[_job(jid, "bits-prepub-build", "failed", web_url="https://gl/j/%d" % jid, + failure_reason=kw.pop("reason", "script_failure"), + finished_at=DAY + "T11:00:00Z", + runner={"description": "bits-build-01"})]) + self.gl.logs[jid] = log + return jid + + def _pass(self): + reconcile.discover(self.conn, self.gl) + reconcile.settle(self.conn, self.gl) + return failures.collect(self.conn, self.gl) + + +class ParseTest(unittest.TestCase): + def test_bits_failure_blocks_are_found_and_cleaned(self): + lines = failures.clean(bits_failure().encode()) + self.assertFalse(any(ESC in ln or "section_" in ln for ln in lines)) + (pkg, ver, excerpt, bv), = failures.parse(lines) + self.assertEqual((pkg, ver, bv), ("arrow", "23.0.1", "0.6 (bits@35b7773abc)")) + self.assertIn("Non-compatible flatbuffers version", excerpt) + self.assertNotIn("Environment", excerpt) + self.assertNotIn("Log File", excerpt) + + def test_several_packages_and_a_repeated_summary(self): + log = bits_failure("ROOT", "6.32") + bits_failure("arrow") + bits_failure("ROOT", "6.32") + self.assertEqual(sorted(p for p, *_ in failures.parse(failures.clean(log))), ["ROOT", "arrow"]) + + def test_redaction(self): + text = failures.redact( + "PRIVATE-TOKEN: glpat-AbCdEf0123456789xyz\n" + "Authorization: Bearer eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2lnbmF0dXJlMTIz\n" + "https://user:pa55word@host/x ghp_abcdefghijklmnopqrstuvwxyz0123\n" + "AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG AKIAIOSFODNN7EXAMPLE\n" + "password = \"two words\" api_key: k-123\n" + "-----BEGIN RSA PRIVATE KEY-----\nMIIE\n-----END RSA PRIVATE KEY-----\n" + "error: undefined reference to `token_count'\n") + for secret in ("AbCdEf0123456789", "eyJhbGci", "pa55word", "abcdefghijklmnop", + "wJalrXUtnFEMI", "IOSFODNN7", "two words", "k-123", "MIIE"): + self.assertNotIn(secret, text, secret) + self.assertIn("https://@host/x", text) + self.assertIn("undefined reference to `token_count'", text) # code is left alone + + def test_redaction_of_what_ci_logs_really_carry(self): + lines = [ + "export PREPUB_API_TOKEN=abcd1234efgh", "BITS_FORGE_TOKEN=zzzz9999yyyy", + "CI_JOB_TOKEN=qwerty12345", "CI_REGISTRY_PASSWORD=Sup3rS3cret", "client_secret=xyzw7788", + 'data: {"token": "tok-77aa88bb", "password":"pw-9988"}', + "GET /api/v4/projects?private_token=ptk55667788&per_page=10", + "git clone https://oauth2:gitlabtoken99@gitlab.cern.ch/a/b.git", + "git clone https://plaintoken1234@github.com/a/b.git", + "bits login --token tk9988776655 --password=hunter2hunter2", + "AWS_ACCESS_KEY_ID=minioadmin12 aws_secret_access_key = 'ab cd ef gh'", + ] + out = failures.redact("\n".join(lines)) + for secret in ("abcd1234efgh", "zzzz9999yyyy", "qwerty12345", "Sup3rS3cret", "xyzw7788", + "tok-77aa88bb", "pw-9988", "ptk55667788", "gitlabtoken99", "plaintoken1234", + "tk9988776655", "hunter2hunter2", "minioadmin12", "ab cd ef gh"): + self.assertNotIn(secret, out, secret) + self.assertIn("per_page=10", out) + self.assertIn("[MASKED]", failures.redact("TOKEN=[MASKED]") + "[MASKED]") + + def test_bits_error_lines_with_their_log_prefix(self): + # bits' default formatter prefixes every line of an error with its level. + plain = bits_failure() + prefixed = "\n".join(("ERROR: " + ln) if i > 4 else ln + for i, ln in enumerate(plain.split("\n"))) + debug = "\n".join((DAY + "@09:00:01:ERROR:LCG:abc12345: " + ln) if i > 4 else ln + for i, ln in enumerate(plain.split("\n"))) + for log in (prefixed, debug): + (pkg, ver, excerpt, bv), = failures.parse(failures.clean(log)) + self.assertEqual((pkg, ver, bv), ("arrow", "23.0.1", "0.6 (bits@35b7773abc)")) + self.assertIn("Non-compatible flatbuffers version", excerpt) + self.assertEqual(failures.fingerprint("arrow", failures.redact(excerpt))[0], + failures.fingerprint("arrow", failures.redact( + failures.parse(failures.clean(plain))[0][2]))[0]) + + def test_a_failure_without_bits_output_is_recognised_across_runs(self): + def run(cid, job): + log = ("$ ./deploy.sh\nfatal: could not read from remote repository\n" + "Uploading artifacts for failed job\n" + "Uploading artifacts... bits-build.log: found 1 matching artifact files\n" + "Uploading artifacts to coordinator... 201 Created id=%d responseStatus=201" + " correlation_id=%s\nCleaning up project directory and file based variables\n" + "ERROR: Job failed: exit code 1\n" % (job, cid)) + ex = failures.last_lines(failures.clean(log)) + return failures.fingerprint("ROOT", ex) + a, b = run("01JABCDEFG1234567890XYZ", 111), run("01JZZZZZZZ9876543210QQQ", 222) + self.assertEqual(a[0], b[0]) + self.assertEqual(a[1][-1], "fatal: could not read from remote repository") + + def test_the_real_error_line_beats_make_noise(self): + ex = (" Matched error lines (last 6):\n src/a.cc:1:1: error: 'foo' was not declared\n" + + "".join(" make[%d]: *** [x%d] Error 2\n" % (i, i) for i in range(5))) + self.assertEqual(failures.key_lines(ex), ["src/a.cc:N:N: error: 'foo' was not declared"]) + + def test_the_same_error_elsewhere_has_the_same_fingerprint(self): + def fp(**kw): + (_, _, ex, _), = failures.parse(failures.clean(bits_failure(**kw))) + return failures.fingerprint("arrow", failures.redact(ex))[0] + same = fp(path="/home/gitlab-runner/jobs/123/sw/BUILD/ffff0000aaaa/arrow", line=43) + self.assertEqual(fp(), same) + self.assertNotEqual(fp(), fp(error="fatal error: zlib.h: No such file or directory")) + self.assertNotEqual(failures.fingerprint("arrow", "x: error: y")[0], + failures.fingerprint("ROOT", "x: error: y")[0]) + self.assertEqual(failures.fingerprint("x", "", "runner_system_failure")[1], + ["reason:runner_system_failure"]) + + +class CollectTest(_DB): + def test_a_failure_becomes_an_issue_named_by_its_signature(self): + self._fail(1, bits_failure()) + self.assertEqual(self._pass(), 1) + (f,) = failures.list_failures(self.conn) + self.assertEqual((f["package"], f["version"], f["community"], f["source"], f["runner"], + f["bits_version"], f["log_url"]), + ("arrow", "23.0.1", "lcg", "log", "bits-build-01", "0.6 (bits@35b7773abc)", + "https://gl/j/10")) + for secret in ("AbCdEf0123", "hunter2", "abc123"): + self.assertNotIn(secret, f["excerpt"]) + (i,) = failures.list_issues(self.conn) + self.assertEqual((i["count"], i["communities"], i["package"]), (1, ["lcg"], "arrow")) + self.assertEqual(i["signature"]["cause"].split()[0], "Headers") + kinds = [e["kind"] for e in events.since(self.conn, limit=100)] + self.assertEqual(kinds[-2:], ["issue.opened", "failure.recorded"]) + self.assertEqual(self._pass(), 0) # collected once + + def test_repeats_are_one_issue_across_communities(self): + self._fail(1, bits_failure()) + self._fail(2, bits_failure(path="/other/jobs/7/sw/BUILD/99887766aa/arrow", line=50), + community="atlas") + self._fail(3, bits_failure(error="fatal error: zlib.h: No such file or directory")) + self._pass() + issues = {i["count"]: i for i in failures.list_issues(self.conn)} + self.assertEqual(sorted(issues), [1, 2]) + self.assertEqual(issues[2]["communities"], ["atlas", "lcg"]) + self.assertIsNone(issues[1]["signature"]) + + def test_without_bits_output_the_last_lines_and_the_reason(self): + self._fail(1, "Preparing environment\nERROR: Job failed (system failure): pod lost\n", + reason="runner_system_failure") + self._pass() + (f,) = failures.list_failures(self.conn) + self.assertEqual(f["package"], "arrow") # the pipeline's own package + self.assertIn("pod lost", f["excerpt"]) + (i,) = failures.list_issues(self.conn) + self.assertEqual(i["signature"]["action"], "Retry on another host") + + def test_a_log_cut_at_gitlabs_limit_is_read_from_the_artifact(self): + jid = self._fail(1, "lots of output\nJob's log exceeded limit of 4194304 bytes.\n") + self.gl.artifacts[jid] = bits_failure() + self._pass() + (f,) = failures.list_failures(self.conn) + self.assertEqual((f["source"], f["version"]), ("artifact", "23.0.1")) + self.assertIn(("artifact", jid, "bits-build.log"), self.gl.calls) + + def test_an_unreadable_artifact_falls_back_to_the_log(self): + jid = self._fail(1, "lots of output\nJob's log exceeded limit of 4194304 bytes.\n") + self.gl.artifacts[jid] = forge_ops.ForgeError(403, "forbidden") + real = self.gl.job_artifact_tail + + def art(job_id, name, max_bytes=1 << 20): + v = self.gl.artifacts.get(job_id) + if isinstance(v, Exception): + raise v + return real(job_id, name, max_bytes) + self.gl.job_artifact_tail = art + with self.assertLogs("bits_orchestrator", "WARNING"): + self.assertEqual(self._pass(), 1) + (f,) = failures.list_failures(self.conn) + self.assertEqual(f["source"], "log") + + def test_an_empty_log_is_source_none(self): + self._fail(1, "") + self._pass() + self.assertEqual(failures.list_failures(self.conn)[0]["source"], "none") + + def test_a_failed_pipeline_with_no_failed_job_is_still_recorded(self): + self.gl.add(1, status="failed", community="lcg", package="ROOT", jobs=[]) + self._pass() + (f,) = failures.list_failures(self.conn) + self.assertEqual((f["job"], f["failure_reason"], f["source"]), (-1, "pipeline:failed", "none")) + self.assertEqual(self._pass(), 0) + + def test_a_job_failing_after_the_build_is_collected_too(self): + self._fail(1, bits_failure()) + self._pass() + self.gl.jobs[1] = self.gl.jobs[1] + [_job(11, "notify", "failed", stage="status")] + self.gl.logs[11] = "curl: (6) Could not resolve host: hooks.example\n" + self.gl.add(1, status="failed", community="lcg", package="arrow", + updated=DAY + "T13:00:00Z", jobs=self.gl.jobs[1]) + self._pass() + self.assertEqual(sorted(f["job"] for f in failures.list_failures(self.conn)), [10, 11]) + + def test_an_unreadable_log_is_retried_later_and_holds_up_nothing(self): + self._fail(1, forge_ops.ForgeError(502, "bad gateway")) + self._fail(2, bits_failure()) + with self.assertLogs("bits_orchestrator", "WARNING"): + self.assertEqual(self._pass(), 1) + row = self.conn.execute("SELECT collected_ms FROM pipelines WHERE id = 1").fetchone() + self.assertLess(row[0], 0) + self.gl.logs[10] = bits_failure() + self.assertEqual(failures.collect(self.conn, self.gl), 0) # not yet + self.conn.execute("UPDATE pipelines SET collected_ms = -1 WHERE id = 1") + self.assertEqual(failures.collect(self.conn, self.gl), 1) + + def test_a_retry_that_fails_again_is_collected_again(self): + self._fail(1, bits_failure()) + self._pass() + self.gl.add(1, status="failed", community="lcg", package="arrow", updated=DAY + "T12:00:00Z", + jobs=[_job(10, "bits-prepub-build", "failed"), + _job(12, "bits-prepub-build", "failed", failure_reason="script_failure")]) + self.gl.logs[12] = bits_failure() + # running in between (the retry), then failed: collected anew + self.conn.execute("UPDATE pipelines SET outcome = 'running' WHERE id = 1") + self._pass() + self.assertEqual([f["job"] for f in failures.list_failures(self.conn)], [12, 10]) + self.assertEqual(failures.list_issues(self.conn)[0]["count"], 2) + + def test_rematch_names_issues_after_a_new_signature(self): + self._fail(1, bits_failure(error="fatal error: zlib.h: No such file or directory")) + self._pass() + self.assertIsNone(failures.list_issues(self.conn)[0]["signature"]) + with db.transaction(self.conn): + self.conn.execute("INSERT INTO signatures (pattern, cause, action, proposed_by," + " accepted_by, created_ms) VALUES ('zlib\\.h', 'zlib missing'," + " 'Add zlib', 'alice', 'carol', 0)") + self.assertEqual(failures.rematch(self.conn), 1) + self.assertEqual(failures.list_issues(self.conn)[0]["signature"]["cause"], "zlib missing") + + def test_seeds_once(self): + n = len(failures.list_signatures(self.conn)) + self.assertEqual(failures.seed_signatures(self.conn), 0) + self.assertEqual(len(failures.list_signatures(self.conn)), n) + + +class BackfillTest(_DB): + def test_older_failed_pipelines_are_read_page_by_page_until_the_horizon(self): + for i in range(1, 46): + self._fail(i, bits_failure(), updated=_day(10 + i % 28) + "T10:00:00Z") + self._fail(99, bits_failure(), updated=_day(400) + "T00:00:00Z") # beyond 365 days + n = 0 + while failures.backfill(self.conn, self.gl, days=365, lookback_days=7, budget=20): + n += 1 + self.assertLess(n, 10) + ids = {r[0] for r in self.conn.execute("SELECT id FROM pipelines")} + self.assertTrue(set(range(1, 46)) <= ids) + self.assertEqual(reconcile._kv_get(self.conn, "backfill.done"), "1") + self.assertEqual(failures.backfill(self.conn, self.gl, days=365), 0) + # They settle and are collected like any other. + reconcile.settle(self.conn, self.gl) + reconcile.settle(self.conn, self.gl) + while failures.collect(self.conn, self.gl): + pass + self.assertEqual(failures.list_issues(self.conn)[0]["count"], 45 + (99 in ids)) + + def test_asking_for_more_days_later_carries_on(self): + self._fail(1, bits_failure(), updated=_day(30) + "T10:00:00Z") + self._fail(2, bits_failure(), updated=_day(200) + "T10:00:00Z") + while failures.backfill(self.conn, self.gl, days=60, budget=1): + pass + ids = {r[0] for r in self.conn.execute("SELECT id FROM pipelines")} + self.assertIn(1, ids) + self.assertEqual(failures.backfill(self.conn, self.gl, days=60, budget=1), 0) + while failures.backfill(self.conn, self.gl, days=400, budget=1): + pass + self.assertIn(2, {r[0] for r in self.conn.execute("SELECT id FROM pipelines")}) + + def test_off_by_default(self): + self.assertEqual(failures.backfill(self.conn, self.gl, days=0), 0) + self.assertEqual(self.gl.calls, []) + + +class TailTest(unittest.TestCase): + def _resp(self, status, chunks): + class R: + status_code = status + + def __enter__(s): + return s + + def __exit__(s, *a): + return False + + def iter_content(s, n): + return iter(chunks) + return R() + + def test_asks_for_the_end_and_caps_a_whole_body(self): + f = forge_ops.GitLabForge("https://gl/api/v4", "t", "42") + seen = {} + + def get(url, headers, timeout, stream, allow_redirects=True): + seen["range"] = headers.get("Range") + return self._resp(206, [b"x" * 10, b"END"]) + with patch.object(forge_ops.requests, "get", get): + self.assertEqual(f.job_log_tail(5, max_bytes=8), b"xxxxxEND") + self.assertEqual(seen["range"], "bytes=-8") + with patch.object(forge_ops.requests, "get", lambda *a, **k: self._resp(200, [b"y" * 1000] * 10)): + self.assertIsNone(f._tail("/x", 8, max_total=5000)) + with patch.object(forge_ops.requests, "get", lambda *a, **k: self._resp(404, [])): + self.assertIsNone(f.job_artifact_tail(5, "bits-build.log")) + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol", + "BITS_FORGE_PROJECT": "42", "BITS_ORCH_DB": self.path}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + self._fail(1, bits_failure()) + self._fail(2, bits_failure(line=50), community="atlas") + self._fail(3, bits_failure(error="fatal error: zlib.h: No such file or directory")) + self._pass() + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer %s" % user}) + + def test_issues_and_their_occurrences_are_limited_to_ones_communities(self): + counts = lambda u, q="": sorted(i["count"] for i in self._get(u, "/ops/issues" + q).json()["issues"]) + self.assertEqual(counts("carol"), [1, 2]) + self.assertEqual(counts("alice"), [1, 1]) # only lcg's occurrences + self.assertEqual(counts("bob"), [1]) + (mine,) = self._get("bob", "/ops/issues").json()["issues"] + self.assertEqual((mine["communities"], mine["platforms"]), (["atlas"], ["el9"])) + self.assertNotIn("lcg", str(mine)) + self.assertEqual(counts("dave"), []) + self.assertEqual(counts("alice", "?community=atlas"), []) + shared = mine["issue"] + occ = self._get("bob", "/ops/issues/%d" % shared).json()["failures"] + self.assertEqual([f["community"] for f in occ], ["atlas"]) + self.assertEqual(len(self._get("carol", "/ops/issues/%d" % shared).json()["failures"]), 2) + only_lcg = [i["issue"] for i in self._get("alice", "/ops/issues").json()["issues"] + if i["issue"] != shared][0] + self.assertEqual(self._get("bob", "/ops/issues/%d" % only_lcg).status_code, 404) + + def test_failures_and_signatures(self): + got = self._get("bob", "/ops/failures").json()["failures"] + self.assertEqual([f["pipeline"] for f in got], [2]) + self.assertNotIn("hunter2", got[0]["excerpt"]) + read = lambda q: [f["pipeline"] for f in self._get("dave", "/ops/failures" + q).json()["failures"]] + self.assertEqual((read(""), read("?community=atlas")), ([], [2])) # any signed-in user + self.assertEqual([f["pipeline"] for f in self._get("carol", "/ops/failures?pipeline=3") + .json()["failures"]], [3]) + sigs = self._get("dave", "/ops/signatures").json()["signatures"] + self.assertEqual(len(sigs), len(failures.SEEDS)) + self.assertEqual(self.client.get("/ops/issues").status_code, 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_hosts.py b/console-backend/tests/test_orch_hosts.py new file mode 100644 index 00000000..ad1bc3b9 --- /dev/null +++ b/console-backend/tests/test_orch_hosts.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""What an overall admin does to a build host, as actions (orch/hosts.py).""" + +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, forge_ops, main +from console_backend.orch import actions, db, events, hosts, kinds + +CAROL = events.Actor("person", "carol") + + +class FakeForge: + def __init__(self, runners=None, fail=None): + self.runners = runners if runners is not None else [ + {"id": 1, "paused": False, "runner_type": "project_type", "tags": ["bits-host-a"]}, + {"id": 2, "paused": True, "runner_type": "project_type", "tags": ["bits-host-a"]}, + {"id": 3, "paused": False, "runner_type": "instance_type", "is_shared": True, + "tags": ["bits-host-a"]}, + {"id": 4, "paused": False, "runner_type": "project_type", "tags": ["bits-host-b"]}] + self.fail = fail + self.calls = [] + + def list_runners(self, tag=""): + return [r for r in self.runners if tag in r["tags"]] + + def set_runner_paused(self, rid, paused): + if self.fail: + raise self.fail + self.calls.append(("paused", rid, paused)) + + def trigger_pipeline(self, ref, variables=None, name=None): + if self.fail: + raise self.fail + self.calls.append(("pipeline", ref, {v["key"]: v["value"] for v in variables})) + return {"id": 77, "web_url": "https://gl/p/77", "status": "created"} + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + +class HostActionsTest(_DB): + def setUp(self): + super().setUp() + self.forge = FakeForge() + self.deps = kinds.Deps(config.Settings(env={"BITS_FORGE_REFS": "main,next"})) + self.deps.forge = lambda: self.forge + + def _run(self, kind, host="a"): + row, _ = actions.request(self.conn, kind, {"host": host}, CAROL, "", True) + actions.run_due(self.conn, self.deps, lambda a, c: True) + return actions.get(self.conn, row["id"]) + + def test_pause_and_resume_change_only_the_hosts_own_runners(self): + done = self._run("host.pause") + self.assertEqual((done["state"], done["result"]), + ("done", {"runners": [1, 2], "changed": [1]})) + self.assertEqual(self.forge.calls, [("paused", 1, True)]) + self.forge.calls.clear() + self.assertEqual(self._run("host.resume")["result"]["changed"], [2]) + self.assertEqual(self.forge.calls, [("paused", 2, False)]) + + def test_a_host_with_no_runner_fails_and_changes_nothing(self): + for kind in ("host.pause", "host.refresh"): + done = self._run(kind, host="gone") + self.assertEqual(done["state"], "failed") + self.assertIn("bits-host-gone", done["error"]) + self.assertEqual(self.forge.calls, []) + + def test_refresh_is_a_pipeline_pinned_to_the_host(self): + done = self._run("host.refresh") + self.assertEqual(done["result"], {"pipeline": 77, "web_url": "https://gl/p/77"}) + self.assertEqual(self.forge.calls, [("pipeline", "main", { + "REFRESH_IMAGES_JOB": "true", "BITS_RUNNER_TAG": "bits-host-a"})]) + + def test_prune_is_a_pipeline_pinned_to_the_host_with_its_target(self): + row, _ = actions.request(self.conn, "host.prune", {"host": "a", "min_free_gb": 250}, + CAROL, "", True) + actions.run_due(self.conn, self.deps, lambda a, c: True) + self.assertEqual(actions.get(self.conn, row["id"])["state"], "done") + self.assertEqual(self.forge.calls, [("pipeline", "main", { + "HOST_PRUNE_JOB": "true", "BITS_PRUNE_MIN_FREE_GB": "250", + "BITS_RUNNER_TAG": "bits-host-a"})]) + for bad in ({"host": "a"}, {"host": "a", "min_free_gb": 0}, + {"host": "a", "min_free_gb": "200"}, {"host": "a", "min_free_gb": True}, + {"host": "a", "min_free_gb": 100001}): + with self.assertRaises(actions.ActionError, msg=bad): + actions.request(self.conn, "host.prune", bad, CAROL, "", True) + + def test_prune_targets_are_saved_per_host_as_events(self): + self.assertEqual(hosts.prune_targets(self.conn), {}) + hosts.set_prune_target(self.conn, "a", 300, CAROL) + self.assertEqual(hosts.set_prune_target(self.conn, "b", 150, CAROL), {"a": 300, "b": 150}) + self.assertEqual(hosts.set_prune_target(self.conn, "a", 400, CAROL), {"a": 400, "b": 150}) + got = [(e["kind"], e["subject"], e["payload"]) for e in events.since(self.conn)] + self.assertEqual(got[-1], ("host.prune_target_set", "host:a", {"host": "a", "min_free_gb": 400})) + for host, gb in (("a b", 200), ("a", 0), ("a", 2.5), ("a", "200"), ("a", None), ("a", True)): + with self.assertRaises(ValueError, msg=(host, gb)): + hosts.set_prune_target(self.conn, host, gb, CAROL) + self.assertEqual(len(events.since(self.conn)), 3) + + def test_gitlab_down_is_retried_a_refusal_fails(self): + self.forge.fail = forge_ops.ForgeError(502, "bad gateway") + self.assertEqual(self._run("host.pause")["state"], "approved") # again later + self.forge.fail = forge_ops.ForgeError(403, "forbidden") + row, _ = actions.request(self.conn, "host.resume", {"host": "b"}, CAROL, "", True) + self.forge.runners[3]["paused"] = True + actions.run_due(self.conn, self.deps, lambda a, c: True) + done = actions.get(self.conn, row["id"]) + self.assertEqual(done["state"], "failed") + self.assertIn("403", done["error"]) + + def test_a_refusal_midway_says_what_was_changed_before_it(self): + self.forge.runners[1]["paused"] = False + calls = [] + + def refuse_second(rid, paused): + calls.append(rid) + if len(calls) == 2: + raise forge_ops.ForgeError(403, "forbidden") + self.forge.set_runner_paused = refuse_second + done = self._run("host.pause") + self.assertEqual(done["state"], "failed") + self.assertIn("runner 2: forge 403: forbidden; changed before it: [1]", done["error"]) + + def test_a_host_name_is_checked(self): + for bad in ("", "-a", "a b", "a/b", "a" * 64, "a;b"): + with self.assertRaises(actions.ActionError, msg=bad): + actions.request(self.conn, "host.pause", {"host": bad}, CAROL, "", True) + + +class SetRunnerPausedTest(unittest.TestCase): + @patch("console_backend.forge_ops.requests.request") + def test_puts_the_runner_outside_the_project(self, req): + req.return_value = type("R", (), {"status_code": 200, "json": lambda s: {"id": 9}})() + forge_ops.GitLabForge("https://gl/api/v4", "tok", "42").set_runner_paused(9, True) + self.assertEqual(req.call_args.args[:2], ("PUT", "https://gl/api/v4/runners/9")) + self.assertEqual(req.call_args.kwargs["json"], {"paused": True}) + + +class HostApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "testbed @alice\n* @carol", + "BITS_FORGE_OPS_TOKEN": "optok", "BITS_FORGE_PROJECT": "42", + "BITS_ORCH_DB": self.path}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + + def _post(self, user, body): + return self.client.post("/ops/actions", headers={"Authorization": "Bearer " + user}, + json=body) + + def test_only_an_overall_admin_acts_on_a_host(self): + body = {"kind": "host.pause", "params": {"host": "a"}} + self.assertEqual(self._post("alice", body).status_code, 403) + r = self._post("carol", body) + self.assertEqual((r.status_code, r.json()["subject"], r.json()["community"]), + (201, "host:a", "")) + self.assertEqual(self._post("carol", {"kind": "host.pause", + "params": {"host": "a b"}}).status_code, 400) + + def test_prune_targets_are_read_by_admins_and_set_by_overall_admins(self): + h = lambda u: {"Authorization": "Bearer " + u} + url = "/ops/hosts/prune-targets" + self.assertEqual(self.client.get(url).status_code, 401) + self.assertEqual(self.client.get(url + "?community=other", headers=h("alice")).status_code, 403) + self.assertEqual(self.client.get(url, headers=h("dave")).status_code, 403) + self.assertEqual(self.client.get(url + "?community=testbed", headers=h("alice")).json(), + {"default": 200, "hosts": {}}) + put = lambda u, body, host="a": self.client.put("/ops/hosts/%s/prune-target" % host, + headers=h(u), json=body) + self.assertEqual(put("alice", {"min_free_gb": 300}).status_code, 403) + self.assertEqual(put("carol", {"min_free_gb": 0}).status_code, 400) + self.assertEqual(put("carol", {"min_free_gb": 300}, host="a%20b").status_code, 400) + self.assertEqual(put("carol", {"min_free_gb": 300}).json(), {"default": 200, "hosts": {"a": 300}}) + self.assertEqual(self.client.get(url, headers=h("alice")).json()["hosts"], {"a": 300}) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_nightlies.py b/console-backend/tests/test_orch_nightlies.py new file mode 100644 index 00000000..0d29344f --- /dev/null +++ b/console-backend/tests/test_orch_nightlies.py @@ -0,0 +1,375 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The nights of the schedules: firing them into the build queue, and their runs.""" + +import datetime as dt +import json +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import approvals, build_queue as bq, db, nightlies as nl, schedules as sch + +UTC = dt.timezone.utc +PLATFORMS = """ +platforms: + - {name: x86_64-el9, image: registry/el9, install_dir: el9-x86_64, arch: el9-x86_64-gcc} + - {name: aarch64-el9, image: registry/el9-arm} +""" +COMMUNITY = """ +title: LCG +platforms: + - x86_64-el9|registry/lcg-el9||el9-lcg + - aarch64-el9 +""" + + +class Reader: + def __init__(self): + self.files = {"config/platforms.yaml": PLATFORMS, "communities/LCG/ui-config.yaml": COMMUNITY, + "config/monitoring.yaml": "build:\n max_jobs_per_container: 16\n"} + self.read = [] + + def file_raw(self, path, ref, max_bytes=None): + self.read.append((path, ref)) + return self.files.get(path, "").encode() or None + + +def _ms(y, mo, d, h, mi): + return int(dt.datetime(y, mo, d, h, mi, tzinfo=UTC).timestamp() * 1000) + + +def _vars(variables): + return {v["key"]: v["value"] for v in variables} + + +class VariablesTest(unittest.TestCase): + def setUp(self): + self.s = sch.check({"name": "n", "package": "ROOT", "version": "v6-36", "platforms": ["x86_64-el9"], + "time": "02:00", "defaults": "release::gcc15", "set": {"release": "LCG_110b"}, + "extra_repos": "https://github.com/bitsorg/lcg.bits.git@LCG_110b", + "extra_build_args": "--keep-going\t--fetch-repos", "rebuild": True, "jobs": 8, + "cvmfs": True}, "LCG") + r = Reader() + self.cfg = nl.config(r, "LCG", "main") + + def test_the_platform_from_the_community_then_the_defaults(self): + self.assertEqual(nl.resolve_platform("x86_64-el9", self.cfg["platforms"], self.cfg["community"]), + ("registry/lcg-el9", "el9-x86_64", "el9-lcg", "x86_64")) + self.assertEqual(nl.resolve_platform("aarch64-el9", self.cfg["platforms"], self.cfg["community"]), + ("registry/el9-arm", "aarch64-el9", "aarch64-el9", "aarch64")) + with self.assertRaises(nl.Unbuildable): + nl.resolve_platform("other", [], {}) + with self.assertRaises(nl.Unbuildable): + nl.resolve_platform("x", [], {"platforms": ["x|img with space"]}) + self.assertEqual(nl.resolve_platform("osx", [{"name": "osx"}], {}), ("", "osx", "osx", "x86_64")) + + def test_a_nights_build_is_what_the_gitlab_fan_out_sends_with_its_defaults_and_day(self): + v = _vars(nl.variables(self.s, "x86_64-el9", self.cfg, None, "2026-10-29")) + self.assertEqual(v, { + "PACKAGE": "ROOT", "PLATFORM": "x86_64-el9", "RUNNER_ARCH": "x86_64", + "BITS_RUNNER_TAG": "bits-build-x86_64", "COMMUNITY": "LCG", "CVMFS_INSTALL_DIR": "el9-x86_64", + "BITS_BUILD_ARGS": "--force-rebuild ROOT --architecture el9-lcg --docker --docker-image" + " registry/lcg-el9 -j 8 --docker-extra-args=--cpus=16 --sandbox off" + " --defaults release::gcc15 --day Thu --set release=LCG_110b" + " --keep-going --fetch-repos", + "DEFAULTS": "release::gcc15", "VERSION": "v6-36", "BITS_EXTRA_REPOS": "https://github.com/bitsorg/lcg.bits.git@LCG_110b", + "PUBLISH_CVMFS": "true", "NIGHTLY_SUFFIX": " [nightly]"}) + v = _vars(nl.variables(self.s, "x86_64-el9", self.cfg, "alice")) + self.assertEqual((v["CERTIFY"], v["BITS_CERTIFIER"]), ("true", "alice")) + self.assertNotIn("--day", v["BITS_BUILD_ARGS"]) + + def test_the_communitys_defaults_overlay_comes_last_as_in_the_console(self): + cfg = dict(self.cfg, community=dict(self.cfg["community"], defaults_overlay="testbed")) + v = _vars(nl.variables(self.s, "x86_64-el9", cfg, None, "2026-10-29")) + self.assertEqual(v["DEFAULTS"], "release::gcc15::testbed") + self.assertIn("--defaults release::gcc15::testbed --day Thu", v["BITS_BUILD_ARGS"]) + s = dict(self.s, defaults="testbed::gcc15") + self.assertEqual(nl.defaults_chain(s, cfg["community"]), "gcc15::testbed") + self.assertEqual(nl.defaults_chain(dict(self.s, defaults=""), cfg["community"]), "testbed") + with self.assertRaises(nl.Unbuildable) as e: + nl.defaults_chain(self.s, {"defaults_overlay": "a b"}) + self.assertIn("release::gcc15::a b", str(e.exception)) + + +class _Nights(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.reader = Reader() + + def _schedule(self, last_fired, **more): + p = {"name": "ROOT nightly", "package": "ROOT", "platforms": ["x86_64-el9", "aarch64-el9"], + "time": "02:00"} + p.update(more) + s = sch.create(self.conn, sch.check(p, "LCG"), "alice") + with db.transaction(self.conn): + self.conn.execute("UPDATE schedules SET last_fired_ms = ? WHERE id = ?", (last_fired, s["id"])) + return sch.get(self.conn, s["id"]) + + def _done(self): + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed'") + + def _runs(self): + return [(r["night"], r["platform"], r["attempt"], r["state"], r["reason"]) for r in self.conn.execute( + "SELECT * FROM schedule_runs ORDER BY night, platform, attempt")] + + + +class StepTest(_Nights): + def test_a_schedule_fires_once_at_its_time(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0)) + self.assertEqual(nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 1, 59)), 0) + self.assertEqual(nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)), 1) + self.assertEqual(nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 5)), 0) + self.assertEqual(self._runs(), [("2026-10-28", "aarch64-el9", 1, "queued", ""), + ("2026-10-28", "x86_64-el9", 1, "queued", "")]) + q = [(r["requested_by"], r["batch"].rsplit(":", 1)[0], r["community"], + _vars(json.loads(r["variables"]))["PLATFORM"]) + for r in self.conn.execute("SELECT * FROM build_queue ORDER BY id")] + self.assertEqual(q, [("schedule:%d" % s["id"], "schedule:%d:2026-10-28" % s["id"], "lcg", p) + for p in ("aarch64-el9", "x86_64-el9")]) + self.assertIn(("communities/LCG/ui-config.yaml", "main"), self.reader.read) + + def test_a_nights_day_is_its_local_date_not_utcs(self): + from zoneinfo import ZoneInfo + self._schedule(_ms(2026, 10, 28, 12, 0), platforms=["x86_64-el9"], time="00:30", + defaults="gcc15::opt::dev3") + nl.step(self.conn, self.reader, "main", ZoneInfo("Europe/Zurich"), now=_ms(2026, 10, 28, 23, 31)) + v = _vars(json.loads(self.conn.execute("SELECT variables FROM build_queue").fetchone()[0])) + self.assertEqual(self._runs()[0][0], "2026-10-29") # Wed 23:31 UTC, Thu in Zurich + self.assertIn("--defaults gcc15::opt::dev3 --day Thu", v["BITS_BUILD_ARGS"]) + self.assertEqual(v["DEFAULTS"], "gcc15::opt::dev3") + + def test_a_missed_night_fires_within_the_grace_and_older_ones_are_not_run(self): + self._schedule(_ms(2026, 10, 25, 12, 0), platforms=["x86_64-el9"]) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 7, 0)) # 5 h late + self.assertEqual([r[0::3] for r in self._runs()], [("2026-10-26", "not_run"), ("2026-10-27", "not_run"), + ("2026-10-28", "queued")]) + self._schedule(_ms(2026, 10, 27, 12, 0), name="late", platforms=["x86_64-el9"]) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 9, 0)) # 7 h late + self.assertEqual(self._runs()[-1][0::3], ("2026-10-28", "not_run")) + + def test_a_platform_still_running_skips_the_next_night(self): + self._schedule(_ms(2026, 10, 27, 12, 0)) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed' WHERE platform = 'aarch64-el9'") + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 1)) + self.assertEqual(self._runs()[2:], [("2026-10-29", "aarch64-el9", 1, "queued", ""), + ("2026-10-29", "x86_64-el9", 1, "skipped", + "the previous night still runs")]) + + def test_a_night_that_cannot_be_built_is_not_run(self): + self._schedule(_ms(2026, 10, 27, 12, 0), platforms=["x86_64-el9", "sparc"]) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + self.assertEqual([r[1::2] for r in self._runs()], [("sparc", "not_run"), ("x86_64-el9", "queued")]) + del self.reader.files["communities/LCG/ui-config.yaml"] + self._done() + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 1)) + self.assertEqual({r[3:] for r in self._runs()[2:]}, + {("not_run", "communities/LCG/ui-config.yaml is not in the console project")}) + + def test_gitlab_unreadable_leaves_the_night_for_the_next_pass_and_others_fire(self): + from console_backend import forge_ops + a = self._schedule(_ms(2026, 10, 27, 12, 0), platforms=["x86_64-el9"]) + sch.create(self.conn, sch.check({"name": "other", "package": "X", "platforms": ["p"], "time": "02:00"}, + "ATLAS"), "carol") + with db.transaction(self.conn): + self.conn.execute("UPDATE schedules SET last_fired_ms = ?", (_ms(2026, 10, 27, 12, 0),)) + self.reader.files.update({"communities/ATLAS/ui-config.yaml": "platforms: [p]\n"}) + real = self.reader.file_raw + + def flaky(path, ref, max_bytes=None): + if "LCG" in path: + raise forge_ops.ForgeError(503, "down") + return real(path, ref) + self.reader.file_raw = flaky + with self.assertRaises(RuntimeError): + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + self.assertEqual([r[1] for r in self._runs()], ["p"]) # ATLAS's fired + self.assertEqual(sch.get(self.conn, a["id"])["last_fired_ms"], _ms(2026, 10, 27, 12, 0)) + self.reader.file_raw = real + self.assertEqual(nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 3, 0)), 1) + + def test_paused_and_changed_schedules_do_not_fire(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0)) + sch.set_enabled(self.conn, s["id"], False, "alice") + self.assertEqual(nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)), 0) + + def test_a_certifying_night_certifies_under_its_standing_approval(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0), certify=True, platforms=["x86_64-el9"]) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + self.assertEqual(self._runs()[0][3:], ("queued", "built, not certified: no standing approval")) + entry = self.conn.execute("SELECT * FROM build_queue").fetchone() + self.assertNotIn("CERTIFY", _vars(json.loads(entry["variables"]))) + aid = approvals.grant_standing(self.conn, nl.subject(s["id"]), s["content_sha256"], ["lcg"], "alice", + 2, 30, {"id": "x"}, b"\0" * 16) + self._done() + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 1)) + entry = self.conn.execute("SELECT * FROM build_queue ORDER BY id DESC").fetchone() + v = _vars(json.loads(entry["variables"])) + self.assertEqual((v["CERTIFY"], v["BITS_CERTIFIER"], entry["certify"]), ("true", "alice", 1)) + with db.transaction(self.conn): # it starts: its pipeline is pre-approved + self.conn.execute("UPDATE build_queue SET pipeline_id = 77, state = 'dispatched' WHERE id = ?", + (entry["id"],)) + pre = approvals.preapprove_queued(self.conn, bq.get(self.conn, entry["id"])) + self.assertIsNotNone(pre) + self.assertEqual(approvals.get(self.conn, aid)["runs_left"], 1) + self.assertEqual(approvals.get(self.conn, pre)["subject"], "77") + with db.transaction(self.conn): # another's approval: not this pipeline's + self.conn.execute("UPDATE approvals SET approver = 'bob' WHERE id = ?", (aid,)) + other = dict(bq.get(self.conn, entry["id"])) + other.update(id=entry["id"], pipeline_id=78) + self.assertIsNone(approvals.preapprove_queued(self.conn, other)) + self.conn.execute("UPDATE approvals SET approver = 'alice' WHERE id = ?", (aid,)) + self._done() + s2 = self._schedule(_ms(2026, 10, 28, 12, 0), name="two", certify=True) # 2 platforms, 1 run left + approvals.grant_standing(self.conn, nl.subject(s2["id"]), s2["content_sha256"], ["lcg"], "alice", + 1, 30, {"id": "x"}, b"\1" * 16) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 2)) + self.assertEqual({r["reason"] for r in self.conn.execute( + "SELECT reason FROM schedule_runs WHERE schedule_id = ?", (s2["id"],))}, + {"built, not certified: the standing approval has 1 runs left"}) + sch.update(self.conn, s["id"], sch.check({"name": "ROOT nightly", "package": "ROOT", "time": "02:00", + "platforms": ["x86_64-el9", "aarch64-el9"], "certify": True}, + "LCG"), "alice", 1) + self._done() + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 30, 2, 1)) # new content: none + self.assertEqual({r["reason"] for r in self.conn.execute( + "SELECT reason FROM schedule_runs WHERE schedule_id = ? AND night = '2026-10-30'", (s["id"],))}, + {"built, not certified: no standing approval"}) + + +class ResultsTest(_Nights): + def _pipeline(self, pid, outcome): + with db.transaction(self.conn): + self.conn.execute("INSERT OR REPLACE INTO pipelines (id, is_build, outcome, variables, jobs," + " first_seen_ms) VALUES (?, 1, ?, '{}', '[]', 0)", (pid, outcome)) + + def _start(self, entry, pid): + with db.transaction(self.conn): + self.conn.execute("UPDATE build_queue SET state = 'dispatched', pipeline_id = ? WHERE id = ?", + (pid, entry)) + + def _fired(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0)) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + return s, [r["queue_entry"] for r in self.conn.execute( + "SELECT queue_entry FROM schedule_runs ORDER BY platform")] # aarch64, x86_64 + + def test_a_run_follows_its_pipeline(self): + s, (arm, x86) = self._fired() + self._start(arm, 500) + nl.results(self.conn, now=_ms(2026, 10, 28, 3, 0)) + self.assertEqual([r[3] for r in self._runs()], ["queued", "queued"]) # not recorded yet + self._pipeline(500, "running") + nl.results(self.conn, now=_ms(2026, 10, 28, 3, 0)) + self.assertEqual([r[3] for r in self._runs()], ["running", "queued"]) + self._pipeline(500, "failed") + with db.transaction(self.conn): + self.conn.execute("INSERT INTO issues (fingerprint, package, title, excerpt, first_seen, last_seen," + " created_ms) VALUES ('f', 'tiff', 't', 'e', '', '', 0)") + self.conn.execute("INSERT INTO failures (job_id, pipeline_id, package, job_name, excerpt, source," + " fingerprint, issue_id, recorded_ms) VALUES (1, 500, 'tiff', 'build', 'e', 'log'," + " 'f', 1, 0)") + nl.results(self.conn, now=_ms(2026, 10, 28, 4, 0)) + r = self.conn.execute("SELECT * FROM schedule_runs WHERE queue_entry = ?", (arm,)).fetchone() + self.assertEqual((r["state"], r["failed_at"], r["pipeline_id"]), ("failed", "tiff (build)", 500)) + self._pipeline(500, "success") # retried in GitLab + with db.transaction(self.conn): + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms)" + " VALUES (500, 'b-1', 'lcg', 0)") + nl.results(self.conn, now=_ms(2026, 10, 28, 6, 0)) + r = self.conn.execute("SELECT * FROM schedule_runs WHERE queue_entry = ?", (arm,)).fetchone() + self.assertEqual((r["state"], r["build_id"], r["failed_at"]), ("passed", "b-1", "")) + self.assertEqual(bq.get(self.conn, x86)["state"], "queued") + nl.results(self.conn, now=_ms(2026, 10, 28, 15, 0)) # 13 h without a host + self.assertEqual(bq.get(self.conn, x86)["state"], "canceled") + self.assertEqual(self._runs()[1][3:], ("not_run", "no free build host for 12 hours")) + ev = self.conn.execute("SELECT kind, payload FROM events ORDER BY seq DESC LIMIT 1").fetchone() + self.assertEqual((ev["kind"], json.loads(ev["payload"])["entry"]), ("queue.canceled", x86)) + + def test_a_canceled_build_is_not_run_and_frees_its_platform(self): + s, (arm, x86) = self._fired() + bq.cancel(self.conn, arm, "alice") + nl.results(self.conn, now=_ms(2026, 10, 28, 3, 0)) + self.assertEqual(self._runs()[0][3], "not_run") + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 1)) + self.assertEqual([r[1::2] for r in self._runs()[2:]], [("aarch64-el9", "queued"), ("x86_64-el9", "skipped")]) + + def test_run_now_and_the_matrix(self): + s, (arm, x86) = self._fired() + self._start(arm, 500) + self._pipeline(500, "failed") + nl.results(self.conn, now=_ms(2026, 10, 28, 3, 0)) + with self.assertRaises(sch.Invalid): + with db.transaction(self.conn): + nl.run_now(self.conn, s["id"], ["sparc"], nl.config(self.reader, "LCG", "main"), "main", UTC, + "alice", now=_ms(2026, 10, 28, 9, 0)) + with db.transaction(self.conn): # Run again: the failed one + got = nl.run_now(self.conn, s["id"], ["aarch64-el9"], nl.config(self.reader, "LCG", "main"), "main", + UTC, "alice", now=_ms(2026, 10, 28, 9, 0)) + self.assertEqual(got, {"aarch64-el9": "queued"}) + with db.transaction(self.conn): # still waiting: nothing + got = nl.run_now(self.conn, s["id"], [], nl.config(self.reader, "LCG", "main"), "main", + UTC, "alice", now=_ms(2026, 10, 28, 9, 5)) + self.assertEqual(got, {"aarch64-el9": "skipped", "x86_64-el9": "skipped"}) + with self.assertRaises(sch.Invalid): + with db.transaction(self.conn): + nl.run_now(self.conn, s["id"], ["aarch64-el9"] * 2, {}, "main", UTC, "alice") + m = nl.matrix(self.conn, {"lcg"}, 3, UTC, now=_ms(2026, 10, 28, 9, 0)) + self.assertEqual(m["nights"], ["2026-10-26", "2026-10-27", "2026-10-28"]) + (row,) = m["rows"] + self.assertEqual({p: (c["2026-10-28"]["state"], c["2026-10-28"]["attempt"]) for p, c in row["cells"].items()}, + {"aarch64-el9": ("queued", 2), "x86_64-el9": ("queued", 1)}) + self.assertEqual(row["failed"]["2026-10-28"], 0) # its latest attempt + self.assertEqual(nl.matrix(self.conn, set(), 3, UTC)["rows"], []) + + + +class VariantsTest(_Nights): + def test_a_night_builds_every_cell_with_its_variant_first(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0), platforms=["x86_64-el9", "aarch64-el9"], defaults="dev3", + variants=["gcc15::opt", "clang::opt"]) + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 28, 2, 1)) + runs = [(r["platform"], r["variant"], r["state"]) for r in self.conn.execute( + "SELECT * FROM schedule_runs ORDER BY platform, variant")] + self.assertEqual(runs, [(p, v, "queued") for p in ("aarch64-el9", "x86_64-el9") + for v in ("clang::opt", "gcc15::opt")]) + q = {(_vars(json.loads(r["variables"]))["PLATFORM"], r["name"], _vars(json.loads(r["variables"]))["DEFAULTS"]) + for r in self.conn.execute("SELECT * FROM build_queue")} + self.assertIn(("x86_64-el9", "ROOT (x86_64-el9, gcc15::opt)", "gcc15::opt::dev3"), q) + self.assertEqual(len(q), 4) + m = nl.matrix(self.conn, {"lcg"}, 1, UTC, now=_ms(2026, 10, 28, 9, 0)) + (row,) = m["rows"] + self.assertEqual([c["key"] for c in row["schedule"]["cells"]], + ["aarch64-el9 clang::opt", "aarch64-el9 gcc15::opt", "x86_64-el9 clang::opt", + "x86_64-el9 gcc15::opt"]) + self.assertEqual(row["cells"]["x86_64-el9 gcc15::opt"]["2026-10-28"]["state"], "queued") + # the next night: each cell still waits, each is skipped on its own + nl.step(self.conn, self.reader, "main", UTC, now=_ms(2026, 10, 29, 2, 1)) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM schedule_runs WHERE state = 'skipped'").fetchone()[0], 4) + + def test_run_again_takes_cells_or_platforms(self): + s = self._schedule(_ms(2026, 10, 27, 12, 0), platforms=["x86_64-el9"], variants=["gcc15::opt", "clang::opt"]) + cfg = nl.config(self.reader, "LCG", "main") + with db.transaction(self.conn): + got = nl.run_now(self.conn, s["id"], [], cfg, "main", UTC, "alice", now=_ms(2026, 10, 28, 9, 0), + cells=["x86_64-el9 clang::opt"]) + self.assertEqual(got, {"x86_64-el9 clang::opt": "queued"}) + for bad in ({"cells": ["x86_64-el9"]}, {"cells": ["x86_64-el9 clang::opt"] * 2}): + with self.assertRaises(sch.Invalid, msg=bad): + with db.transaction(self.conn): + nl.run_now(self.conn, s["id"], [], cfg, "main", UTC, "alice", **bad) + with db.transaction(self.conn): + got = nl.run_now(self.conn, s["id"], ["x86_64-el9"], cfg, "main", UTC, "alice", now=_ms(2026, 10, 28, 9, 5)) + self.assertEqual(got, {"x86_64-el9 clang::opt": "skipped", "x86_64-el9 gcc15::opt": "queued"}) + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_outside.py b/console-backend/tests/test_orch_outside.py new file mode 100644 index 00000000..606fa1e9 --- /dev/null +++ b/console-backend/tests/test_orch_outside.py @@ -0,0 +1,408 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M4, second half: the outside copy of the audit chain.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from console_backend import config +from console_backend.orch import db, events, outside + +# The shared test vector of bits-services/audit-log/test_auditlog.py: the +# receiver's hash function must stay this one. +VECTOR = [ + ({"seq": 1, "ts_ms": 1760000000000, "kind": "worker.started", "subject": "worker", + "actor_kind": "worker", "actor_id": "orchestrator", "cause": None, + "payload": '{"schema":5,"note":"é"}'}, + "d8e5ed955e51314b180e89473c75025f3b81370fe49e9df3a8ce0d4b826fb264"), + ({"seq": 2, "ts_ms": 1760000000001, "kind": "shadow.match", "subject": "certification:7", + "actor_kind": "worker", "actor_id": "orchestrator", "cause": "action:3", "payload": "{}"}, + "1790c4478017c426a5d33654235a2a38f36a4eb22dacf545d9aec3b84c9db2e0"), +] + + +class FakeOutside: + """The audit-log service's rules, in memory.""" + + def __init__(self): + self.rows = [] + + def head(self): + if not self.rows: + return {"seq": 0, "hash": events.GENESIS, "count": 0} + return {"seq": self.rows[-1]["seq"], "hash": self.rows[-1]["hash"]} + + def send(self, rows): + for r in rows: + seq = len(self.rows) + if r["seq"] <= seq: + if self.rows[r["seq"] - 1] != r: + return False, {"error": "event %d differs" % r["seq"], "head": self.head()} + continue + prev = self.rows[-1]["hash"] if self.rows else events.GENESIS + if r["seq"] != seq + 1 or r["prev_hash"] != prev or events._digest(prev, r) != r["hash"]: + return False, {"error": "does not link", "head": self.head()} + self.rows.append(dict(r)) + return True, {"head": self.head()} + + def events(self, after, limit=1000): + return [dict(r) for r in self.rows[after:after + limit]] + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.conn = db.connect(os.path.join(self.dir, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.out = FakeOutside() + + def add(self, n): + for i in range(n): + with db.transaction(self.conn): + events.append(self.conn, "test.event", "s", events.WORKER, {"i": i}) + + def kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + +class VectorTest(unittest.TestCase): + def test_the_receiver_hashes_as_the_chain_does(self): + prev = events.GENESIS + for row, want in VECTOR: + self.assertEqual(events._digest(prev, row), want) + prev = want + + +class ShipTest(_DB): + def test_events_are_sent_in_batches_after_the_outside_head(self): + self.add(7) + self.assertEqual(outside.ship(self.conn, self.out, batch=3), 7) + self.assertEqual(self.out.head()["seq"], 7) + self.assertEqual(outside.ship(self.conn, self.out), 0) + self.add(2) + self.assertEqual(outside.ship(self.conn, self.out), 2) + self.assertEqual(outside.status(self.conn)["outside"]["seq"], 9) + self.assertEqual(outside.verify(self.conn, self.out), (9, None)) + self.assertEqual(outside.ship(self.conn, None), 0) + + def test_a_rewritten_event_stops_shipping_until_cleared(self): + self.add(3) + outside.ship(self.conn, self.out) + self.conn.execute("UPDATE events SET hash = ? WHERE seq = 3", ("f" * 64,)) + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + self.assertEqual(outside.ship(self.conn, self.out), 0) + alarm = outside.status(self.conn)["alarm"] + self.assertIn("event 3 changed", alarm["reason"]) + self.assertEqual(self.kinds().count("outside.diverged"), 1) + n = self.out.head()["seq"] + self.assertEqual(outside.ship(self.conn, self.out), 0) # stays stopped + self.assertEqual((self.out.head()["seq"], self.kinds().count("outside.diverged")), (n, 1)) + self.assertIn("differs", outside.verify(self.conn, self.out)[1]) + self.assertTrue(outside.clear(self.conn, "carol")) + self.assertFalse(outside.clear(self.conn, "carol")) + self.assertIn("outside.cleared", self.kinds()) + + def test_the_outside_head_is_checked_against_this_chain(self): + self.add(3) + outside.ship(self.conn, self.out) + self.out.rows[2]["hash"] = "e" * 64 # (the fake keeps it; the service would not) + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + outside.ship(self.conn, self.out) + self.assertIn("differs from the outside copy", outside.status(self.conn)["alarm"]["reason"]) + + def test_an_edit_that_keeps_the_hashes_is_found_by_the_daily_walk(self): + self.add(3) + outside.ship(self.conn, self.out) + self.conn.execute("UPDATE events SET payload = '{\"i\":7}' WHERE seq = 2") + self.assertEqual(outside.ship(self.conn, self.out), 0) # not seen by the quick check + self.assertIsNone(outside.status(self.conn)["alarm"]) + self.conn.execute("UPDATE kv SET value = json_set(value, '$.full_ms', 0)" + " WHERE key = 'outside.verified'") + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + outside.ship(self.conn, self.out) + self.assertIn("breaks at event 2", outside.status(self.conn)["alarm"]["reason"]) + self.assertIn("differs", outside.verify(self.conn, self.out)[1]) + + def test_an_outside_copy_that_lost_events(self): + self.add(4) + outside.ship(self.conn, self.out) + del self.out.rows[2:] + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + self.assertEqual(outside.ship(self.conn, self.out), 0) + self.assertIn("went back from event 4 to 2", outside.status(self.conn)["alarm"]["reason"]) + outside.clear(self.conn, "carol") # it was re-seeded on purpose + self.assertGreater(outside.ship(self.conn, self.out), 0) + self.assertEqual(outside.verify(self.conn, self.out)[1], None) + + def test_batches_stay_under_the_services_body_limit(self): + for i in range(6): + with db.transaction(self.conn): + events.append(self.conn, "big", "s", events.WORKER, {"x": "y" * 60000}) + sizes = [] + send = self.out.send + self.out.send = lambda rows: (sizes.append(len(rows)), send(rows))[1] + with patch.object(outside, "BATCH_BYTES", 130000): + self.assertEqual(outside.ship(self.conn, self.out), 6) + self.assertEqual(sizes, [2, 2, 2]) + + def test_a_database_older_than_the_outside_copy(self): + self.add(4) + outside.ship(self.conn, self.out) + self.conn.execute("DELETE FROM events WHERE seq > 2") # restored from an old copy + self.conn.execute("DELETE FROM kv WHERE key = 'outside.verified'") + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + outside.ship(self.conn, self.out) + self.assertIn("older copy", outside.status(self.conn)["alarm"]["reason"]) + # The alarm itself is the local event 3 now: not the outside one. + self.assertIn("event 3 differs", outside.verify(self.conn, self.out)[1]) + + def test_a_refused_batch_is_an_alarm(self): + self.add(2) + self.out.send = lambda rows: (False, {"error": "nope", "head": {"seq": 0}}) + with self.assertLogs("bits_orchestrator.outside", "ERROR"): + self.assertEqual(outside.ship(self.conn, self.out), 0) + self.assertIn("refused", outside.status(self.conn)["alarm"]["reason"]) + + def test_verify_finds_a_broken_outside_chain(self): + self.add(3) + outside.ship(self.conn, self.out) + self.out.rows[1]["payload"] = '{"i":9}' + self.assertIn("own chain breaks at 2", outside.verify(self.conn, self.out)[1]) + + +class RecoverTest(_DB): + def restored(self): + self.add(4) + outside.ship(self.conn, self.out) + self.conn.execute("DELETE FROM events WHERE seq > 2") # restored from an old copy + self.conn.execute("DELETE FROM kv WHERE key = 'outside.head'") + + def test_a_restored_database_gets_back_the_events_it_lacks(self): + self.restored() + self.assertEqual(outside.behind(self.conn, self.out), (2, 4)) + self.assertEqual(outside.recover(self.conn, self.out, "carol"), 2) + self.assertEqual(events.verify(self.conn), (5, None)) + self.assertEqual(self.kinds()[-1], "outside.recovered") + self.assertEqual(self.out.rows, [outside._row(r) for r in self.conn.execute( + "SELECT * FROM events WHERE seq <= 4 ORDER BY seq")]) + self.assertEqual(outside.ship(self.conn, self.out), 1) # and the chain goes on + self.assertIsNone(outside.status(self.conn)["alarm"]) + self.assertIsNone(outside.behind(self.conn, self.out)) + self.assertEqual(outside.recover(self.conn, self.out, "carol"), 0) + + def test_the_set_is_brought_up_to_the_recovered_events(self): + from console_backend.orch import certified + a, b = b'{"build_id": "a", "packages": []}', b'{"build_id": "b", "packages": []}' + with db.transaction(self.conn): + certified.change(self.conn, add={"lcg/a.json": a}) + outside.ship(self.conn, self.out) + copy = os.path.join(self.dir, "copy.db") + self.conn.execute("VACUUM INTO ?", (copy,)) # the backup + with db.transaction(self.conn): # a out, b in, since + certified.change(self.conn, add={"lcg/b.json": b}, remove={"lcg/a.json"}) + events.append(self.conn, "mirror.pushed", "certified", events.WORKER, + {"version": 2, "commit": "c2", "added": 1, "removed": 1}) + outside.ship(self.conn, self.out) + self.conn.close() + self.conn = db.connect(copy) # restored + with self.assertRaisesRegex(outside.OutsideError, "--mirror"): + outside.recover(self.conn, self.out, "carol") # b's bytes are not here + mirror = os.path.join(self.dir, "mirror") + os.makedirs(os.path.join(mirror, "manifests", "lcg")) + with open(os.path.join(mirror, "manifests", "lcg", "b.json"), "wb") as fh: + fh.write(b) + self.conn.execute("INSERT INTO boms (sha256, body, size, first_ms) VALUES (?,?,?,0)", + (__import__("hashlib").sha256(b).hexdigest(), b, len(b))) + with self.assertRaisesRegex(outside.OutsideError, "--mirror"): + outside.recover(self.conn, self.out, "carol") # held here is not enough + self.assertGreater(outside.recover(self.conn, self.out, "carol", mirror=mirror), 0) + live = [r[0] for r in self.conn.execute( + "SELECT path FROM certified WHERE removed_version IS NULL")] + self.assertEqual(live, ["lcg/b.json"]) # a stays out + self.assertEqual(list(json.loads(outside._kv(self.conn, "certify.decertified"))), ["a"]) + self.assertEqual((outside._kv(self.conn, "mirror.version"), certified.current(self.conn)[2]), + ("2", "c2")) # not pushed again + self.assertEqual(certified.current(self.conn)[0], 2) + with db.transaction(self.conn): # numbering goes on + self.assertEqual(certified.change(self.conn, remove={"lcg/b.json"})[0], 3) + + def _set_history(self, before, after): + """A database copied after *before*, the outside copy also after *after*; + returns the restored copy's path.""" + before(self.conn) + outside.ship(self.conn, self.out) + copy = os.path.join(self.dir, "copy.db") + self.conn.execute("VACUUM INTO ?", (copy,)) + after(self.conn) + outside.ship(self.conn, self.out) + self.conn.close() + self.conn = db.connect(copy) + + def test_a_build_decertified_in_one_arch_only_is_blocked(self): + from console_backend.orch import certified + el9, el8 = (b'{"build_id": "B", "packages": [{"hash": "h", "tarball_sha256": "s",' + b' "effective_architecture": "%s"}]}' % a for a in (b"el9", b"el8")) + + def before(conn): + with db.transaction(conn): + certified.change(conn, add={"lcg/B.el9.json": el9, "lcg/B.el8.json": el8}) + + def after(conn): + with db.transaction(conn): + certified.change(conn, remove={"lcg/B.el9.json"}) + self._set_history(before, after) + outside.recover(self.conn, self.out, "carol") + self.assertEqual(list(json.loads(outside._kv(self.conn, "certify.decertified"))), ["B"]) + + def test_a_set_that_does_not_match_its_digest_is_not_recovered(self): + from console_backend.orch import certified + + def after(conn): + with db.transaction(conn): + certified.change(conn, add={"lcg/a.json": b"{}"}) + self._set_history(lambda conn: None, after) + mirror = os.path.join(self.dir, "mirror") + os.makedirs(os.path.join(mirror, "manifests", "lcg")) + with open(os.path.join(mirror, "manifests", "lcg", "a.json"), "wb") as fh: + fh.write(b"{}") + with patch.object(certified, "digest_of", return_value="forged"): + with self.assertRaisesRegex(outside.OutsideError, "nothing recovered"): + outside.recover(self.conn, self.out, "carol", mirror=mirror) + self.assertEqual(certified.current(self.conn)[0], 0) + + def test_a_pipeline_given_up_on_is_followed_past(self): + from console_backend.orch import certified + + def after(conn): + certified._give_up(conn, 41, "c41", "test") + self._set_history(lambda conn: None, after) + outside.recover(self.conn, self.out, "carol") + self.assertEqual(certified.followed(self.conn), 41) + + def test_an_outside_copy_that_does_not_link_gives_nothing(self): + self.restored() + self.out.rows[3]["payload"] = '{"i":99}' # edited outside + with self.assertRaises(outside.OutsideError): + outside.recover(self.conn, self.out, "carol") + self.assertEqual(events.verify(self.conn), (2, None)) + + def test_a_database_that_diverged_is_not_recovered(self): + self.restored() + with db.transaction(self.conn): # started on the old copy + events.append(self.conn, "worker.started", "worker", events.WORKER, {}) + self.assertIsNone(outside.behind(self.conn, self.out)) # event 3 is not theirs + self.assertEqual(outside.recover(self.conn, self.out, "carol"), 0) + + def test_the_worker_does_not_start_on_a_restored_database(self): + from types import SimpleNamespace + from console_backend.orch import worker + self.restored() + w = SimpleNamespace(outside=self.out, conn=self.conn) + with self.assertRaises(outside.OutsideError) as cm: + worker.Worker._check_outside(w) + self.assertIn("recover", str(cm.exception)) + self.assertEqual(events.verify(self.conn), (2, None)) # nothing appended + with self.assertRaises(RuntimeError), db.transaction(self.conn): + events.append(self.conn, "approval.created", "x", events.WORKER, {}) # the console + outside.recover(self.conn, self.out, "carol") + worker.Worker._check_outside(w) + with db.transaction(self.conn): + events.append(self.conn, "worker.started", "worker", events.WORKER, {}) + + def test_signing_waits_for_a_comparison_after_start(self): + from types import SimpleNamespace + from console_backend.orch import worker + s = config.Settings(env={"BITS_ORCH_AUDIT_URL": "http://audit-log:8443"}) + self.add(1) + outside.ship(self.conn, self.out) + self.assertIsNone(outside.sign_hold(self.conn, s)) + + class Down: + def head(self): + raise outside.OutsideError("unreachable") + with self.assertLogs("bits_orchestrator", "WARNING"): + worker.Worker._check_outside(SimpleNamespace(outside=Down(), conn=self.conn)) + self.assertIn("not been checked", outside.sign_hold(self.conn, s)) + + def test_malformed_outside_events_are_not_imported(self): + self.restored() + r = dict(self.out.rows[2], actor_kind="nobody") + r["hash"] = events._digest(r["prev_hash"], r) + self.out.rows[2] = r + self.out.rows[3] = dict(self.out.rows[3], prev_hash=r["hash"]) + self.out.rows[3]["hash"] = events._digest(r["hash"], self.out.rows[3]) + with self.assertRaises(outside.OutsideError): + outside.recover(self.conn, self.out, "carol") + self.assertEqual(events.verify(self.conn), (2, None)) + + def test_signing_waits_for_the_outside_copy(self): + s = config.Settings(env={"BITS_ORCH_AUDIT_URL": "http://audit-log:8443"}) + now = db.now_ms() + self.assertIn("not been checked yet", outside.sign_hold(self.conn, s, now)) + self.add(1) + outside.ship(self.conn, self.out) + self.assertIsNone(outside.sign_hold(self.conn, s)) + self.assertIn("not been checked for", outside.sign_hold(self.conn, s, now + 11 * 60000)) + outside._alarm(self.conn, "rewritten", None) + self.assertIn("diverged (rewritten)", outside.sign_hold(self.conn, s)) + self.assertIsNone(outside.sign_hold(self.conn, config.Settings(env={}))) + + +class FakeHTTP: + def __init__(self, status, body): + self.status, self.body, self.calls = status, body, [] + + def request(self, method, url, headers, json, timeout, verify): + self.calls.append((method, url, headers, json, verify)) + st, body = self.status, self.body + + class R: + status_code = st + + def json(self): + return body + return R() + + +class ClientTest(unittest.TestCase): + def test_settings_token_file_and_responses(self): + self.assertIsNone(outside.Client.from_settings(config.Settings(env={}))) + with self.assertRaises(outside.OutsideError): + outside.Client.from_settings(config.Settings(env={"BITS_ORCH_AUDIT_URL": "https://a"})) + for url in ("http://cvmfs-bits-01.cern.ch:8443", "http://10.0.0.1:8443"): + with self.assertRaises(outside.OutsideError): + outside.Client.from_settings(config.Settings(env={"BITS_ORCH_AUDIT_URL": url, + "BITS_ORCH_AUDIT_TOKEN": "t"})) + c = outside.Client.from_settings(config.Settings(env={ # the service on this host + "BITS_ORCH_AUDIT_URL": "http://audit-log:8443", "BITS_ORCH_AUDIT_TOKEN": "t"})) + self.assertEqual(c.url, "http://audit-log:8443") + with self.assertRaises(outside.OutsideError): + outside.Client.from_settings(config.Settings(env={"BITS_ORCH_AUDIT_URL": "https://a", + "BITS_ORCH_AUDIT_TOKEN": "/nope"})) + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + tok = os.path.join(d, "t") + with open(tok, "w") as fh: + fh.write("sekrit\n") + c = outside.Client.from_settings(config.Settings(env={ + "BITS_ORCH_AUDIT_URL": "https://cvmfs-bits-01:8443/", "BITS_ORCH_AUDIT_TOKEN": tok, + "BITS_ORCH_AUDIT_CAFILE": "/ca.pem"})) + c.http = FakeHTTP(409, {"error": "x", "head": {"seq": 1}}) + ok, data = c.send([{"seq": 1}]) + self.assertEqual((ok, data["head"]["seq"]), (False, 1)) + method, url, headers, body, verify = c.http.calls[0] + self.assertEqual((method, url, headers["Authorization"], verify), + ("POST", "https://cvmfs-bits-01:8443/v1/events", "Bearer sekrit", "/ca.pem")) + c.http = FakeHTTP(401, {"error": "a bearer token is required"}) + with self.assertRaises(outside.OutsideError): + c.head() + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_phases.py b/console-backend/tests/test_orch_phases.py new file mode 100644 index 00000000..e3ccaf3e --- /dev/null +++ b/console-backend/tests/test_orch_phases.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""What a finished build's job spent its time on (the console's timeline).""" + +import json +import os +import shutil +import tempfile +import unittest + +from console_backend import forge_ops +from console_backend.orch import db, phases + +T0 = 1792600000 # seconds + + +def _iso(s): + import datetime + return datetime.datetime.fromtimestamp(s, datetime.timezone.utc).isoformat().replace("+00:00", "Z") + + +def _job(jid=7, start=T0 + 60, end=T0 + 1000, stage="build"): + return {"id": jid, "stage": stage, "name": "bits-prepub-build", "status": "success", + "started_at": _iso(start), "finished_at": _iso(end), "runner": "pcphsft204"} + + +def _rec(end, total, commit, outcome="published"): + return {"ts": _iso(end).replace("Z", ".000456789Z"), "total_s": total, "commit_s": commit, + "outcome": outcome} + + +class FakeReader: + def __init__(self, files=None, fail=None): + self.files, self.fail, self.calls = files or {}, fail, [] + + def job_artifact(self, job_id, name): + self.calls.append((job_id, name)) + if self.fail: + raise self.fail + return self.files.get((job_id, name)) + + +class SegmentsTest(unittest.TestCase): + def test_the_job_its_marks_and_prepubs_records(self): + m = phases.marks("build_start %d\nbuild_end %d\npublish_start %d\npublish_end %d\nnoise\n" + % (T0 + 100, T0 + 700, T0 + 710, T0 + 990)) + recs = [_rec(T0 + 900, 150, 50), _rec(T0 + 980, 200, 30), # overlapping packages + _rec(T0 + 990, 0, 0), {"ts": "bad"}, "x"] + segs = phases.segments((T0 + 10) * 1000, _job(), m, recs) + ms = lambda s: (T0 + s) * 1000 + self.assertEqual(segs, [["queued", ms(10), ms(60)], ["setup", ms(60), ms(100)], + ["build", ms(100), ms(700)], ["prepub", ms(750), ms(950)], + ["gateway", ms(850), ms(900)], ["gateway", ms(950), ms(980)]]) + + def test_without_artifacts_the_jobs_own_times(self): + ms = lambda s: (T0 + s) * 1000 + self.assertEqual(phases.segments(None, _job(), {}, []), [["build", ms(60), ms(1000)]]) + m = {"publish_start": ms(500), "publish_end": ms(900)} + self.assertEqual(phases.segments(ms(0), _job(), m, []), + [["queued", ms(0), ms(60)], ["build", ms(60), ms(500)], + ["publish", ms(500), ms(900)]]) + self.assertEqual(phases.segments(ms(0), {"id": 1}, {}, []), []) + + def test_within_the_jobs_times(self): + ms = lambda s: (T0 + s) * 1000 + m = {"build_start": ms(1200)} # the runner's clock ahead + recs = [_rec(T0 - 100, 50, 10), # an earlier attempt's package + {"ts": _iso(T0 + 990), "total_s": float("nan")}, _rec(T0 + 990, 100, -5)] + self.assertEqual(phases.segments(ms(0), _job(), m, recs), + [["queued", ms(0), ms(60)], ["setup", ms(60), ms(1000)], + ["prepub", ms(890), ms(990)], + ["build", ms(1000), ms(1000)]]) + + def test_where_a_failed_job_stopped(self): + failed = dict(_job(), status="failed") + m = lambda *marks: {k: T0 for k in marks} + self.assertEqual(phases.failed_in(failed, {}), "") # marks not kept + self.assertEqual(phases.failed_in(failed, m("build_start")), "build") + self.assertEqual(phases.failed_in(failed, m("build_start", "build_end")), "build") + self.assertEqual(phases.failed_in(failed, m("build_start", "build_end", "build_ok")), "certify") + self.assertEqual(phases.failed_in(failed, m("build_end", "build_ok", "publish_start")), "publish") + # the publish's own end is written before its finalize, which can fail + self.assertEqual(phases.failed_in(failed, m("build_ok", "publish_start", "publish_end")), "publish") + self.assertEqual(phases.failed_in(failed, m("build_ok", "publish_start", "publish_end", + "finish_start")), "finish") + self.assertEqual(phases.failed_in(failed, m("build_ok", "finish_start")), "finish") # no publish + self.assertEqual(phases.failed_in(_job(), m("build_start")), "") # it passed + + def test_the_latest_build_attempt(self): + jobs = [_job(5), _job(9), _job(11, stage="test"), dict(_job(12), started_at=None)] + self.assertEqual(phases.build_job(jobs)["id"], 9) + self.assertIsNone(phases.build_job([])) + + +class StepTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + def _pipeline(self, pid, jobs, outcome="success", seen=None): + with db.transaction(self.conn): + self.conn.execute("INSERT OR REPLACE INTO pipelines (id, is_build, outcome, jobs," + " created_at, first_seen_ms) VALUES (?, 1, ?, ?, ?, ?)", + (pid, outcome, json.dumps(jobs), _iso(T0), seen or db.now_ms())) + + def test_recorded_once_again_when_retried_and_served(self): + self._pipeline(100, [_job(7)]) + self._pipeline(101, [_job(8)], outcome="running") + self._pipeline(102, [_job(9)], seen=1) # too old: artifacts expired + reader = FakeReader({(7, phases.PHASES_FILE): b"build_start %d\n" % (T0 + 100), + (7, "measurements-100.json"): json.dumps( + [_rec(T0 + 990, 100, 40)]).encode()}) + self.assertEqual(phases.step(self.conn, reader), 1) + got = phases.for_pipelines(self.conn, [100, 101]) + self.assertEqual(list(got), [100]) + self.assertEqual(got[100]["host"], "pcphsft204") + self.assertEqual([s[0] for s in got[100]["segments"]], + ["queued", "setup", "build", "prepub", "gateway"]) + self.assertEqual(phases.step(self.conn, reader), 0) # done + self._pipeline(100, [_job(7), _job(13, start=T0 + 2000, end=T0 + 3000)]) + self.assertEqual(phases.step(self.conn, FakeReader()), 1) # retried: again, job 13 + self.assertEqual(phases.for_pipelines(self.conn, [100])[100]["segments"][-1][0], "build") + + def test_the_host_is_the_build_host_it_was_pinned_to(self): + self._pipeline(100, [_job(7)]) + self._pipeline(101, [_job(8)]) + self._pipeline(102, [_job(9)]) + with db.transaction(self.conn): + for pid, v in ((100, {"BITS_RUNNER_TAG": "bits-host-gjuan-desktop"}), + (101, {"BITS_RUNNER_TAG": "bits-build-x86_64"}), (102, {"BITS_RUNNER_TAG": "bits-host-"})): + self.conn.execute("UPDATE pipelines SET variables = ? WHERE id = ?", (json.dumps(v), pid)) + phases.step(self.conn, FakeReader()) + got = phases.for_pipelines(self.conn, [100, 101, 102]) + self.assertEqual([got[p]["host"] for p in (100, 101, 102)], + ["gjuan-desktop", "pcphsft204", "pcphsft204"]) # else the runner's description + self.assertEqual(phases.host({"runner": "r"}, "not json"), "r") + + def test_a_failed_publish_is_recorded_and_served(self): + self._pipeline(100, [dict(_job(7), status="failed")], outcome="failed") + reader = FakeReader({(7, phases.PHASES_FILE): b"build_start %d\nbuild_end %d\nbuild_ok %d\npublish_start %d\n" + % (T0 + 100, T0 + 500, T0 + 500, T0 + 600)}) + phases.step(self.conn, reader) + self.assertEqual(phases.for_pipelines(self.conn, [100])[100]["failed_in"], "publish") + self._pipeline(100, [dict(_job(7), status="failed"), _job(13, start=T0 + 2000, end=T0 + 3000)], + outcome="success") + phases.step(self.conn, FakeReader({(13, phases.PHASES_FILE): b"build_start %d\n" % (T0 + 2100)})) + self.assertEqual(phases.for_pipelines(self.conn, [100])[100]["failed_in"], "") # retried: passed + + def test_the_queues_request_time_and_gitlab_failing(self): + self._pipeline(100, [_job(7)]) + with db.transaction(self.conn): + self.conn.execute("INSERT INTO build_queue (batch, community, requested_by, ref," + " variables, digest, target, state, pipeline_id, created_ms," + " updated_ms) VALUES ('b', 'lcg', 'a', 'main', '[]', 'd'," + " 'bits-build-x86_64', 'dispatched', 100, ?, 0)", ((T0 - 600) * 1000,)) + with self.assertRaises(forge_ops.ForgeError): + phases.step(self.conn, FakeReader(fail=forge_ops.ForgeError(503, "down"))) + self.assertEqual(phases.for_pipelines(self.conn, [100]), {}) + phases.step(self.conn, FakeReader(fail=forge_ops.ForgeError(403, "no"))) # not transient + self.assertEqual(phases.for_pipelines(self.conn, [100])[100]["segments"][0], + ["queued", (T0 - 600) * 1000, (T0 + 60) * 1000]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_reconcile.py b/console-backend/tests/test_orch_reconcile.py new file mode 100644 index 00000000..b14efbe9 --- /dev/null +++ b/console-backend/tests/test_orch_reconcile.py @@ -0,0 +1,915 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M1: reconciling builds from GitLab and S3, the read API, +and the CI hints.""" + +import json +import os +import shutil +import tempfile +import unittest +import time +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import ci_auth, config, forge_ops, main +from console_backend.orch import certifier, db, events, kinds, reconcile, sbom_scan, worker + +# The pipelines' day: yesterday, so they stay inside the lookback window and +# short of being stale whenever the tests run. +DAY = time.strftime("%Y-%m-%d", time.gmtime(time.time() - 86400)) + + +def _job(i, name, status, stage="build", **kw): + return dict({"id": i, "name": name, "stage": stage, "status": status, + "allow_failure": False}, **kw) + + +class FakeReader: + """GitLab as the worker reads it: pipelines (by id), their variables, jobs.""" + + def __init__(self): + self.pipelines, self.variables, self.jobs = {}, {}, {} + self.calls = [] + + def add(self, pid, status="running", updated=DAY + "T10:00:00Z", source="api", + community="testbed", package="ROOT", jobs=None): + self.pipelines[pid] = {"id": pid, "status": status, "source": source, + "name": "[%s] %s 6.32 (el9)" % (community, package), + "ref": "main", "sha": "abc", "web_url": "https://gl/p/%d" % pid, + "created_at": DAY + "T09:00:00Z", "updated_at": updated} + self.variables[pid] = ([{"key": "COMMUNITY", "value": community.upper()}, + {"key": "PACKAGE", "value": package}, + {"key": "VERSION", "value": "6.32"}, + {"key": "PLATFORM", "value": "el9"}] if package else []) + self.jobs[pid] = jobs if jobs is not None else [_job(pid * 10, "build", "running")] + + def list_pipelines(self, updated_after, page=1, per_page=100): + self.calls.append(("list", updated_after, page)) + w = reconcile._when + got = sorted((p for p in self.pipelines.values() if w(p["updated_at"]) > w(updated_after)), + key=lambda p: (w(p["updated_at"]), p["id"])) + return [dict(p) for p in got[(page - 1) * per_page: page * per_page]] # a snapshot + + def get_pipeline(self, pid): + self.calls.append(("get", pid)) + if pid not in self.pipelines: + raise forge_ops.ForgeError(404, "404 Not found") + return dict(self.pipelines[pid]) + + def pipeline_variables(self, pid): + self.calls.append(("vars", pid)) + return self.variables[pid] + + def pipeline_jobs(self, pid): + self.calls.append(("jobs", pid)) + if isinstance(self.jobs.get(pid), Exception): + raise self.jobs[pid] + if pid not in self.pipelines: + raise forge_ops.ForgeError(404, "404 Not found") + return self.jobs[pid] + + +class FakeS3: + def __init__(self, objects=None, page=2): + self.objects = {k: json.dumps(v).encode() for k, v in (objects or {}).items()} + self.page = page + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise KeyError(Key) + + class B: + def __init__(s, d): + s.d = d + + def read(s): + return s.d + return {"Body": B(self.objects[Key])} + + def list_objects_v2(self, Bucket, Prefix, ContinuationToken=None): + keys = sorted(k for k in self.objects if k.startswith(Prefix)) + start = int(ContinuationToken or 0) + chunk = keys[start:start + self.page] + more = start + self.page < len(keys) + return {"Contents": [{"Key": k} for k in chunk], "IsTruncated": more, + "NextContinuationToken": str(start + self.page) if more else None} + + +def _bom(arch, n=2, build_id="b-1"): + return {"build_id": build_id, "effective_architecture": arch, "published_at": DAY + "T11:00:00Z", + "requested_packages": ["ROOT"], + "packages": [{"package": "p%d" % i, "hash": "h%d" % i} for i in range(n)]} + + +class _DB(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.gl = FakeReader() + + def _kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + def _row(self, pid): + return self.conn.execute("SELECT * FROM pipelines WHERE id = ?", (pid,)).fetchone() + + def _pass(self, store=None): + reconcile.apply_hints(self.conn, self.gl) + reconcile.discover(self.conn, self.gl) + reconcile.settle(self.conn, self.gl) + reconcile.index(self.conn, store) + + +class OutcomeTest(unittest.TestCase): + def test_outcome_comes_from_the_latest_job_attempts(self): + o = reconcile.outcome_from_jobs + self.assertEqual(o("running", [_job(1, "build", "running")]), "running") + self.assertEqual(o("running", [_job(1, "build", "success"), + _job(2, "status", "success", stage="status")]), "success") + # bits' progress status holds the pipeline running after every job ended. + self.assertEqual(o("running", [_job(1, "build", "success")]), "success") + # GitLab failed it on a timed-out status, but every job succeeded. + self.assertEqual(o("failed", [_job(1, "build", "success")]), "success") + # A failed build decides, even with a later job wedged. + self.assertEqual(o("running", [_job(1, "build", "failed"), + _job(2, "status", "waiting_for_resource", stage="status")]), + "failed") + # A retried job: the latest attempt counts. + self.assertEqual(o("running", [_job(1, "build", "failed"), _job(3, "build", "running")]), + "running") + self.assertEqual(o("success", [_job(1, "build", "failed"), _job(3, "build", "success")]), + "success") + self.assertEqual(o("failed", [_job(1, "build", "success"), + _job(2, "lint", "failed", stage="test", allow_failure=True)]), + "success") + self.assertEqual(o("canceled", [_job(1, "build", "canceled")]), "canceled") + self.assertEqual(o("pending", []), "running") + self.assertEqual(o("failed", []), "failed") + + +class ReconcileTest(_DB): + def test_a_build_is_seen_settled_and_its_builds_recorded(self): + self.gl.add(1) + self._pass() + r = self._row(1) + self.assertEqual((r["is_build"], r["community"], r["package"], r["outcome"]), + (1, "testbed", "ROOT", "running")) + self.assertEqual(self._kinds(), ["pipeline.seen"]) + # It finishes; the build index is in S3. + self.gl.add(1, status="success", updated=DAY + "T12:00:00Z", + jobs=[_job(10, "build", "success", runner={"description": "bits-build-01"})]) + s3 = FakeS3({"BUILDS/by-pipeline/1/b-1.json": {"build_id": "b-1", + "keys": ["MANIFESTS/b-1/x.el9.json", + "MANIFESTS/b-1/x.share.json"]}, + "MANIFESTS/b-1/x.el9.json": _bom("el9", 3), + "MANIFESTS/b-1/x.share.json": _bom("share", 1)}, page=1) + self._pass((s3, "bkt")) + r = self._row(1) + self.assertEqual(r["outcome"], "success") + self.assertEqual(json.loads(r["jobs"])[0]["runner"], "bits-build-01") + got = reconcile.list_builds(self.conn)[0]["builds"] + self.assertEqual(got, [{"build_id": "b-1", "state": "published", "arches": ["el9", "share"], + "packages": 4, "top_level": ["ROOT"], + "published_at": DAY + "T11:00:00Z"}]) + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.success", "build.published"]) + # Nothing new: nothing is written again. + self._pass((s3, "bkt")) + self.assertEqual(len(self._kinds()), 3) + self.assertEqual(events.verify(self.conn)[1], None) + + def test_variables_are_read_once_and_non_builds_are_kept_aside(self): + self.gl.add(1) + self.gl.add(2, source="push", package="") # Pages / push pipeline + self.gl.add(3, package="") # an api pipeline that is no build + self._pass() + self._pass() + self.assertEqual([c for c in self.gl.calls if c[0] == "vars"], [("vars", 1), ("vars", 3)]) + self.assertEqual([b["pipeline"] for b in reconcile.list_builds(self.conn)], [1]) + + def test_the_cursor_moves_and_overlaps(self): + self.gl.add(1, updated=DAY + "T10:00:00Z") + reconcile.discover(self.conn, self.gl) + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.updated_after"), + DAY + "T10:00:00Z") + self.gl.calls.clear() + reconcile.discover(self.conn, self.gl) + self.assertEqual(self.gl.calls[0][1], DAY + "T09:55:00Z") # 5 minutes back + + def _count(self): + return self.conn.execute("SELECT count(*) FROM pipelines WHERE id != 0").fetchone()[0] + + def test_many_pages_continue_on_the_next_pass(self): + for i in range(1, 251): + self.gl.add(i, updated=DAY + "T10:%02d:%02dZ" % (i // 60, i % 60)) + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 100) + reconcile.discover(self.conn, self.gl, max_rounds=1) + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 250) + + def test_more_than_a_page_in_one_second(self): + for i in range(1, 251): + self.gl.add(i, updated=DAY + "T10:00:00Z") + for _ in range(4): + reconcile.discover(self.conn, self.gl, max_rounds=1) + self.assertEqual(self._count(), 250) + reconcile.discover(self.conn, self.gl) # and then it settles down + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.continue"), "0") + + def test_a_pipeline_updating_while_pages_are_read_is_not_skipped(self): + for i in range(1, 151): + self.gl.add(i, status="success", updated=DAY + "T10:%02d:%02dZ" % (i // 60, i % 60), + jobs=[_job(i * 10, "build", "success")]) + real = self.gl.list_pipelines + + def shifting(updated_after, page=1, per_page=100): + got = real(updated_after, page, per_page) + if 1 in self.gl.pipelines and self.gl.pipelines[1]["updated_at"] < DAY + "T11": + self.gl.pipelines[1]["updated_at"] = DAY + "T11:00:00Z" # moves to the end + return got + self.gl.list_pipelines = shifting + reconcile.discover(self.conn, self.gl) + self.assertEqual(self._count(), 150) + + def test_times_with_offsets_and_fractions(self): + # After the clocks go back: the later update has the smaller local time. + self.gl.add(1, updated="2026-10-25T02:50:00.250+02:00") # 00:50 UTC + self.gl.add(2, updated="2026-10-25T02:10:00+01:00", community="x") # 01:10 UTC + reconcile.discover(self.conn, self.gl) + self.assertEqual(reconcile._kv_get(self.conn, "gitlab.updated_after"), + "2026-10-25T01:10:00Z") + + def test_a_running_pipeline_is_rechecked_at_most_every_two_minutes(self): + self.gl.add(1) + self._pass() + self.gl.calls.clear() + self._pass() + self.assertNotIn(("jobs", 1), self.gl.calls) + self.conn.execute("UPDATE pipelines SET checked_ms = 1") + self._pass() + self.assertIn(("jobs", 1), self.gl.calls) + + def test_a_failed_build_with_a_wedged_status_job(self): + self.gl.add(1, status="running", jobs=[ + _job(10, "build", "failed", failure_reason="script_failure"), + _job(11, "update-status", "waiting_for_resource", stage="status")]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "failed") + e = events.since(self.conn)[-1] + self.assertEqual((e["kind"], e["payload"]["failed_jobs"]), ("pipeline.failed", ["build"])) + # Later updates to the wedged job do not flap the outcome. + self.gl.add(1, status="running", updated=DAY + "T13:00:00Z", + jobs=self.gl.jobs[1]) + self._pass() + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.failed"]) + + def test_a_retry_runs_again_and_publishes(self): + self.gl.add(1, status="failed", jobs=[_job(10, "build", "failed")]) + self._pass() + self.gl.add(1, status="running", updated=DAY + "T13:00:00Z", + jobs=[_job(10, "build", "failed"), _job(12, "build", "running")]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "running") + self.gl.add(1, status="success", updated=DAY + "T14:00:00Z", + jobs=[_job(10, "build", "failed"), _job(12, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "M/b.json": _bom("el9", build_id="b")}) + self._pass((s3, "bkt")) + self.assertEqual(self._kinds(), ["pipeline.seen", "pipeline.failed", "pipeline.running", + "pipeline.success", "build.published"]) + + def test_several_builds_in_one_pipeline_and_bad_index_entries(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/opt.json": {"build_id": "opt", "keys": ["M/o.json"]}, + "BUILDS/by-pipeline/1/dbg.json": {"build_id": "dbg", + "keys": ["M/d.json", "M/missing.json"]}, + "BUILDS/by-pipeline/1/junk.json": ["not", "an", "index"], + "BUILDS/by-pipeline/10/x.json": {"build_id": "other", "keys": []}, + "M/o.json": _bom("el9-opt", build_id="opt"), + "M/d.json": _bom("el9-dbg", build_id="dbg")}) + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass((s3, "bkt")) + got = {b["build_id"]: b["arches"] for b in reconcile.list_builds(self.conn)[0]["builds"]} + self.assertEqual(got, {"opt": ["el9-opt"], "dbg": ["el9-dbg"]}) + + def test_no_index_yet_is_retried_then_given_up(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + s3 = FakeS3({}) + self._pass((s3, "bkt")) + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) + s3.objects.update({k: json.dumps(v).encode() for k, v in { + "BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "M/b.json": _bom("el9", build_id="b")}.items()}) + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) # not yet due + self.conn.execute("UPDATE pipelines SET indexed_ms = 1") + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 1) + # Long after it finished, a miss is not looked at again (without a hint). + self.gl.add(2, status="success", jobs=[_job(20, "build", "success")]) + self._pass((s3, "bkt")) + self.conn.execute("UPDATE pipelines SET indexed_ms = 1, checked_ms = 1 WHERE id = 2") + s3.objects["BUILDS/by-pipeline/2/c.json"] = json.dumps( + {"build_id": "c", "keys": ["M/b.json"]}).encode() + self.assertEqual(reconcile.index(self.conn, (s3, "bkt")), 0) + reconcile.add_hint(self.conn, 2, "published") + self._pass((s3, "bkt")) + self.assertEqual(len(reconcile.list_builds(self.conn, pipelines=[2])[0]["builds"]), 1) + + def test_a_running_pipelines_builds_are_recorded_once_in_the_store(self): + self.gl.add(3, status="running", jobs=[_job(30, "build", "running")]) + s3 = FakeS3({}) + self._pass((s3, "bkt")) # nothing there yet + self.assertEqual(self._row(3)["outcome"], "running") + s3.objects.update({k: json.dumps(v).encode() for k, v in { + "BUILDS/by-pipeline/3/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "M/b.json": _bom("el9", build_id="b")}.items()}) + reconcile.add_hint(self.conn, 3, "published") # after the BOM step + self._pass((s3, "bkt")) + self.assertEqual([b["build_id"] for b in reconcile.list_builds(self.conn, pipelines=[3])[0]["builds"]], + ["b"]) + self.assertEqual(self._row(3)["outcome"], "running") + + def test_a_hint_fetches_a_pipeline_older_than_the_window(self): + self.gl.add(7, status="success", updated="2020-01-01T00:00:00Z", + jobs=[_job(70, "build", "success")]) + self._pass() + self.assertIsNone(self._row(7)) + reconcile.add_hint(self.conn, 7, "published") + self._pass() + self.assertEqual(self._row(7)["outcome"], "success") + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + + def test_hints_gitlab_does_not_know_are_dropped_others_kept(self): + reconcile.add_hint(self.conn, 99, "failed") # GitLab: 404 + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + self.gl.get_pipeline = lambda pid: (_ for _ in ()).throw(forge_ops.ForgeError(502, "x")) + reconcile.add_hint(self.conn, 98, "failed") # GitLab unwell: kept + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 1) + self.conn.execute("UPDATE hints SET ts_ms = 1") # a day later: dropped + with self.assertLogs("bits_orchestrator", "WARNING"): + reconcile.apply_hints(self.conn, self.gl) + self.assertEqual(self.conn.execute("SELECT count(*) FROM hints").fetchone()[0], 0) + + def test_one_pipeline_gitlab_cannot_answer_for_does_not_hold_up_the_others(self): + self.gl.add(1) + self.gl.add(2, updated=DAY + "T10:01:00Z") + self._pass() + del self.gl.pipelines[1] # deleted on GitLab + self.gl.jobs[2] = [_job(20, "build", "success")] + self.conn.execute("UPDATE pipelines SET checked_ms = 1") + self._pass() + self.assertEqual((self._row(1)["outcome"], self._row(2)["outcome"]), ("canceled", "success")) + gone = [e for e in events.since(self.conn) if e["subject"] == "pipeline:1"][-1] + self.assertTrue(gone["payload"]["gone"]) + self.gl.add(3, updated=DAY + "T10:02:00Z") + self.gl.jobs[3] = forge_ops.ForgeError(502, "bad gateway") + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass() + self.assertEqual(self._row(3)["outcome"], "running") + self.assertGreater(self._row(3)["checked_ms"], 0) # backs off, tried later + + def test_a_manual_job_does_not_hold_a_pipeline_running(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success"), + _job(11, "deploy", "manual", stage="deploy", + allow_failure=True)]) + self._pass() + self.assertEqual(self._row(1)["outcome"], "success") + + def test_a_pipeline_running_for_days_is_settled_from_gitlab(self): + self.gl.add(1) + self.gl.pipelines[1]["created_at"] = "2026-01-01T00:00:00Z" + self._pass() + self.assertEqual(self._row(1)["outcome"], "failed") + self.assertTrue(events.since(self.conn)[-1]["payload"]["stale"]) + + def test_a_failing_index_records_nothing_and_does_not_block_others(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + self.gl.add(2, status="success", updated=DAY + "T10:01:00Z", + jobs=[_job(20, "build", "success")]) + s3 = FakeS3({"BUILDS/by-pipeline/1/a.json": {"build_id": "a", "keys": ["M/a.json"]}, + "BUILDS/by-pipeline/1/b.json": {"build_id": "b", "keys": ["M/b.json"]}, + "BUILDS/by-pipeline/2/c.json": {"build_id": "c", "keys": ["M/c.json"]}, + "M/a.json": _bom("el9", build_id="a"), "M/c.json": _bom("el9", build_id="c")}) + real = reconcile._bom_facts + + def boom(s3_, bucket, keys): + if keys == ["M/b.json"]: + raise RuntimeError("store hiccup") + return real(s3_, bucket, keys) + with patch.object(reconcile, "_bom_facts", boom), \ + self.assertLogs("bits_orchestrator", "WARNING"): + self._pass((s3, "bkt")) + got = {b["pipeline"]: [x["build_id"] for x in b["builds"]] + for b in reconcile.list_builds(self.conn)} + self.assertEqual(got, {1: [], 2: ["c"]}) # all of 1 or none of it + reconcile.add_hint(self.conn, 1, "published") + self._pass((s3, "bkt")) + got = [x["build_id"] for x in reconcile.list_builds(self.conn, pipelines=[1])[0]["builds"]] + self.assertEqual(got, ["a", "b"]) + + def test_unreadable_variables_fall_back_to_the_name(self): + self.gl.add(1, community="lcg", package="ROOT") + self.gl.pipelines[1]["name"] = "[LCG] ROOT 6.32.08 (x86_64-el9) [nightly]" + self.gl.add(2, community="lcg", package="", updated=DAY + "T10:01:00Z") + self.gl.pipelines[2]["name"] = "Pages" # no build + def refuse(pid): + raise forge_ops.ForgeError(403, "GET /pipelines/%s/variables -> 403" % pid) + self.gl.pipeline_variables = refuse + with self.assertLogs("bits_orchestrator", "WARNING"): + self._pass() + r = self._row(1) + self.assertEqual((r["is_build"], r["community"], r["package"], r["version"], r["platform"]), + (1, "lcg", "ROOT", "6.32.08", "x86_64-el9")) + self.assertEqual(self._row(2)["is_build"], 0) + self.assertEqual(reconcile._from_name("[ALICE] O2 (el9)")["VERSION"], "") + + def test_variables_are_read_with_the_ops_token(self): + s = config.Settings(env={"GITLAB_API_URL": "https://gl/api/v4", "BITS_FORGE_PROJECT": "42", + "BITS_FORGE_READ_TOKEN": "ro", "BITS_FORGE_OPS_TOKEN": "ops"}) + r = kinds.Deps(s).reader() + self.assertEqual(r._token, "ro") + self.assertEqual(r.pipeline_variables.__self__._token, "ops") + s.forge_ops_token = "" + self.assertFalse(hasattr(kinds.Deps(s).reader().pipeline_variables, "__self__") + and kinds.Deps(s).reader().pipeline_variables.__self__._token == "ops") + + def test_variable_names_are_case_insensitive(self): + self.gl.add(1) + self.gl.variables[1] = [{"key": "community", "value": "TestBed"}, + {"key": "Package", "value": "ROOT"}] + self._pass() + self.assertEqual((self._row(1)["community"], self._row(1)["package"]), ("testbed", "ROOT")) + + def test_the_listing_without_finished_at_does_not_count_as_a_change(self): + self.gl.add(1, status="success", jobs=[_job(10, "build", "success")]) + self._pass() + self.gl.pipelines[1]["finished_at"] = DAY + "T10:30:00Z" + reconcile.record_pipeline(self.conn, self.gl, self.gl.get_pipeline(1)) + self.assertEqual(self._row(1)["finished_at"], DAY + "T10:30:00Z") + listed = {k: v for k, v in self.gl.pipelines[1].items() if k != "finished_at"} + before = self._row(1)["checked_ms"] + reconcile.record_pipeline(self.conn, self.gl, listed) + self.assertEqual((self._row(1)["checked_ms"], self._row(1)["finished_at"]), + (before, DAY + "T10:30:00Z")) + + def test_list_filters(self): + self.gl.add(1, community="testbed") + self.gl.add(2, community="other", updated=DAY + "T10:01:00Z") + self.gl.add(3, community="testbed", status="failed", updated=DAY + "T10:02:00Z", + jobs=[_job(30, "build", "failed")]) + self._pass() + ids = lambda **kw: [b["pipeline"] for b in reconcile.list_builds(self.conn, **kw)] + self.assertEqual(ids(), [3, 2, 1]) + self.assertEqual(ids(communities={"testbed"}), [3, 1]) + self.assertEqual(ids(communities=set()), []) + self.assertEqual(ids(outcome="failed"), [3]) + self.assertEqual(ids(pipelines=[1, 2]), [2, 1]) + detail = reconcile.get_build(self.conn, 3) + self.assertEqual((detail["jobs"][0]["status"], detail["variables"]["PACKAGE"]), + ("failed", "ROOT")) + self.assertEqual([e["kind"] for e in detail["events"]], ["pipeline.seen", "pipeline.failed"]) + + +class CliBuildTest(_DB): + """A build made with the CLI is recorded once the worker certified it.""" + + def _certified(self, bid, star, archs, group="LCG", at=None): + at = db.now_ms() if at is None else at + aid = self.conn.execute( + "INSERT INTO approvals (kind, subject, groups, approver, via, created_ms, expires_ms)" + " VALUES ('preapproval', ?, ?, 'buncic', 'cli', 0, ?)", + (bid, json.dumps([group]), at + 86400000)).lastrowid + for arch, state, key in [("*", star, None)] + archs: + self.conn.execute( + "INSERT INTO certify_runs (pipeline_id, build_id, arch, state, approval_id, boms," + " created_ms, updated_ms) VALUES (0, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT" + " (pipeline_id, build_id, arch) DO UPDATE SET state = excluded.state," + " approval_id = excluded.approval_id, boms = excluded.boms," + " updated_ms = excluded.updated_ms", + (bid, arch, state, aid, json.dumps([[key, "s"]] if key else []), at, at)) + self.conn.commit() + + def test_a_certified_cli_build_is_recorded_and_scanned(self): + self._certified("cli-1", "signed", [("el9-opt", "signed", "M/cli-1/r.el9-opt.json"), + ("el9", "signed", "M/cli-1/r.el9.json")]) + self._certified("cli-2", "refused", [("el9-opt", "signed", "M/cli-2/r.el9-opt.json")]) + self._certified("cli-3", "signing", [("el9-opt", "signed", "M/cli-3/r.el9-opt.json")]) + s3 = FakeS3({"M/cli-1/r.el9-opt.json": _bom("el9-opt", 3, "cli-1"), + "M/cli-1/r.el9.json": _bom("el9", 1, "cli-1")}) + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 1) + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 0) # once + b = self.conn.execute("SELECT * FROM builds").fetchall() + self.assertEqual([(r["pipeline_id"], r["build_id"], r["community"], r["packages"]) + for r in b], [(0, "cli-1", "lcg", 4)]) + self.assertEqual(json.loads(b[0]["arches"]), ["el9", "el9-opt"]) + self.assertIn("build.published", self._kinds()) + with patch.object(sbom_scan, "SETTLE_MS", 0): + self.assertEqual(sbom_scan.due(self.conn, "db", db.now_ms() + 1), ["cli-1"]) + sbom_scan._failed(self.conn, "cli-1", "boom") # its scan is listed + self.assertEqual([(r["build_id"], r["community"]) for r in sbom_scan.listing(self.conn, {"lcg"})], + [("cli-1", "lcg")]) + # Not a pipeline: neither listed as one nor taken for a certifiable build. + self.assertEqual(reconcile.list_builds(self.conn), []) + self.conn.execute("INSERT INTO kv (key, value) VALUES ('certify.since', '0')") + self.conn.commit() + on = config.Settings(env={"BITS_ORCH_CERTIFY": "1"}) + self.assertEqual(certifier.trigger(self.conn, on, self.gl), 0) + self.assertEqual(self.conn.execute("SELECT state FROM certify_runs WHERE build_id = 'cli-1'" + " AND arch = '*'").fetchone()[0], "signed") + + + def test_certified_again_it_is_recorded_from_that_approval(self): + # Refused with el9 already signed, then approved again: el9 of the old run is not this one's. + self._certified("cli-1", "refused", [("el9", "signed", "M/old.el9.json")], at=1) + self._certified("cli-1", "signed", [("el9-opt", "signed", "M/new.el9-opt.json")], at=2) + s3 = FakeS3({"M/old.el9.json": _bom("el9", 5, "cli-1"), + "M/new.el9-opt.json": _bom("el9-opt", 3, "cli-1")}) + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 1) + row = lambda: self.conn.execute("SELECT * FROM builds WHERE build_id = 'cli-1'").fetchone() + self.assertEqual((json.loads(row()["bom_keys"]), row()["packages"]), (["M/new.el9-opt.json"], 3)) + # Published and certified again later: recorded anew, so scanned again. + s3.objects["M/again.el9-opt.json"] = json.dumps(_bom("el9-opt", 4, "cli-1")).encode() + self._certified("cli-1", "signed", [("el9-opt", "signed", "M/again.el9-opt.json")], + at=db.now_ms() + 1000) + with patch.object(db, "now_ms", return_value=db.now_ms() + 2000): + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 1) + self.assertEqual(row()["packages"], 4) + + def test_a_bom_not_read_is_tried_again(self): + self._certified("cli-1", "signed", [("el9-opt", "signed", "M/cli-1.json")]) + s3 = FakeS3({}) + with self.assertLogs("bits_orchestrator", "WARNING"): + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 0) + s3.objects["M/cli-1.json"] = json.dumps(_bom("el9-opt", 2, "cli-1")).encode() + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 1) + with patch.object(reconcile, "INDEX_GIVE_UP_MS", -1): # and once given up, without it + self._certified("cli-2", "signed", [("el9-opt", "signed", "M/gone.json")]) + with self.assertLogs("bits_orchestrator", "WARNING"): + self.assertEqual(reconcile.index_cli(self.conn, (s3, "bkt")), 1) + + +class WorkerStepsTest(_DB): + def test_the_worker_runs_reconcile_before_actions(self): + w = worker.Worker(self.conn, config.Settings(env={})) + self.assertEqual([n for n, _ in w.steps], ["actions", "hints", "discover", "settle", "index", + "index_cli", "collect", "backfill", "phases", "trace", "ship", "backup"]) + w.deps.reader = lambda: self.gl + w.deps.store = lambda: None + self.gl.add(1) + w.run_once() + self.assertEqual(self._row(1)["outcome"], "running") + + def test_the_scan_step_follows_trace_when_a_scanner_is_set(self): + w = worker.Worker(self.conn, config.Settings(env={"BITS_ORCH_SCANNER_URL": "http://s:8080"})) + names = [n for n, _ in w.steps] + self.assertEqual(names[names.index("trace") + 1:names.index("trace") + 3], ["scan", "history"]) + + def test_the_cvmfs_status_step_follows_index_when_asked_for(self): + w = worker.Worker(self.conn, config.Settings(env={"BITS_ORCH_CVMFS_STATUS": "1"})) + names = [n for n, _ in w.steps] + self.assertEqual(names[names.index("index") + 1], "cvmfs_status") + w.steps = [s for s in w.steps if s[0] == "cvmfs_status"] + with self.assertLogs("bits_orchestrator", "ERROR"): + w.run_once() # no ops token, no store: says so + got = [e for e in events.since(self.conn) if e["kind"] == "worker.step_failed"] + self.assertIn("BITS_ORCH_CVMFS_STATUS needs", got[0]["payload"]["error"]) + + def test_unconfigured_gitlab_is_one_event(self): + w = worker.Worker(self.conn, config.Settings(env={})) + w.steps = [s for s in w.steps if s[0] == "hints"] + with self.assertLogs("bits_orchestrator", "ERROR"): + w.run_once() + w.run_once() + got = [e for e in events.since(self.conn) if e["kind"] == "worker.step_failed"] + self.assertEqual(len(got), 1) + self.assertIn("GitLab not configured", got[0]["payload"]["error"]) + + def test_the_store_is_built_from_the_real_s3_backend(self): + from bits_helpers import sync + d = kinds.Deps(config.Settings(env={"BITS_ORCH_STORE": "b3://bits-store::rw"})) + with patch.object(sync.Boto3RemoteSync, "_s3_init", + lambda self: setattr(self, "s3", "client")): + self.assertEqual(d.store(), ("client", "bits-store")) + + def test_store_settings(self): + d = kinds.Deps(config.Settings(env={})) + self.assertIsNone(d.store()) + d = kinds.Deps(config.Settings(env={"BITS_ORCH_STORE": "https://x/y"})) + with self.assertRaises(kinds.NotConfigured): + d.store() + + +class ApiTest(_DB): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", + "BITS_ADMINS_POLICY": "testbed @alice\nother @bob\n* @carol", + "BITS_FORGE_PROJECT": "42", "BITS_ORCH_DB": self.path, + "BITS_OIDC_ISSUER": "https://gl", "BITS_OIDC_CI_AUDIENCE": "bits", + "BITS_OIDC_JWKS_URL": "https://gl/jwks"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + self.gl.add(1, community="testbed") + self.gl.add(2, community="other", updated=DAY + "T10:01:00Z") + self._pass() + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer %s" % user}) + + def test_certifications_are_those_of_ones_communities(self): + now = 1 + self.conn.execute("INSERT INTO actions (kind, params, idempotency_key, subject, community," + " state, requested_by_kind, requested_by_id, attempts, error, created_ms," + " updated_ms) VALUES ('certify', '{}', 'k', 'certify:2:b3', 'other'," + " 'approved', 'worker', 'w', 2, 'OSError: down', 1, 1)") + for pid, bid, arch, state, reason in ((1, "b1", "*", "signing", "store down"), + (2, "b3", "*", "signing", ""), + (2, "b2", "*", "signed", ""), (0, "cli", "*", "signed", "")): + self.conn.execute("INSERT INTO certify_runs (pipeline_id, build_id, arch, state, reason," + " created_ms, updated_ms) VALUES (?,?,?,?,?,?,?)", + (pid, bid, arch, state, reason, now, now)) + self.conn.commit() + got = lambda u: [(r["build_id"], r["reason"]) for r in self._get(u, "/ops/certify").json()["certify"]] + self.assertEqual(got("alice"), [("b1", "")]) # no raw reasons + self.assertEqual(got("carol"), [("cli", ""), ("b2", ""), + ("b3", "retried (attempt 2): OSError: down"), ("b1", "store down")]) + self.assertEqual(got("dave"), []) + self.assertIs(self._get("alice", "/ops/certify").json()["owns_set"], False) + + def test_builds_are_those_of_ones_communities(self): + ids = lambda u, q="": [b["pipeline"] for b in self._get(u, "/ops/builds" + q).json()["builds"]] + self.assertEqual(ids("alice"), [1]) + self.assertEqual(ids("bob"), [2]) + self.assertEqual(ids("carol"), [2, 1]) + self.assertEqual(ids("dave"), []) # no community named + self.assertEqual(ids("dave", "?community=testbed"), [1]) # any signed-in user reads one + self.assertEqual(ids("alice", "?community=other"), [2]) + self.assertEqual(ids("carol", "?community=OTHER"), [2]) + self.assertEqual(ids("carol", "?pipelines=1,99"), [1]) + self.assertEqual(self._get("alice", "/ops/builds?pipelines=x").status_code, 400) + self.assertEqual(self._get("alice", "/ops/builds?outcome=weird").status_code, 400) + self.assertEqual(self.client.get("/ops/builds").status_code, 401) + + def test_one_build_with_its_jobs_and_events(self): + r = self._get("alice", "/ops/builds/1") + self.assertEqual((r.status_code, r.json()["package"]), (200, "ROOT")) + self.assertEqual([e["kind"] for e in r.json()["events"]], ["pipeline.seen"]) + self.assertIn("variables", r.json()) + r = self._get("bob", "/ops/builds/1") # not its admin: read-only + self.assertEqual((r.status_code, r.json()["package"]), (200, "ROOT")) + self.assertNotIn("events", r.json()) + self.assertNotIn("variables", r.json()) + self.assertIn("jobs", r.json()) + with db.transaction(self.conn): + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, arches, bom_keys," + " top_level, packages, published_at, recorded_ms) VALUES" + " (1, 'b1', 'testbed', '[]', '[]', '[]', 0, '', 0)") + self.conn.execute("INSERT INTO certify_runs (pipeline_id, build_id, arch, state, reason," + " approver, created_ms, updated_ms) VALUES (1, 'b1', '*', 'signed', ''," + " 'alice', 1, 1)") + cert = lambda u, q: self._get(u, q).json() + self.assertEqual(cert("alice", "/ops/builds/1")["builds"][0]["certify"]["approver"], "alice") + for u, q in (("bob", "/ops/builds/1"), ("dave", "/ops/builds?community=TESTBED")): + b = cert(u, q) + b = (b["builds"][0] if "pipeline" in b else b["builds"][0]["builds"][0]) + self.assertEqual(b["certify"]["state"], "signed") + self.assertNotIn("approver", b["certify"]) + self.assertEqual(self._get("carol", "/ops/builds/77").status_code, 404) + + def test_trace_names_the_publishing_pipeline_to_its_community(self): + with db.transaction(self.conn): + self.conn.execute("INSERT INTO published_paths (repo, path, build_id, job_id, ts_ms)" + " VALUES ('r', 'a/ROOT/6-1', '1', 'j', 1)") + q = "/ops/trace?repo=r&path=/a/ROOT/6-1/lib/x.so" + (pub,) = self._get("alice", q).json()["publishes"] + self.assertEqual((pub["pipeline"], pub["visible"], pub["package"]), (1, True, "ROOT")) + (pub,) = self._get("bob", q).json()["publishes"] + self.assertEqual(pub, dict(pub, pipeline=1, visible=False)) + self.assertNotIn("approval", pub) + self.assertEqual(self.client.get(q).status_code, 401) + + def test_a_builds_sbom_for_any_signed_in_user(self): + from console_backend.orch import api + + class Missing(Exception): + response = {"Error": {"Code": "NoSuchKey"}} + + class S3(FakeS3): + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise Missing(Key) + return FakeS3.get_object(self, Bucket, Key) + rec = lambda pid, bid: self.conn.execute( + "INSERT INTO builds (pipeline_id, build_id, community, arches, bom_keys, top_level," + " packages, published_at, recorded_ms) VALUES (?,?,?,?,?,?,?,?,?)", + (pid, bid, "testbed", "[]", "[]", "[]", 0, "", 0)) + rec(1, "b-1") + self.conn.commit() + s3 = S3({"MANIFESTS/b-1/sbom.cdx.json": {"bomFormat": "CycloneDX"}, + "MANIFESTS/b-1/sbom.spdx.json": {"spdxVersion": "SPDX-2.3"}}) + with patch.object(api, "_store", lambda: (s3, "bkt")): + r = self._get("dave", "/ops/builds/1/sbom") # administers nothing + self.assertEqual((r.status_code, r.json()), (200, {"bomFormat": "CycloneDX"})) + r = self._get("bob", "/ops/builds/1/sbom?format=spdx") + self.assertEqual(r.json(), {"spdxVersion": "SPDX-2.3"}) + self.assertEqual(self.client.get("/ops/builds/1/sbom").status_code, 401) + self.assertEqual(self._get("dave", "/ops/builds/1/sbom?format=x").status_code, 400) + self.assertEqual(self._get("dave", "/ops/builds/2/sbom").status_code, 404) + rec(1, "b-2") + self.conn.commit() + r = self._get("dave", "/ops/builds/1/sbom") # several builds: say which + self.assertEqual(r.status_code, 400) + self.assertIn("b-1, b-2", r.json()["detail"]) + self.assertEqual(self._get("dave", "/ops/builds/1/sbom?build=b-2").status_code, 404) + with patch.object(api, "_store", lambda: None): + self.assertEqual(self._get("dave", "/ops/builds/1/sbom?build=b-1").status_code, 503) + + def down(): + raise OSError("credentials proxy unavailable") + with patch.object(api, "_store", down): + self.assertEqual(self._get("dave", "/ops/builds/1/sbom?build=b-1").status_code, 503) + + def test_risk_history_is_that_of_ones_communities(self): + from console_backend.orch import sbom_scan + for pid, bid in ((1, "b-1"), (2, "b-2")): + self.conn.execute( + "INSERT INTO builds (pipeline_id, build_id, community, recorded_ms) VALUES" + " (?, ?, (SELECT community FROM pipelines WHERE id = ?), 0)", (pid, bid, pid)) + self.conn.execute("INSERT INTO sbom_scans (build_id, db_built, scanned_ms, index_version)" + " VALUES (?, 'd', 1, ?)", (bid, sbom_scan.INDEX_VERSION)) + self.conn.commit() + got = lambda u, q="": [r["community"] for r in self._get(u, "/ops/sbom/history" + q).json()["releases"]] + self.assertEqual(got("alice"), ["testbed"]) + self.assertEqual(got("carol"), ["other", "testbed"]) + self.assertEqual(got("carol", "?community=other&days=0"), ["other"]) + self.assertEqual(got("dave"), []) + self.assertEqual(self.client.get("/ops/sbom/history").status_code, 401) + + def test_scans_are_those_of_ones_communities_and_their_findings_their_admins(self): + from console_backend.orch import api, sbom_scan + for pid, bid in ((1, "b-1"), (2, "b-2")): + self.conn.execute( + "INSERT INTO builds (pipeline_id, build_id, community, recorded_ms) VALUES" + " (?, ?, (SELECT community FROM pipelines WHERE id = ?), 0)", (pid, bid, pid)) + self.conn.execute("INSERT INTO sbom_scans (build_id, db_built, scanned_ms, high)" + " VALUES ('b-1', 'd', 1, 2)") + self.conn.execute("INSERT INTO sbom_scans (build_id, db_built, scanned_ms) VALUES ('b-2', 'd', 1)") + self.conn.commit() + sbom_scan._write(main.settings.orch_scans_dir, "b-1", [{"id": "CVE-9"}]) + ids = lambda u, q="": [s["build_id"] for s in self._get(u, "/ops/sbom/scans" + q).json()["scans"]] + self.assertEqual(ids("alice"), ["b-1"]) + self.assertEqual(ids("carol"), ["b-1", "b-2"]) + self.assertEqual(ids("carol", "?flagged=true"), ["b-1"]) + self.assertEqual(ids("alice", "?community=other"), []) + self.assertEqual(ids("dave"), []) + self.assertEqual(self.client.get("/ops/sbom/scans").status_code, 401) + s3 = FakeS3({"MANIFESTS/b-1/sbom.cdx.json": {"bomFormat": "CycloneDX"}}) + with patch.object(api, "_store", lambda: (s3, "bkt")): + seen = [self._get(u, "/ops/builds/1/sbom?vulnerabilities=1") for u in ("alice", "carol", "bob")] + self.assertEqual([(r.status_code, r.json().get("vulnerabilities")) for r in seen], + [(200, [{"id": "CVE-9"}]), (200, [{"id": "CVE-9"}]), (403, None)]) # testbed's admins + + def test_packages_findings_and_their_triage_for_the_communities_admins(self): + from console_backend.orch import api, sbom_scan + from test_orch_sbom_scan import SBOM, VULNS + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms)" + " VALUES (1, 'b-1', 'testbed', 0)") + self.conn.commit() + sbom_scan._scanned(self.conn, main.settings.orch_scans_dir, "b-1", "sha", "d", {"high": 2}, + VULNS, SBOM) + post = lambda u, path, body: self.client.post(path, json=body, + headers={"Authorization": "Bearer %s" % u}) + rule = {"community": "testbed", "vuln_id": "CVE-A", "package": "zlib", + "state": "not_affected", "justification": "code_not_present", "reason": "no gzip"} + self.assertEqual(post("dave", "/ops/sbom/triage", rule).status_code, 403) + self.assertEqual(post("bob", "/ops/sbom/triage", rule).status_code, 403) + self.assertEqual(post("alice", "/ops/sbom/triage", dict(rule, state="fixed")).status_code, 400) + rid = post("alice", "/ops/sbom/triage", rule).json()["id"] + self.assertEqual([r["id"] for r in self._get("carol", "/ops/sbom/triage").json()["rules"]], [rid]) + self.assertEqual(self._get("bob", "/ops/sbom/triage").json()["rules"], []) + + (scan,) = self._get("alice", "/ops/sbom/scans").json()["scans"] + self.assertEqual((scan["effective"]["high"], scan["suppressed"], scan["identified"]), (1, 1, 3)) + rows = self._get("alice", "/ops/sbom/packages?name=libxml2&version=2.9").json()["packages"] + self.assertEqual([(r["build_id"], r["component_version"]) for r in rows], [("b-1", "2.9.13")]) + self.assertEqual(self._get("bob", "/ops/sbom/packages?name=libxml2").json()["packages"], []) + self.assertEqual(self._get("alice", "/ops/sbom/packages").status_code, 400) + self.assertIn("zlib", self._get("alice", "/ops/sbom/names").json()["names"]) + plan = self._get("alice", "/ops/sbom/plan").json() + self.assertEqual([(r["build_id"], [w["wave"] for w in r["waves"]]) for r in plan["releases"]], + [("b-1", [0])]) # CVE-A is known exploited + self.assertEqual([c["name"] for c in plan["converged"]], ["libxml2"]) + self.assertEqual(self._get("bob", "/ops/sbom/plan").json(), {"releases": [], "converged": []}) + (lic,) = self._get("alice", "/ops/sbom/licences").json()["releases"] + self.assertEqual((lic["build_id"], lic["open"]), ("b-1", {"conflict": 0, "unknown": 4})) + lic = self._get("alice", "/ops/sbom/licences/b-1").json() + self.assertEqual(sorted({f["kind"] for f in lic["findings"]}), ["unknown"]) + self.assertEqual(self._get("bob", "/ops/sbom/licences").json(), {"releases": []}) + self.assertEqual(self._get("bob", "/ops/sbom/licences/b-1").status_code, 404) + self.assertEqual([r["build_id"] for r in self._get("alice", "/ops/sbom/dependencies").json()["releases"]], + ["b-1"]) + dep = self._get("alice", "/ops/sbom/dependencies/b-1").json() + self.assertEqual((dep["top"], dep["previous"]), (["ROOT", "numpy"], None)) + self.assertEqual(self._get("bob", "/ops/sbom/dependencies/b-1").status_code, 404) + found = self._get("alice", "/ops/sbom/vulnerabilities/CVE-A").json()["builds"] + self.assertEqual(sorted((r["name"], r["suppressed"]) for r in found), + [("libxml2", None), ("zlib1g", rid)]) + + s3 = FakeS3({"MANIFESTS/b-1/sbom.cdx.json": SBOM}) + sbom_scan._write(main.settings.orch_scans_dir, "b-1", VULNS) + with patch.object(api, "_store", lambda: (s3, "bkt")): + self.assertEqual(self._get("dave", "/ops/builds/1/sbom?vulnerabilities=1").status_code, 403) + self.assertEqual(self._get("alice", "/ops/builds/1/sbom?format=spdx&vulnerabilities=1") + .status_code, 400) + doc = self._get("alice", "/ops/builds/1/sbom?vulnerabilities=1").json() + self.assertEqual([(v["id"], v.get("analysis", {}).get("state")) for v in doc["vulnerabilities"]], + [("CVE-A", None), ("CVE-A", "not_affected"), ("CVE-B", None), ("CVE-C", None)]) + self.assertIn({"name": "bits:community", "value": "testbed"}, doc["metadata"]["properties"]) + self.assertIn({"name": "bits:scanned", "value": "d"}, doc["metadata"]["properties"]) + self.assertNotIn("vulnerabilities", self._get("dave", "/ops/builds/1/sbom").json()) + self.conn.execute("UPDATE sbom_packages SET not_shipped = 'tool' WHERE ref = 'libxml2@1'") + self.conn.commit() + doc = self._get("alice", "/ops/builds/1/sbom?vulnerabilities=1").json() + self.assertEqual([(v["id"], v["analysis"]["state"], v["properties"]) for v in doc["vulnerabilities"] + if v["id"] == "CVE-B"], + [("CVE-B", "not_affected", [{"name": "bits:not_shipped", "value": "tool"}])]) + s3.objects["MANIFESTS/b-1/sbom.cdx.json"] = json.dumps(dict(SBOM, vulnerabilities=[{"id": "FORGED"}])).encode() + os.unlink(sbom_scan._path(main.settings.orch_scans_dir, "b-1")) + doc = self._get("alice", "/ops/builds/1/sbom?vulnerabilities=1").json() + self.assertNotIn("vulnerabilities", doc) # only the scan's + self.assertIn({"name": "bits:scanned", "value": ""}, doc["metadata"]["properties"]) + + self.assertEqual(post("bob", "/ops/sbom/triage/%d/revoke" % rid, {}).status_code, 403) + self.assertEqual(post("alice", "/ops/sbom/triage/%d/revoke" % rid, {}).status_code, 200) + self.assertEqual(post("alice", "/ops/sbom/triage/%d/revoke" % rid, {}).status_code, 409) + self.assertEqual(post("alice", "/ops/sbom/triage/999/revoke", {}).status_code, 404) + self.assertEqual(self._get("alice", "/ops/sbom/scans").json()["scans"][0]["suppressed"], 0) + + def _ci(self, claims, body, kind="published"): + seen = {} + + def verify(t, s, key=None, audience=None): + seen["aud"] = audience + return claims + with patch.object(ci_auth, "verify_ci_token", verify): + r = self.client.post("/ops/ci/" + kind, json=body, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(seen.get("aud"), "bits-orchestrator") # never the signing audience + return r + + def test_ci_hints_need_an_audience_of_their_own(self): + good = {"project_id": "42", "ref_protected": "true", "pipeline_id": "5"} + main.settings.orch_ci_audience = "" + with patch.object(ci_auth, "verify_ci_token", lambda *a, **k: good): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 503) + main.settings.orch_ci_audience = main.settings.oidc_ci_audience + with patch.object(ci_auth, "verify_ci_token", lambda *a, **k: good): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 503) + + def test_ci_hints_need_the_build_projects_token_for_that_pipeline(self): + good = {"project_id": "42", "project_path": "bits/console", "ref_protected": "true", + "pipeline_id": "5"} + self.assertEqual(self._ci(good, {"pipeline": 5}).json(), {"status": "noted"}) + with patch.object(ci_auth, "_jwks_client") as jw: # the real check: its own audience + import jwt as _jwt + with patch.object(_jwt, "decode", lambda *a, **k: dict(good, aud=k["audience"])): + jw.return_value.get_signing_key_from_jwt.return_value.key = "k" + c = ci_auth.verify_ci_token("t", main.settings, audience="bits-orchestrator") + self.assertEqual(c["aud"], "bits-orchestrator") + self.assertEqual(self._ci(good, {"pipeline": 5}, "failed").status_code, 200) + self.assertEqual(self.conn.execute("SELECT kind FROM hints").fetchall()[0][0], "failed") + self.assertEqual(self._ci(dict(good, project_id="7", project_path="x/y"), + {"pipeline": 5}).status_code, 403) + self.assertEqual(self._ci(dict(good, ref_protected="false"), {"pipeline": 5}).status_code, + 403) + self.assertEqual(self._ci(good, {"pipeline": 6}).status_code, 403) + self.assertEqual(self._ci(good, {"pipeline": "x"}).status_code, 400) + self.assertEqual(self.client.post("/ops/ci/published", json={"pipeline": 5}).status_code, + 401) + + def bad(t, s, key=None, audience=None): + raise Exception("bad signature") + with patch.object(ci_auth, "verify_ci_token", bad): + r = self.client.post("/ops/ci/published", json={"pipeline": 5}, + headers={"Authorization": "Bearer jwt"}) + self.assertEqual(r.status_code, 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_replay.py b/console-backend/tests/test_orch_replay.py new file mode 100644 index 00000000..a50e2265 --- /dev/null +++ b/console-backend/tests/test_orch_replay.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""M8a: the worker's own certification of a build, replayed beside the CI's.""" + +import json +from unittest.mock import patch + +from bits_helpers import trust +from console_backend import config +from console_backend.orch import db, shadow + +from test_orch_shadow import SETTINGS, _Base, _bom + + +class NoSuchKey(Exception): + response = {"Error": {"Code": "NoSuchKey"}} + + +class FakeS3: + """The store's copies of the builds' BOMs (MANIFESTS//...).""" + + def __init__(self): + self.docs = {} + + def list_objects_v2(self, Bucket, Prefix, **kw): + return {"Contents": [{"Key": k} for k in sorted(self.docs) if k.startswith(Prefix)]} + + def get_object(self, Bucket, Key): + import io + if Key not in self.docs: + raise NoSuchKey() + return {"Body": io.BytesIO(json.dumps(self.docs[Key]).encode())} + + +class FakeConsole: + def __init__(self): + self.files = {} + + def file_raw(self, path, ref, max_bytes=0): + return self.files.get((path, ref)) + + +class ReplayTest(_Base): + def setUp(self): + super().setUp() + self.settings = config.Settings(env=dict(SETTINGS, BITS_ADMINS_POLICY="lcg @alice\n* @carol")) + self.s3, self.console = FakeS3(), FakeConsole() + self.bom = _bom("b2", "el9", ("q", "h2")) # source_pipeline_id 7 + self.store = {("el9", "h2"): "sh2"} + self.s3.docs["MANIFESTS/b2/el9.json"] = dict(self.bom) + self.gl.trees["aaaa000"] = {} + self.gl.trees["aaaa111"] = {"manifests/lcg/b2.el9.host-1.json": + dict(self.bom, certified_by=["bot"])} + now = db.now_ms() + self.conn.execute("INSERT INTO pipelines (id, is_build, community, sha, variables," + " first_seen_ms) VALUES (7, 1, 'lcg', 'c0ffee', ?, ?)", + (json.dumps({"COMMUNITY": "LCG"}), now)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, bom_keys, recorded_ms)" + " VALUES (7, 'b2', '[\"MANIFESTS/b2/el9.json\"]', ?)", (now,)) + self.approve("alice") + + def approve(self, user, subject="7", created=None, expires=None, revoked=None): + now = db.now_ms() + self.conn.execute("INSERT INTO approvals (kind, subject, groups, approver, created_ms," + " expires_ms, revoked_ms) VALUES ('preapproval', ?, '[\"lcg\"]', ?, ?, ?, ?)", + (subject, user, created or now - 1000, expires or now + 3600_000, revoked)) + + def replay(self, archs=("el9",), started=None): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", list(archs))) + self.gl.jobs[5][0]["started_at"] = started # None: now + shadow.discover(self.conn, self.gl, self.settings) + while shadow.compare_one(self.conn, self.gl, (self.s3, "bucket"), self.settings, + self.cache, trusted=self.trusted, + probe_factory=lambda c: self.probe, + console=self.console): + pass + return self.run_rows(5, "replay") + + def test_the_worker_certifies_what_the_ci_did(self): + rows = self.replay() + self.assertEqual({a: r["state"] for a, r in rows.items()}, {"el9": "match"}) + self.assertEqual(self.run_rows(5)["el9"]["state"], "match") + self.assertIn("replay.match", self.kinds()) + + def test_no_preapproval_no_certification(self): + self.conn.execute("DELETE FROM approvals") + self.approve("alice", expires=db.now_ms() - 1) # expired before the CI signed + self.approve("alice", revoked=db.now_ms() - 1) # revoked + self.approve("alice", subject="8") # another build's + row = self.replay()["*"] + self.assertEqual(row["state"], "refused") + self.assertIn("no console pre-approval", json.loads(row["diff"])["error"]) + self.assertIn("replay.refused", self.kinds()) + + def test_the_preapprover_must_administer_every_group(self): + self.conn.execute("DELETE FROM approvals") + self.approve("bob") + self.assertIn("bob (who pre-approved pipeline 7) is not an admin for lcg", + json.loads(self.replay()["*"]["diff"])["error"]) + + def test_the_boms_must_be_under_the_builds_group(self): + self.console.files[("communities/LCG/ui-config.yaml", "c0ffee")] = b"certify_group: stacks\n" + row = self.replay()["*"] + self.assertEqual(row["state"], "refused") + self.assertIn("certifies into stacks", json.loads(row["diff"])["error"]) + + def test_without_the_store_copy_there_is_nothing_to_certify(self): + self.s3.docs.clear() + row = self.replay()["*"] + self.assertEqual(row["state"], "missing") + self.assertIn("no store copy", json.loads(row["diff"])["error"]) + + def test_cli_builds_are_not_the_workers_in_m8a(self): + bom = {k: v for k, v in self.bom.items() if k != "source_pipeline_id"} + self.gl.trees["aaaa111"] = {"manifests/lcg/b2.el9.host-1.json": bom} + self.assertIn("CLI build", json.loads(self.replay()["*"]["diff"])["error"]) + + def test_the_store_copy_is_what_the_worker_certifies(self): + self.s3.docs["MANIFESTS/b2/el9.json"] = dict(self.bom, extra="store") + row = self.replay()["el9"] + # A field the common manifest does not carry: the same result, but noted. + self.assertEqual(row["state"], "match") + self.assertEqual(json.loads(row["diff"])["replay"]["store_differs_from_mr"], + ["MANIFESTS/b2/el9.json"]) + # Other bytes claimed for the package: the store refutes the worker's input. + self.s3.docs["MANIFESTS/b2/el9.json"]["packages"][0]["tarball_sha256"] = "sh-other" + self.conn.execute("DELETE FROM sign_runs") + self.conn.execute("UPDATE certifications SET state = 'pending'") + while shadow.compare_one(self.conn, self.gl, (self.s3, "bucket"), self.settings, + self.cache, trusted=self.trusted, + probe_factory=lambda c: self.probe, console=self.console): + pass + rows = self.run_rows(5, "replay") + self.assertEqual({a: r["state"] for a, r in rows.items()}, {"*": "refused"}) + self.assertIn("the worker refuses", json.loads(rows["*"]["diff"])["error"]) + + def test_no_key_policy_is_a_refusal(self): + with patch.object(trust, "load_key_policy", lambda dirs=None: None): + row = self.replay()["*"] + self.assertIn("fails closed", json.loads(row["diff"])["error"]) + + def test_only_the_pipelines_own_store_copy_counts(self): + # The same build id, built again by another pipeline: its copy is not this one's. + self.s3.docs["MANIFESTS/b2/zzhost-2026.el9.json"] = dict(self.bom, source_pipeline_id=9) + self.assertEqual(self.replay()["el9"]["state"], "match") + + def test_the_pushs_deletions_are_applied(self): + old = _bom("b1", "el9", ("p", "h1")) + self.gl.trees["aaaa000"] = {"manifests/lcg/b1.el9.host-0.json": old} + self.store[("el9", "h1")] = "sh1" + self.assertEqual({a: r["state"] for a, r in self.replay().items()}, {"el9": "match"}) + + def test_the_approval_must_be_in_force_when_the_ci_signed(self): + started = "2026-10-11T09:30:00Z" + at = shadow._ms(started) + self.conn.execute("DELETE FROM approvals") + self.approve("alice", created=at + 1000) # granted after the CI started + self.assertEqual(self.replay(started=started)["*"]["state"], "refused") + self.conn.execute("DELETE FROM approvals") + self.conn.execute("DELETE FROM sign_runs") + self.conn.execute("UPDATE certifications SET state = 'pending'") + self.approve("alice", created=at - 1000, expires=at + 3600_000, + revoked=at + 1000) # revoked after it signed + self.assertEqual(self.replay(started=started)["el9"]["state"], "match") + + def test_an_overall_admin_may_preapprove_any_group(self): + self.conn.execute("DELETE FROM approvals") + self.approve("carol") + self.assertEqual(self.replay()["el9"]["state"], "match") diff --git a/console-backend/tests/test_orch_sbom_history.py b/console-backend/tests/test_orch_sbom_history.py new file mode 100644 index 00000000..28dfbf9b --- /dev/null +++ b/console-backend/tests/test_orch_sbom_history.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Each release's risk over time (console_backend.orch.sbom_history).""" + +import datetime +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from console_backend.orch import db, sbom_history, sbom_scan, vuln_triage + + +def _ms(day): + d = datetime.datetime.strptime(day, "%Y-%m-%d").replace(hour=12, tzinfo=datetime.timezone.utc) + return int(d.timestamp() * 1000) + + +class HistoryTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + for pid, community, package in ((1, "lcg", "LCG"), (2, "lcg", "LCG"), (3, "alice", "O2")): + self.conn.execute("INSERT INTO pipelines (id, is_build, community, package, first_seen_ms)" + " VALUES (?, 1, ?, ?, 0)", (pid, community, package)) + # (pipeline, build, community, top_level, published, findings: (vuln, severity, risk, shipped)) + for pid, bid, community, top, day, findings in ( + (1, "lcg_opt-aaaaaaaaaaaa", "lcg", [], "2026-09-01", + [("CVE-1", "critical", 80.0, True), ("CVE-2", "high", 0.0, True), + ("CVE-T", "high", 40.0, True), ("CVE-S", "medium", 9.0, False)]), + (2, "lcg_dbg-bbbbbbbbbbbb", "lcg", [], "2026-10-01", [("CVE-2", "high", 30.0, True)]), + (0, "o2physics-cccccccccccc", "alice", ["O2Physics"], "2026-10-05", + [("CVE-3", "low", 1.0, True)])): + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, top_level," + " published_at, recorded_ms) VALUES (?,?,?,?,?,0)", + (pid, bid, community, json.dumps(top), day + "T10:00:00Z")) + self.conn.execute("INSERT INTO sbom_scans (build_id, db_built, scanned_ms, index_version)" + " VALUES (?, 'd', 1, ?)", (bid, sbom_scan.INDEX_VERSION)) + for vuln, sev, risk, shipped in findings: + ref = "ref-" + vuln + self.conn.execute("INSERT OR IGNORE INTO sbom_packages (build_id, ref, name, version," + " not_shipped) VALUES (?, ?, 'zlib', '1.3', ?)", + (bid, ref, "" if shipped else "tool")) + self.conn.execute("INSERT INTO sbom_findings (build_id, vuln_id, ref, severity, risk)" + " VALUES (?,?,?,?,?)", (bid, vuln, ref, sev, risk)) + self.conn.commit() + vuln_triage.add(self.conn, "lcg", "CVE-T", "not_affected", "not reachable", "bob", + justification="code_not_reachable") + + def _points(self, out, top): + (r,) = [r for r in out["releases"] if r["top"] == top] + return [(p["day"], p["build_id"][:7], p["critical"], p["high"], p["medium"], p["risk"]) + for p in r["points"]] + + def test_the_first_record_has_the_builds_at_their_publish_days(self): + self.assertEqual(sbom_history.snapshot(self.conn, now=_ms("2026-10-10")), 5) # 2 + 3 days + out = sbom_history.history(self.conn, now=_ms("2026-10-10")) + # Triaged and not shipped findings left out; a finding without grype's risk by severity. + self.assertEqual(self._points(out, "LCG"), [ + ("2026-09-01", "lcg_opt", 1, 1, 0, 105.0), + ("2026-10-01", "lcg_dbg", 0, 1, 0, 30.0), + ("2026-10-10", "lcg_dbg", 0, 1, 0, 30.0)]) + (now,) = [r for r in out["releases"] if r["top"] == "LCG"][0]["points"][-1:] + self.assertEqual(now["risks"], {"critical": 0.0, "high": 30.0, "medium": 0.0, "low": 0.0, + "other": 0.0}) + (first,) = [r for r in out["releases"] if r["top"] == "LCG"][0]["points"][:1] + self.assertEqual(first["risks"]["critical"], 80.0) # recorded, by severity + # A build of no pipeline: its release is the package it asked for. + self.assertEqual([(r["community"], r["top"]) for r in out["releases"]], + [("alice", "O2Physics"), ("lcg", "LCG")]) + self.assertEqual([m["build_id"][:7] for m in out["builds"]], ["lcg_opt", "lcg_dbg", "o2physi"]) + + def test_a_day_is_recorded_once_and_then_stands_for_that_day(self): + self.assertEqual(sbom_history.snapshot(self.conn, now=_ms("2026-10-02")), 4) + self.assertEqual(sbom_history.snapshot(self.conn, now=_ms("2026-10-02")), 0) # once a day + # A new vulnerability in the newest build after that day: the records keep the old risk. + self.conn.execute("INSERT INTO sbom_findings (build_id, vuln_id, ref, severity, risk)" + " VALUES ('lcg_dbg-bbbbbbbbbbbb', 'CVE-9', 'ref-CVE-2', 'critical', 70)") + self.conn.commit() + out = sbom_history.history(self.conn, now=_ms("2026-10-10")) + self.assertEqual(self._points(out, "LCG"), [ + ("2026-09-01", "lcg_opt", 1, 1, 0, 105.0), + ("2026-10-01", "lcg_dbg", 0, 1, 0, 30.0), + ("2026-10-02", "lcg_dbg", 0, 1, 0, 30.0), # the record + ("2026-10-10", "lcg_dbg", 1, 1, 0, 100.0)]) # today, afresh + # The next day: the newest build of each release, and the alice build of 10-05 is + # not before it. + self.assertEqual(sbom_history.snapshot(self.conn, now=_ms("2026-10-03")), 2) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM sbom_history").fetchone()[0], 6) + + def test_a_build_scanned_after_the_first_record_gets_its_day(self): + self.conn.execute("UPDATE sbom_scans SET db_built = '' WHERE build_id = 'lcg_opt-aaaaaaaaaaaa'") + self.conn.commit() + sbom_history.snapshot(self.conn, now=_ms("2026-10-02")) + self.conn.execute("UPDATE sbom_scans SET db_built = 'd'") # scanned now + self.conn.commit() + sbom_history.snapshot(self.conn, now=_ms("2026-10-03")) + days = [p[0] for p in self._points(sbom_history.history(self.conn, now=_ms("2026-10-03")), "LCG")] + self.assertEqual(days, ["2026-09-01", "2026-10-01", "2026-10-02", "2026-10-03"]) + + def test_the_earlier_days_are_recorded_a_few_a_pass(self): + with patch.object(sbom_history, "BACKFILL", 1): + # Today's 2 each pass, and 1 of the 3 earlier days, until none is left. + got = [sbom_history.snapshot(self.conn, now=_ms("2026-10-10")) for _ in range(4)] + self.assertEqual(got, [3, 3, 3, 0]) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM sbom_history").fetchone()[0], 5) + + def test_a_build_of_two_pipelines_is_of_the_newest_s_release(self): + self.conn.execute("INSERT INTO pipelines (id, is_build, community, package, first_seen_ms)" + " VALUES (5, 1, 'alice', 'O2', 0)") + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, published_at," + " recorded_ms) VALUES (5, 'lcg_opt-aaaaaaaaaaaa', 'alice', '2026-10-07T00:00:00Z', 0)") + self.conn.commit() + got = sbom_history._builds(self.conn)["lcg_opt-aaaaaaaaaaaa"] + self.assertEqual(got, ("alice", "O2", "2026-09-01")) # the day it was first published + + def test_communities_and_days_narrow_it(self): + sbom_history.snapshot(self.conn, now=_ms("2026-10-10")) + out = sbom_history.history(self.conn, communities={"alice"}, days=30, now=_ms("2026-10-10")) + self.assertEqual([r["top"] for r in out["releases"]], ["O2Physics"]) + out = sbom_history.history(self.conn, communities={"lcg"}, days=20, now=_ms("2026-10-10")) + self.assertEqual([p[0] for p in self._points(out, "LCG")], ["2026-10-01", "2026-10-10"]) + self.assertEqual(sbom_history.history(self.conn, communities=set()), + {"releases": [], "builds": []}) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_sbom_licences.py b/console-backend/tests/test_orch_sbom_licences.py new file mode 100644 index 00000000..a8d5de4c --- /dev/null +++ b/console-backend/tests/test_orch_sbom_licences.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""orch.sbom_licences: licence and dependency analysis of the scanned builds.""" + +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import db, sbom_licences as L, sbom_scan, vuln_triage + + +def comp(name, licence="MIT", version="1", runtime=(), build=(), **props): + c = {"bom-ref": name + "@1", "name": name, "version": version, + "properties": [{"name": "bits:" + k, "value": v} for k, v in props.items()] + + [{"name": "bits:build_requires", "value": b} for b in build]} + if licence: + c["licenses"] = [{"expression": licence}] + return c, {"ref": name + "@1", "dependsOn": [d + "@1" for d in runtime]} + + +def sbom(*parts, top=("app",)): + return {"bomFormat": "CycloneDX", "components": [c for c, _ in parts], + "metadata": {"component": {"bom-ref": "release", "name": "r"}}, + "dependencies": [{"ref": "release", "dependsOn": [t + "@1" for t in top]}] + + [d for _, d in parts]} + + +STACK = (comp("app", "GPL-2.0-only", runtime=["libfoo", "CMake", "py"], build=["gcc"]), + comp("libfoo", "Apache-2.0", runtime=["libgpl3"]), + comp("libgpl3", "GPL-3.0-or-later"), + comp("CMake", "BSD-3-Clause", runtime=["agpltool"]), # a build tool: not followed + comp("agpltool", "AGPL-3.0-only"), + comp("py", "LicenseRef-bits-Python", redistributable="none"), + comp("gcc", "GPL-3.0-or-later WITH GCC-exception-3.1"), # build only + comp("orphan", ""), + comp("zlib", "", provided_by="system")) + + +class ClassTest(unittest.TestCase): + def test_classes(self): + cls = L.licence_class + self.assertEqual([cls(x) for x in ("MIT", "GPL-2.0-only", "GPL-2.0", "GPL-2.0-or-later", + "GPL-2.0+", "GPL-3.0", "AGPL-3.0", "LGPL-2.1-only", + "MPL-2.0", "LicenseRef-x", "", "NOASSERTION")], + ["permissive", "gpl2only", "gpl2only", "gpl2plus", "gpl2plus", "gpl3", "agpl", + "lgpl", "weak", "custom", "unknown", "unknown"]) + for unreviewed in ("GNU GPL v2", "Proprietary", "GPL", "GPLv3", "MIT AND", "(MIT"): + self.assertEqual(cls(unreviewed), "custom", unreviewed) + self.assertEqual(cls("GPL-3.0-only OR MIT"), "permissive") + self.assertEqual(cls("MIT AND GPL-3.0-only"), "gpl3") + self.assertEqual(cls("(GPL-2.0-only)"), "gpl2only") + self.assertEqual(cls("GPL-2.0-only AND (MIT OR Apache-2.0)"), "gpl2only") + self.assertEqual(cls("(MIT OR Apache-2.0) AND GPL-3.0-only"), "gpl3") + self.assertEqual(cls("GPL-3.0-or-later WITH GCC-exception-3.1"), "permissive") + self.assertEqual(cls("GPL-2.0-only WITH Linux-syscall-note"), "gpl2only") + + def test_conflicts(self): + self.assertTrue(L.conflict("GPL-2.0-only", "GPL-3.0-only")) + self.assertTrue(L.conflict("GPL-3.0-only", "GPL-2.0")) + self.assertTrue(L.conflict("GPL-2.0-only", "Apache-2.0")) + self.assertTrue(L.conflict("GPL-2.0-only", "Apache-2.0 AND MIT")) + self.assertIsNone(L.conflict("GPL-2.0-only", "Apache-2.0 OR MIT")) # MIT chosen + for dep in ("GPL-2.0-only OR GPL-3.0-only", "GPL-3.0-only OR GPL-2.0-only"): + self.assertIsNone(L.conflict("GPL-3.0-only", dep), dep) # either order + self.assertIsNone(L.conflict("GPL-2.0-only", "LGPL-2.1-only OR Apache-2.0")) + self.assertIsNone(L.conflict("GPL-2.0-only", "MIT AND (Apache-2.0 OR GPL-2.0-only)")) + self.assertIsNone(L.conflict("GPL-2.0-only OR GPL-3.0-only", "Apache-2.0")) + self.assertIsNone(L.conflict("GPL-2.0-only", "Apache-2.0 WITH LLVM-exception")) + self.assertEqual([L.licence_class(x) for x in ("mit", "apache-2.0", "MPL-2.0+")], + ["permissive", "permissive", "weak"]) + self.assertIsNone(L.conflict("GPL-2.0-or-later", "Apache-2.0")) + self.assertIsNone(L.conflict("MIT", "GPL-3.0-only")) + self.assertTrue(L.restricted("none") and L.restricted("sources") and not L.restricted("all")) + + +class AnalysisTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.dir = os.path.join(d, "scans") + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.n = 0 + + def build(self, bid, doc, community="lcg", platform="x86_64-el9"): + self.n += 1 + self.conn.execute("INSERT INTO pipelines (id, is_build, community, package, version, platform," + " first_seen_ms) VALUES (?, 1, ?, 'LCG', '1', ?, 0)", (self.n, community, platform)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms, published_at)" + " VALUES (?, ?, ?, ?, ?)", (self.n, bid, community, self.n, "2026-10-%02d" % self.n)) + self.conn.commit() + sbom_scan._scanned(self.conn, self.dir, bid, "sha%d" % self.n, "db", {}, [], doc) + + def test_indexed(self): + self.build("r-aaaaaaaaaaaa", sbom(*STACK)) + g = L.graph(self.conn, "r-aaaaaaaaaaaa") + app = g["app@1"] + self.assertEqual((app["license"], app["top"], app["runtime"]), + ("GPL-2.0-only", 1, ["CMake@1", "libfoo@1", "py@1"])) + self.assertEqual(app["links"], ["CMake@1", "gcc@1", "libfoo@1", "py@1"]) # build links too + self.assertEqual((g["py@1"]["redistributable"], g["libfoo@1"]["top"]), ("none", 0)) + + def test_licences(self): + self.build("r-aaaaaaaaaaaa", sbom(*STACK)) + rules = vuln_triage.Rules([]) + out = L.licences(L.graph(self.conn, "r-aaaaaaaaaaaa"), "lcg", rules) + (con,) = [f for f in out["findings"] if f["kind"] == "conflict"] + self.assertEqual((con["package"], con["more"]), ("app", 0)) + self.assertEqual([(u["package"], u["path"]) for u in con["using"]], + [("libfoo", ["app", "libfoo"]), ("libgpl3", ["app", "libfoo", "libgpl3"])]) # not agpltool + self.assertEqual(sorted(f["package"] for f in out["findings"] if f["kind"] == "unknown"), + ["orphan", "py"]) # not the system zlib + (reach,) = out["copyleft"] + self.assertEqual((reach["top"], [c["package"] for c in reach["copyleft"]]), ("app", ["libgpl3"])) + self.assertEqual(out["restricted"], ["py"]) + self.assertEqual(out["classes"]["gpl2only"], 1) + + def test_accepted_findings_and_the_report(self): + self.build("r-aaaaaaaaaaaa", sbom(*STACK)) + vuln_triage.add(self.conn, "lcg", L.finding_id("conflict"), "will_not_fix", "dynamic only", "ann", + package="app") + (line,) = L.licence_lines(self.conn)["releases"] + self.assertEqual((line["open"], line["accepted"]), ({"conflict": 0, "unknown": 2}, 1)) + rel = L.licence_report(self.conn, "r-aaaaaaaaaaaa") + self.assertTrue(rel["indexed"]) + acc = {(f["kind"], f["package"]): bool(f["rule"]) for f in rel["findings"]} + self.assertEqual(acc, {("conflict", "app"): True, ("unknown", "orphan"): False, + ("unknown", "py"): False}) + self.assertEqual(L.licence_lines(self.conn, communities=set())["releases"], []) + self.assertIsNone(L.licence_report(self.conn, "r-aaaaaaaaaaaa", communities={"lhcb"})) + self.conn.execute("UPDATE sbom_scans SET index_version = 2") # not indexed for it yet + rel = L.licence_report(self.conn, "r-aaaaaaaaaaaa") + self.assertEqual((rel["indexed"], "findings" in rel), (False, False)) + self.assertNotIn("open", L.licence_lines(self.conn)["releases"][0]) + + def test_dependencies_and_diff(self): + self.build("r-aaaaaaaaaaaa", sbom(*STACK)) + newer = [c for c in STACK if c[0]["name"] != "orphan"] + [comp("extra")] + newer[1] = comp("libfoo", "MIT", version="2", runtime=["libgpl3"]) + self.build("r-bbbbbbbbbbbb", sbom(*newer)) + rep = L.dependency_report(self.conn, "r-bbbbbbbbbbbb") + most = {r["package"]: (r["rebuilds"], r["runtime"]) for r in rep["most_depended"]} + self.assertEqual(most["libgpl3"], (2, 2)) # libfoo, app + self.assertEqual(most["gcc"], (1, 0)) # app builds with it + self.assertEqual((rep["build_only"], rep["unneeded"]), (["gcc"], ["extra"])) + self.assertEqual(rep["previous"], "r-aaaaaaaaaaaa") + d = rep["diff"] + self.assertEqual((d["added"], d["removed"]), ([{"package": "extra", "version": "1"}], + [{"package": "orphan", "version": "1"}])) + self.assertEqual(d["version"], [{"package": "libfoo", "from": "1", "to": "2"}]) + self.assertEqual(d["licence"], [{"package": "libfoo", "from": "Apache-2.0", "to": "MIT"}]) + self.assertIsNone(L.dependency_report(self.conn, "r-aaaaaaaaaaaa")) # not the newest + + def test_versions_across_lines(self): + self.build("r-aaaaaaaaaaaa", sbom(*STACK)) + self.build("s-aaaaaaaaaaaa", sbom(*[comp("app", version="9"), comp("libfoo")]), community="lhcb") + out = L.dependency_lines(self.conn) + self.assertEqual([r["line"] for r in out["releases"]], ["r", "s"]) + (app,) = [v for v in out["versions"] if v["package"] == "app"] + self.assertEqual(sorted(app["versions"]), ["1", "9"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_sbom_plan.py b/console-backend/tests/test_orch_sbom_plan.py new file mode 100644 index 00000000..21694d59 --- /dev/null +++ b/console-backend/tests/test_orch_sbom_plan.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""orch.sbom_plan: upgrade plans from the scanned builds.""" + +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import db, sbom_plan, sbom_scan, vuln_triage + + +def comp(name, version, *deps, **props): + return ({"bom-ref": name + "@1", "name": name, "version": version, "cpe": "cpe:x", + "properties": [{"name": "bits:" + k, "value": v} for k, v in props.items()]}, + {"ref": name + "@1", "dependsOn": [d + "@1" for d in deps]}) + + +def vuln(vid, severity, fixes_by_ref, risk=0, kev=""): + props = [{"name": "grype:risk", "value": str(risk)}] if risk else [] + if kev: + props.append({"name": "grype:kev", "value": kev}) + return {"id": vid, "ratings": [{"severity": severity}], "properties": props, + "affects": [{"ref": r + "@1", "versions": [{"version": f, "status": "unaffected"} for f in fx]} + for r, fx in fixes_by_ref.items()]} + + +class PlanTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.dir = os.path.join(d, "scans") + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.n = 0 + + def build(self, bid, community, components, vulns, platform="x86_64-el9"): + self.n += 1 + self.conn.execute("INSERT INTO pipelines (id, is_build, community, package, version, platform," + " first_seen_ms) VALUES (?, 1, ?, 'LCG', '1', ?, 0)", (self.n, community, platform)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms, published_at)" + " VALUES (?, ?, ?, ?, ?)", (self.n, bid, community, self.n, "2026-10-%02d" % self.n)) + self.conn.commit() + doc = {"components": [c for c, _ in components], "metadata": {"component": {"bom-ref": "release"}}, + "dependencies": [{"ref": "release", "dependsOn": ["app@1"]}] + [d for _, d in components]} + sbom_scan._scanned(self.conn, self.dir, bid, "sha%d" % self.n, "db", {}, vulns, doc) + + def lcg(self, bid="release_gcc15_opt-aaaaaaaaaaaa", curl="8.1.0", community="lcg", pin=None, **kw): + comps = [comp("app", "1", "curl", "go", "zlib"), comp("curl", curl, "zlib", **({"pinned_by": pin} if pin else {})), + comp("go", "1.12.5"), comp("zlib", "1:1.2-3", provided_by="system", package="zlib"), + comp("left", "1.0")] + vulns = kw.get("vulns", [ + vuln("CVE-K", "high", {"go": ["1.20.10", "1.21.3"]}, risk=78.75, kev="2023-10-10"), + vuln("CVE-C1", "critical", {"curl": ["8.4.0"]}, risk=9), + vuln("CVE-C2", "medium", {"curl": ["8.6.0", "9.0.0rc1"]}, risk=2), + vuln("CVE-M", "high", {"go": ["2.0.0"]}, risk=5), + vuln("CVE-Z", "low", {"zlib": ["1:1.2-4"]}, risk=1), + vuln("CVE-N", "medium", {"left": []}, risk=3)]) + self.build(bid, community, comps, vulns) + + def test_a_release_is_planned_in_waves(self): + self.lcg() + (pl,) = sbom_plan.plan(self.conn)["releases"] + self.assertEqual((pl["line"], pl["risk"], pl["findings"]), ("release_gcc15_opt", 98.75, 6)) + w = {x["wave"]: x for x in pl["waves"]} + (go,) = w[0]["packages"] # known exploited; its major upgrade comes along + self.assertEqual((go["name"], go["target"], go["major"], go["kev"]), ("go", "2.0.0", True, ["CVE-K"])) + (curl,) = w[1]["packages"] # the newest fix it needs, no release candidate + self.assertEqual((curl["target"], curl["fixes"], curl["rebuild"]), ("8.6.0", ["CVE-C1", "CVE-C2"], 2)) + self.assertEqual((w[0]["rebuild"], w[0]["risk_left"], w[1]["risk_left"]), (2, 15.0, 4.0)) + self.assertEqual([(h["name"], h["target"]) for h in pl["host"]], [("zlib", "1:1.2-4")]) + self.assertEqual([(u["name"], u["unfixed"]) for u in pl["unfixed"]], [("left", ["CVE-N"])]) + + def test_triaged_findings_are_left_out(self): + self.lcg() + vuln_triage.add(self.conn, "lcg", "CVE-K", "false_positive", "not that go", "ann", package="go") + vuln_triage.add(self.conn, "lcg", "CVE-M", "false_positive", "not that go", "ann", package="go") + (pl,) = sbom_plan.plan(self.conn)["releases"] + self.assertEqual([x["wave"] for x in pl["waves"]], [1]) + self.assertEqual(pl["triaged"], 2) + + def test_what_the_stack_does_not_ship_is_not_planned(self): + comps = [comp("app", "1", "zlib"), comp("perf", "6.1", sbom="tool"), + comp("zlib", "1:1.2-3", provided_by="system"), comp("ssl", "3.0", provided_by="system")] + self.build("r-aaaaaaaaaaaa", "lcg", comps, [ + vuln("CVE-P", "high", {"perf": ["6.2"], "ssl": ["3.1"]}, risk=9), + vuln("CVE-Z", "low", {"zlib": ["1:1.2-4"]}, risk=1)]) + (pl,) = sbom_plan.plan(self.conn)["releases"] + self.assertEqual((pl["waves"], [h["name"] for h in pl["host"]]), ([], ["zlib"])) + self.assertEqual((pl["findings"], pl["not_shipped"]), (1, 2)) + + def test_the_newest_build_of_a_line_counts_and_lines_are_per_platform(self): + self.lcg() + self.lcg(bid="release_gcc15_opt-bbbbbbbbbbbb", curl="8.6.0") + self.build("release_gcc15_opt-cccccccccccc", "lcg", [comp("app", "1")], [], platform="aarch64-el9") + lines = sbom_plan.release_lines(self.conn) + self.assertEqual([(r["build_id"][-4:], r["platform"]) for r in lines], + [("cccc", "aarch64-el9"), ("bbbb", "x86_64-el9")]) + + def test_one_version_for_all_communities_unless_pinned(self): + self.lcg() + self.lcg(bid="release_lhcb_opt-bbbbbbbbbbbb", community="lhcb", curl="8.4.0", pin="defaults:lhcb") + self.lcg(bid="release_atlas_opt-cccccccccccc", community="atlas", curl="8.8.0", vulns=[]) + c = {x["name"]: x for x in sbom_plan.plan(self.conn)["converged"]}["curl"] + # 8.6.0 is enough, but ATLAS already builds 8.8.0 and it scanned clean: reuse it + self.assertEqual((c["version"], c["in_use_clean"]), ("8.8.0", ["release_atlas_opt-cccccccccccc"])) + acts = {r["community"]: (r["version"], r["action"]) for r in c["releases"]} + self.assertEqual(acts, {"lcg": ("8.1.0", "upgrade"), "lhcb": ("8.4.0", "pinned"), + "atlas": ("8.8.0", "keep")}) + self.assertEqual(c["pinned"], 1) + self.assertNotIn("zlib", {x["name"] for x in sbom_plan.plan(self.conn)["converged"]}) # the host's + + def test_a_version_in_use_that_is_vulnerable_is_not_the_answer(self): + self.lcg() # curl 8.1.0 needs 8.6.0 + self.lcg(bid="release_lhcb_opt-bbbbbbbbbbbb", community="lhcb", curl="8.6.0", + vulns=[vuln("CVE-X", "high", {"curl": ["8.7.1"]}, risk=7)]) + c = {x["name"]: x for x in sbom_plan.plan(self.conn, {"lcg", "lhcb"})["converged"]}["curl"] + self.assertEqual((c["version"], c["in_use_clean"]), ("8.7.1", [])) + self.assertEqual(sbom_plan.plan(self.conn, set()), {"releases": [], "converged": []}) + + def test_a_version_in_use_with_unfixed_findings_is_not_reused(self): + self.lcg() # curl 8.1.0 needs 8.6.0 + self.lcg(bid="release_lhcb_opt-bbbbbbbbbbbb", community="lhcb", curl="9.0.0", + vulns=[vuln("CVE-U", "critical", {"curl": []}, risk=50)]) + c = {x["name"]: x for x in sbom_plan.plan(self.conn)["converged"]}["curl"] + self.assertEqual(c["version"], "8.6.0") + + def test_versions_compare(self): + k = sbom_plan._vkey + self.assertTrue(k("1.10.0") > k("1.9.9") > k("v1.9")) + self.assertTrue(k("1:1.0") > k("2.0")) # an epoch first + self.assertTrue(k("8.0p1") > k("8.0") > k("8.0rc1")) + self.assertEqual(sbom_plan._need(["9.0.0rc1", "8.6.0", "8.0"], "8.1"), "8.6.0") + self.assertIsNone(sbom_plan._need(["2.0b1"], "1.0")) + self.assertEqual(sbom_plan._need(["1.1.1c"], "1.1.1"), "1.1.1c") # OpenSSL letters + self.assertEqual(sbom_plan._need(["0.0~git20240101.4a5b6c-1"], "0.0~git2023"), + "0.0~git20240101.4a5b6c-1") # not a pre-release + self.assertEqual(sbom_plan._need(["2.0.devel1", "1.5.arch"], "1.0"), "1.5.arch") + self.assertEqual(sbom_plan._major("1:3.0.7"), sbom_plan._major("3.0.8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_sbom_scan.py b/console-backend/tests/test_orch_sbom_scan.py new file mode 100644 index 00000000..354d5fe2 --- /dev/null +++ b/console-backend/tests/test_orch_sbom_scan.py @@ -0,0 +1,393 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""orch.sbom_scan: the builds' SBOMs scanned by the sbom-scanner service.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from console_backend.orch import db, events, sbom_scan, vuln_triage + + +class Store: + def __init__(self, objects): + self.objects = objects + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise KeyError(Key) + + class B: + def __init__(s, d): + s.d = d + + def read(s): + return s.d + return {"Body": B(self.objects[Key])} + + +class Scanner: + """The service: its database date, and a canned answer per SBOM.""" + + def __init__(self, built="2026-10-23T06:30:15Z", summary=None, fail=None, updating=False, + vulns=None): + self.built, self.summary, self.fail, self.updating = built, summary or {}, fail, updating + self.vulns = vulns if vulns is not None else [{"id": "CVE-1"}] + self.scanned = [] + + def get(self, url): + assert url.endswith("/v1/status"), url + return {"db": {"built": self.built} if self.built else None, "updating": self.updating} + + def post(self, url, body): + assert url.endswith("/v1/scan"), url + if self.fail: + raise self.fail + self.scanned.append(json.loads(body)["name"]) + return {"db": {"built": self.built}, "summary": self.summary, + "sbom": {"bomFormat": "CycloneDX", "vulnerabilities": self.vulns}} + + +class Fixture(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + self.dir = os.path.join(d, "sbom-scans") + db.migrate(self.conn) + self.objects = {} + for i, (bid, community) in enumerate([("b-1", "atlas"), ("b-2", "lhcb"), ("b-3", "atlas")]): + self.conn.execute("INSERT INTO pipelines (id, is_build, community, package, version," + " first_seen_ms) VALUES (?, 1, ?, 'Athena', '25', 0)", (i + 1, community)) + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms)" + " VALUES (?, ?, ?, ?)", (i + 1, bid, community, i)) + self.objects["MANIFESTS/%s/sbom.cdx.json" % bid] = json.dumps({"name": bid}).encode() + self.conn.commit() + + def _step(self, scanner, **kw): + return sbom_scan.step(self.conn, (Store(self.objects), "bkt"), "http://scanner:8080", self.dir, + get=scanner.get, post=scanner.post, **kw) + + def _scans(self): + return {r["build_id"]: dict(r) for r in self.conn.execute("SELECT * FROM sbom_scans")} + + def _events(self): + return [e for e in events.since(self.conn, limit=100) if e["kind"] == "sbom.scanned"] + + +class SbomScanTest(Fixture): + def test_newest_builds_are_scanned_first_a_few_a_pass(self): + s = Scanner(summary={"critical": 1, "high": 2, "packages": 2, "vulnerabilities": 3}) + with patch.object(sbom_scan, "PER_PASS", 2): + self.assertEqual(self._step(s), 2) + self.assertEqual(s.scanned, ["b-3", "b-2"]) + self.assertEqual(self._step(s), 1) + scan = self._scans()["b-1"] + self.assertEqual((scan["critical"], scan["high"], scan["packages"], scan["vulnerabilities"], + scan["error"]), (1, 2, 2, 3, "")) + self.assertEqual(sbom_scan.vulnerabilities(self.dir, "b-1"), [{"id": "CVE-1"}]) + self.assertIsNone(sbom_scan.vulnerabilities(self.dir, "b-9")) + self.assertEqual(self._step(s), 0) # nothing is due any more + + def test_a_newer_database_scans_again_and_a_change_is_an_event(self): + self._step(Scanner(summary={"high": 1})) + self.assertEqual(len(self._events()), 3) # first findings + self._step(Scanner(built="2026-10-24T06:00:00Z", summary={"high": 1})) + self.assertEqual(len(self._events()), 3) # same counts: no event + s2 = Scanner(built="2026-10-25T06:00:00Z", summary={"high": 1, "critical": 1}) + self.assertEqual(self._step(s2), 3) + ev = self._events()[-1] + self.assertEqual((ev["payload"]["critical"], ev["payload"]["was"]), (1, [0, 1, 0, 0, 0])) + self.assertEqual(self._scans()["b-1"]["db_built"], "2026-10-25T06:00:00Z") + + def test_a_newer_publication_of_a_build_scans_it_again(self): + self._step(Scanner()) + self.assertEqual(self._step(Scanner()), 0) + self.conn.execute("UPDATE sbom_scans SET scanned_ms = 1000") + self.conn.execute("INSERT INTO pipelines (id, is_build, community, first_seen_ms)" + " VALUES (9, 1, 'atlas', 0)") + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms)" + " VALUES (9, 'b-1', 'atlas', 2000)") # published after that scan + self.conn.commit() + self.objects["MANIFESTS/b-1/sbom.cdx.json"] = b'{"name": "b-1 again"}' + s = Scanner(summary={"high": 1}) + self.assertEqual(self._step(s), 1) + self.assertEqual(s.scanned, ["b-1 again"]) + self.assertEqual(self._scans()["b-1"]["high"], 1) + self.assertEqual(self._step(Scanner()), 0) # and then not again + + def test_a_first_clean_scan_is_no_event(self): + self._step(Scanner()) + self.assertEqual(self._events(), []) + + def test_a_failed_scan_keeps_the_last_good_counts(self): + self._step(Scanner(summary={"high": 2})) + late = Scanner(built="2026-10-24T06:00:00Z", fail=sbom_scan.Failed("scanner answered 500: boom")) + self._step(late) + scan = self._scans()["b-1"] + self.assertEqual((scan["high"], scan["db_built"]), (2, "2026-10-23T06:30:15Z")) + self.assertIn("boom", scan["error"]) + self.assertEqual(sbom_scan.vulnerabilities(self.dir, "b-1"), [{"id": "CVE-1"}]) + with patch.object(sbom_scan, "RETRY_MS", -1): + self._step(Scanner(built="2026-10-24T06:00:00Z", summary={"high": 2})) + self.assertEqual(self._scans()["b-1"]["error"], "") + self.assertEqual(len(self._events()), 3) # the same counts again: no event + + def test_a_build_just_recorded_waits_for_its_sbom(self): + now = db.now_ms() + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, community, recorded_ms)" + " VALUES (1, 'b-new', 'atlas', ?)", (now,)) + self.conn.commit() + self.assertNotIn("b-new", sbom_scan.due(self.conn, "d", now, limit=10)) + self.assertIn("b-new", sbom_scan.due(self.conn, "d", now + sbom_scan.SETTLE_MS, limit=10)) + + def test_a_missing_sbom_is_kept_and_retried_later(self): + del self.objects["MANIFESTS/b-3/sbom.cdx.json"] + self._step(Scanner()) + self.assertIn("no sbom.cdx.json", self._scans()["b-3"]["error"]) + self.objects["MANIFESTS/b-3/sbom.cdx.json"] = b'{"name": "b-3"}' + self.assertEqual(self._step(Scanner()), 0) # not before RETRY_MS + with patch.object(sbom_scan, "RETRY_MS", -1): + self.assertEqual(self._step(Scanner()), 1) + + def test_a_busy_or_updating_scanner_records_nothing(self): + self.assertEqual(self._step(Scanner(fail=sbom_scan.Busy("busy"))), 0) + self.assertEqual(self._step(Scanner(updating=True)), 0) + self.assertEqual(self._scans(), {}) + + def test_no_url_scans_nothing_and_no_database_is_an_error(self): + self.assertEqual(sbom_scan.step(self.conn, None, "", ""), 0) + with self.assertRaisesRegex(RuntimeError, "no valid vulnerability database"): + self._step(Scanner(built="")) + with self.assertRaises(RuntimeError): + sbom_scan.step(self.conn, None, "http://scanner:8080", self.dir) + + def test_a_pass_stops_starting_scans_after_its_budget(self): + ticks = iter([0, 0, sbom_scan.BUDGET_S + 1, sbom_scan.BUDGET_S + 2]) + self.assertEqual(self._step(Scanner(), clock=lambda: next(ticks)), 1) + + def test_an_odd_build_id_is_a_safe_file_name(self): + p = sbom_scan._path(self.dir, "../x") + self.assertEqual(os.path.dirname(p), self.dir) + self.assertNotIn("..", os.path.basename(p)) + + def test_listing_is_most_severe_first_and_scoped(self): + self._step(Scanner()) + self.conn.execute("UPDATE sbom_scans SET high = 3 WHERE build_id = 'b-1'") + self.conn.execute("UPDATE sbom_scans SET critical = 1 WHERE build_id = 'b-2'") + self.conn.execute("UPDATE sbom_scans SET index_version = 0") # the scanner's counts + self.conn.commit() + ids = lambda **kw: [r["build_id"] for r in sbom_scan.listing(self.conn, **kw)] + self.assertEqual(ids(), ["b-2", "b-1", "b-3"]) + self.assertEqual(ids(flagged=True), ["b-2", "b-1"]) + self.assertEqual(ids(communities={"atlas"}), ["b-1", "b-3"]) + self.assertEqual(ids(communities=set()), []) + row = sbom_scan.listing(self.conn, communities={"lhcb"})[0] + self.assertEqual((row["pipeline_id"], row["package"], row["critical"]), (2, "Athena", 1)) + + + +def component(ref, name, version, purl="", cpe="", **props): + c = {"bom-ref": ref, "name": name, "version": version, + "properties": [{"name": k.replace("_", ":", 1), "value": v} for k, v in props.items()]} + if purl: + c["purl"] = purl + if cpe: + c["cpe"] = cpe + return c + + +SBOM = {"bomFormat": "CycloneDX", "components": [ + component("libxml2@1", "libxml2", "2.9.13", cpe="cpe:2.3:a:xmlsoft:libxml2:2.9.13:*:*:*:*:*:*:*"), + component("ROOT@1", "ROOT", "6.32.0", purl="pkg:github/root-project/root@v6-32-00", + bits_pinned_by="defaults:lhcb", bits_build_requires="numpy"), + component("zlib@system", "zlib1g", "1:1.3-1", purl="pkg:deb/ubuntu/zlib1g@1:1.3-1?arch=amd64", + bits_provided_by="system", bits_package="zlib"), + component("numpy@1", "numpy", "2.0.1", purl="pkg:pypi/numpy@2.0.1"), + component("libxml2x@1", "libxml2", "2.90.0")], + "metadata": {"component": {"bom-ref": "release", "name": "b"}}, + "dependencies": [{"ref": "release", "dependsOn": ["ROOT@1", "numpy@1"]}, + {"ref": "ROOT@1", "dependsOn": ["libxml2@1"]}, + {"ref": "numpy@1", "dependsOn": ["zlib@system"]}]} +VULNS = [{"id": "CVE-A", "ratings": [{"severity": "high"}], + "properties": [{"name": "grype:risk", "value": "42.5"}, {"name": "grype:epss", "value": "0.5"}, + {"name": "grype:kev", "value": "2024-01-02"}], + "affects": [{"ref": "libxml2@1", "versions": [{"version": "2.10.0", "status": "unaffected"}, + {"version": "2.9.14", "status": "unaffected"}]}, + {"ref": "zlib@system"}]}, + {"id": "CVE-B", "ratings": [{"severity": "info"}], "affects": [{"ref": "libxml2@1"}]}, + {"id": "CVE-C", "ratings": [{"severity": "critical"}], "affects": [{"ref": "gone"}]}] + + +class SbomIndexTest(Fixture): + def setUp(self): + super().setUp() + for bid in ("b-1", "b-2", "b-3"): + self.objects["MANIFESTS/%s/sbom.cdx.json" % bid] = json.dumps(dict(SBOM, name=bid)).encode() + self.conn.execute("UPDATE pipelines SET version = '1' || id") + self.conn.commit() + self.assertEqual(self._step(Scanner(summary={"high": 2, "other": 1}, vulns=VULNS)), 3) + + def test_a_scan_indexes_the_sbom(self): + scan = self._scans()["b-1"] + self.assertEqual((scan["components"], scan["identified"], scan["index_version"]), + (5, 3, sbom_scan.INDEX_VERSION)) # cpe, deb, pypi; not github, generic + rows = self.conn.execute("SELECT * FROM sbom_packages WHERE build_id = 'b-1'" + " AND ref = 'zlib@system'").fetchone() + self.assertEqual((rows["name"], rows["alias"], rows["system"]), ("zlib1g", "zlib", 1)) + findings = self.conn.execute("SELECT vuln_id, ref, severity FROM sbom_findings" + " WHERE build_id = 'b-1' ORDER BY 1, 2").fetchall() + self.assertEqual([tuple(f) for f in findings], [("CVE-A", "libxml2@1", "high"), + ("CVE-A", "zlib@system", "high"), ("CVE-B", "libxml2@1", "other")]) + + def test_a_scan_from_before_the_index_is_due_again(self): + self.assertEqual(sbom_scan.due(self.conn, "2026-10-23T06:30:15Z", db.now_ms()), []) + self.conn.execute("UPDATE sbom_scans SET index_version = 0 WHERE build_id = 'b-2'") + self.conn.commit() + self.assertEqual(sbom_scan.due(self.conn, "2026-10-23T06:30:15Z", db.now_ms()), ["b-2"]) + row = {r["build_id"]: r for r in sbom_scan.listing(self.conn)}["b-2"] + self.assertIsNone(row["effective"]) # not indexed yet: the raw counts + + def test_effective_counts_leave_out_what_the_triage_covers(self): + row = {r["build_id"]: r for r in sbom_scan.listing(self.conn)}["b-1"] + self.assertEqual((row["effective"]["high"], row["effective"]["other"], row["suppressed"]), + (2, 1, 0)) + vuln_triage.add(self.conn, "atlas", "CVE-A", "false_positive", "not our zlib", "ann", + package="zlib") # its bits name + rows = {r["build_id"]: r for r in sbom_scan.listing(self.conn)} + self.assertEqual((rows["b-1"]["effective"]["high"], rows["b-1"]["suppressed"]), (1, 1)) + self.assertEqual(rows["b-2"]["suppressed"], 0) # lhcb's: not atlas's rule + self.assertEqual(rows["b-1"]["high"], 2) # the scan's own counts stay + for vuln in ("cve-a", "CVE-B"): # any case + vuln_triage.add(self.conn, "atlas", vuln, "will_not_fix", "accepted", "ann") + ids = lambda **kw: [r["build_id"] for r in sbom_scan.listing(self.conn, **kw)] + self.conn.execute("UPDATE sbom_scans SET scanned_ms = scanned_ms + 1000 WHERE build_id = 'b-3'") + self.conn.commit() + self.assertEqual(ids(), ["b-2", "b-3", "b-1"]) # b-1 and b-3: nothing open + self.assertEqual(ids(flagged=True), ["b-2"]) + self.conn.execute("UPDATE sbom_scans SET identified = 0 WHERE build_id = 'b-3'") + self.conn.commit() + self.assertEqual(ids(flagged=True), ["b-2", "b-3"]) # not scannable: flagged too + + def test_which_builds_contain_a_package(self): + found = lambda *a, **kw: sorted((r["build_id"], r["component_version"]) + for r in sbom_scan.packages(self.conn, *a, **kw)) + self.assertEqual(found("LIBXML2"), [(b, v) for b in ("b-1", "b-2", "b-3") + for v in ("2.9.13", "2.90.0")]) + self.assertEqual(found("libxml2", "2.9"), [("b-1", "2.9.13"), ("b-2", "2.9.13"), + ("b-3", "2.9.13")]) + self.assertEqual(found("libxml2", "2.9.x", communities={"lhcb"}), [("b-2", "2.9.13")]) + self.assertEqual(found("libxml2", "2.9.1"), []) + self.assertEqual(found("zlib", communities={"lhcb"}), [("b-2", "1:1.3-1")]) + self.assertEqual(found("libxml2", communities=set()), []) + (row,) = [r for r in sbom_scan.packages(self.conn, "libxml2", "2.9.13", communities={"lhcb"})] + self.assertEqual((row["version"], row["findings"]), ("12", [ + {"id": "CVE-A", "severity": "high", "suppressed": None}, + {"id": "CVE-B", "severity": "other", "suppressed": None}])) + self.assertEqual(found("libxml2", "_.9%"), []) # no wildcards of its own + self.assertIn("zlib", sbom_scan.names(self.conn, {"atlas"})) + self.assertEqual(sbom_scan.names(self.conn, set()), []) + + def test_which_builds_a_vulnerability_was_found_in(self): + rid = vuln_triage.add(self.conn, "atlas", "CVE-A", "will_not_fix", "accepted", "ann", + package="libxml2", version="2.9.13") + rows = sbom_scan.affected(self.conn, "CVE-A") + self.assertEqual(sorted((r["build_id"], r["name"], r["suppressed"]) for r in rows), [ + ("b-1", "libxml2", rid), ("b-1", "zlib1g", None), ("b-2", "libxml2", None), + ("b-2", "zlib1g", None), ("b-3", "libxml2", rid), ("b-3", "zlib1g", None)]) + self.assertEqual(sbom_scan.affected(self.conn, "CVE-C"), []) # matched no component + self.assertEqual(len(sbom_scan.affected(self.conn, "CVE-A", communities={"lhcb"})), 2) + + def test_a_rescan_of_the_same_sbom_keeps_its_package_rows(self): + rowid = lambda: self.conn.execute("SELECT MIN(rowid) FROM sbom_packages").fetchone()[0] + before, seen = rowid(), len(self._events()) + self._step(Scanner(built="2026-10-24T06:00:00Z", summary={"high": 2, "other": 1}, + vulns=VULNS[:1])) + self.assertEqual(rowid(), before) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM sbom_findings WHERE" + " build_id = 'b-1'").fetchone()[0], 2) + self.assertEqual(len(self._events()), seen) # same counts: no event + + def test_an_odd_sbom_is_indexed_as_far_as_it_goes(self): + doc = dict(SBOM, name="b-3", components=[{"bom-ref": "x", "name": ["x"], "properties": 5}], + dependencies=[{"ref": "x", "dependsOn": "y"}]) + odd = [{"id": "CVE-A", "affects": [{"ref": ["x"]}, "y", {"ref": "x", "versions": 3}]}, "z"] + self.assertEqual(sbom_scan.index(doc, odd)[:2], (1, 0)) + self.assertEqual(sbom_scan.index(doc, odd)[3], [("CVE-A", "x", "other", 0.0, 0.0, "", "[]")]) + + def test_an_indexing_error_is_a_failed_scan_and_blocks_no_other(self): + real = sbom_scan.index + def index(doc, vulns): + if doc.get("name") == "b-3": + raise TypeError("odd") + return real(doc, vulns) + self.conn.execute("UPDATE sbom_scans SET index_version = 0") + self.conn.commit() + with patch.object(sbom_scan, "index", index): + self.assertEqual(self._step(Scanner(vulns=VULNS)), 2) + scans = self._scans() + self.assertIn("could not index", scans["b-3"]["error"]) + self.assertEqual(scans["b-1"]["index_version"], sbom_scan.INDEX_VERSION) + + def test_threat_fixes_pins_and_dependencies_are_indexed(self): + f = self.conn.execute("SELECT * FROM sbom_findings WHERE build_id = 'b-1' AND vuln_id = 'CVE-A'" + " AND ref = 'libxml2@1'").fetchone() + self.assertEqual((f["risk"], f["epss"], f["kev"], json.loads(f["fixed_in"])), + (42.5, 0.5, "2024-01-02", ["2.10.0", "2.9.14"])) + p = self.conn.execute("SELECT * FROM sbom_packages WHERE build_id = 'b-1' AND ref = 'ROOT@1'").fetchone() + self.assertEqual((p["pinned_by"], json.loads(p["depends_on"])), ("defaults:lhcb", ["libxml2@1", "numpy@1"])) + + def test_what_the_stack_does_not_ship_is_left_out(self): + doc = json.loads(json.dumps(SBOM)) + doc["components"] += [ + component("perf@1", "perf", "6.1", cpe="cpe:2.3:a:linux:perf:6.1:*:*:*:*:*:*:*", bits_sbom="tool"), + component("elfutils@system", "elfutils", "0.190", bits_provided_by="system"), + component("ssl@system", "openssl", "3.0", bits_provided_by="system"), + component("bpf@1", "libbpf", "1.5", cpe="cpe:2.3:a:libbpf:libbpf:1.5:*:*:*:*:*:*:*")] + doc["components"][1]["properties"].append({"name": "bits:build_requires", "value": "openssl"}) + doc["dependencies"][0]["dependsOn"].append("perf@1") # asked for, still a tool + doc["dependencies"].append({"ref": "perf@1", "dependsOn": ["elfutils@system", "bpf@1", "libxml2@1"]}) + vulns = VULNS + [{"id": "CVE-P", "ratings": [{"severity": "critical"}], + "affects": [{"ref": r} for r in ("perf@1", "elfutils@system", "ssl@system", + "bpf@1")]}] + self.objects["MANIFESTS/b-3/sbom.cdx.json"] = json.dumps(dict(doc, name="b-3")).encode() + self.conn.execute("UPDATE sbom_scans SET index_version = 0 WHERE build_id = 'b-3'") + self.conn.commit() + self._step(Scanner(summary={"critical": 1, "high": 2, "other": 1}, vulns=vulns)) + why = {r["ref"]: r["not_shipped"] for r in self.conn.execute( + "SELECT ref, not_shipped FROM sbom_packages WHERE build_id = 'b-3'")} + self.assertEqual({r: w for r, w in why.items() if w}, + {"perf@1": "tool", "elfutils@system": "tool-only", "bpf@1": "tool-only", + "ssl@system": "build-only"}) # not libxml2: ROOT uses it too + row = {r["build_id"]: r for r in sbom_scan.listing(self.conn)}["b-3"] + self.assertEqual((row["effective"]["critical"], row["effective"]["high"], row["not_shipped"]), + (0, 2, 4)) # zlib, used at run time, stays + (perf,) = sbom_scan.packages(self.conn, "perf") + self.assertEqual((perf["not_shipped"], perf["findings"][0]["id"]), ("tool", "CVE-P")) + out = vuln_triage.annotate(vulns, {}, vuln_triage.Rules([]), "atlas", why) + self.assertEqual([(v["bom-ref"] if "bom-ref" in v else v["id"], [a["ref"] for a in v["affects"]], + v.get("analysis", {}).get("state")) for v in out if v["id"] == "CVE-P"], + [("CVE-P-build-only", ["ssl@system"], "not_affected"), + ("CVE-P-tool", ["perf@1"], "not_affected"), + ("CVE-P-tool-only", ["elfutils@system", "bpf@1"], "not_affected")]) + + def test_without_a_release_root_only_tools_are_left_out(self): + doc = dict(SBOM, metadata={}, components=SBOM["components"] + [ + component("perf@1", "perf", "6.1", bits_sbom="tool")]) + rows = {p[0]: p[-1] for p in sbom_scan.index(doc, [])[2]} + self.assertEqual({r: w for r, w in rows.items() if w}, {"perf@1": "tool"}) + + def test_a_sbom_that_is_not_json_is_a_failed_scan(self): + self.objects["MANIFESTS/b-1/sbom.cdx.json"] = b"[1" + self.conn.execute("UPDATE sbom_scans SET index_version = 0") + self.conn.commit() + self._step(Scanner(vulns=VULNS)) + self.assertIn("not JSON", self._scans()["b-1"]["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_schedules.py b/console-backend/tests/test_orch_schedules.py new file mode 100644 index 00000000..654480ce --- /dev/null +++ b/console-backend/tests/test_orch_schedules.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Nightly schedules: what they are, when they run, and their API.""" + +import datetime as dt +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch +from zoneinfo import ZoneInfo + +from fastapi.testclient import TestClient + +from console_backend import config, main +from console_backend.orch import db, schedules as sch + +ZURICH = ZoneInfo("Europe/Zurich") + + +def _p(**more): + p = {"name": "ROOT nightly", "package": "ROOT", "platforms": ["x86_64-el9", "aarch64-el9"], + "time": "02:00", "defaults": "release::gcc15", "certify": True} + p.update(more) + return p + + +def _ms(y, mo, d, h, mi, tz=dt.timezone.utc): + return int(dt.datetime(y, mo, d, h, mi, tzinfo=tz).timestamp() * 1000) + + +class CheckTest(unittest.TestCase): + def test_a_schedule_and_its_defaults(self): + s = sch.check(_p(), "LCG") + self.assertEqual((s["community"], s["days"], s["certify_groups"], s["enabled"], s["jobs"]), + ("lcg", sch.EVERY_DAY, ["lcg"], True, 0)) + + def test_what_is_refused(self): + for bad in ({"package": "ROOT; rm"}, {"platforms": []}, {"platforms": ["a", "a"]}, + {"time": "2:00"}, {"time": "24:00"}, {"days": 0}, {"days": 128}, + {"extra_repos": "http://x/y@main"}, {"set": {"a b": "c"}}, {"jobs": -1}, + {"certify": "yes"}, {"extra_build_args": "--x '$(id)'"}, {"name": ""}, + {"platforms": ["x86_64-el9\n"]}, {"set": {"k\n": "v"}}, {"set": {"k": "v\n"}}, + {"extra_build_args": "--a\x1cb"}, {"extra_build_args": "--a\u2028b"}, + {"extra_repos": "https://h/p@-x"}, {"extra_build_args": "--keep-going --defaults x"}, + {"extra_build_args": "--defaults=x"}): + with self.assertRaises(sch.Invalid, msg=bad): + sch.check(_p(**bad), "lcg") + + def test_variants_are_defaults_chains_and_count_in_the_hash_only_when_there(self): + s = sch.check(_p(variants=["gcc15::opt", "clang::opt"]), "lcg") + self.assertEqual((s["variants"], sch.cells(s)[:2]), + (["clang::opt", "gcc15::opt"], [("aarch64-el9", "clang::opt"), ("aarch64-el9", "gcc15::opt")])) + plain = sch.check(_p(), "lcg") + self.assertEqual(plain["variants"], []) + self.assertEqual(sch.cells(plain), [("aarch64-el9", ""), ("x86_64-el9", "")]) + h = sch.content_sha256(plain) + import hashlib # the hash schedules had before variants + before = hashlib.sha256(json.dumps({k: plain[k] for k in sch._CONTENT}, sort_keys=True, + separators=(",", ":")).encode()).hexdigest() + self.assertEqual(h, before) + self.assertNotEqual(h, sch.content_sha256(s)) + self.assertEqual(sch.check(_p(variants=["gcc15::", "::clang"]), "lcg")["variants"], ["clang", "gcc15"]) + for bad in (["a b"], [""], ["x", "x"], "gcc15", ["v%d" % i for i in range(17)], ["::"], ["a", "a::"], + ["v" * 150]): + with self.assertRaises(sch.Invalid, msg=bad): + sch.check(_p(variants=bad), "lcg") + with self.assertRaises(sch.Invalid): # 32 platforms x 3 variants: too many builds + sch.check(_p(platforms=["p%d" % i for i in range(32)], variants=["a", "b", "c"]), "lcg") + self.assertEqual((sch.cell_key("x86_64-el9", ""), sch.cell_key("x86_64-el9", "gcc15::opt")), + ("x86_64-el9", "x86_64-el9 gcc15::opt")) + + def test_migration_22_keeps_the_runs_and_lets_variants_share_a_night(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + conn = db.connect(os.path.join(d, "old.db")) + self.addCleanup(conn.close) + for n in range(21): # a database of migration 21 + with db.transaction(conn): + for stmt in db.MIGRATIONS[n]: + conn.execute(stmt) + conn.execute("PRAGMA user_version=%d" % (n + 1)) + with db.transaction(conn): + conn.execute("INSERT INTO schedules (community, community_dir, name, package, version, defaults," + " extra_repos, set_vars, platforms, rebuild, debug, jobs, extra_build_args, days, time," + " certify, cvmfs, certify_groups, enabled, content_sha256, revision, last_fired_ms," + " created_by, created_ms, updated_by, updated_ms) VALUES ('lcg', 'LCG', 'n', 'ROOT', ''," + " '', '', '{}', '[\"x86_64-el9\"]', 0, 0, 0, '', 127, '02:00', 0, 0, '[\"lcg\"]', 1," + " 'h', 1, 0, 'a', 0, 'a', 0)") + conn.execute("INSERT INTO schedule_runs (schedule_id, night, platform, attempt, state, content_sha256," + " created_ms, updated_ms) VALUES (1, '2026-10-28', 'x86_64-el9', 1, 'passed', 'h', 0, 0)") + db.migrate(conn) + r = conn.execute("SELECT * FROM schedule_runs").fetchone() + self.assertEqual((r["platform"], r["variant"], r["state"]), ("x86_64-el9", "", "passed")) + self.assertEqual(sch.get(conn, 1)["variants"], []) + with db.transaction(conn): + conn.execute("INSERT INTO schedule_runs (schedule_id, night, platform, variant, attempt, state," + " content_sha256, created_ms, updated_ms) VALUES (1, '2026-10-28', 'x86_64-el9', 'gcc15', 1," + " 'queued', 'h', 0, 0)") + names = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'index'" + " AND tbl_name = 'schedule_runs'")} + self.assertTrue({"schedule_runs_night", "schedule_runs_entry", "schedule_runs_state"} <= names) + + def test_the_content_hash_leaves_out_when_and_whether(self): + h = sch.content_sha256(sch.check(_p(), "lcg")) + self.assertEqual(h, sch.content_sha256(sch.check(_p(time="03:30", days=1, enabled=False), "lcg"))) + self.assertNotEqual(h, sch.content_sha256(sch.check(_p(platforms=["x86_64-el9"]), "lcg"))) + self.assertNotEqual(h, sch.content_sha256(sch.check(_p(defaults="release"), "lcg"))) + self.assertEqual(h, sch.content_sha256(sch.check(_p(platforms=["aarch64-el9", "x86_64-el9"]), "lcg"))) + r = sch.content_sha256(sch.check(_p(extra_repos="https://h/p@main"), "lcg")) + self.assertEqual(r, sch.content_sha256(sch.check(_p(extra_repos="https://h/p@main;"), "lcg"))) + + +class WhenTest(unittest.TestCase): + def test_the_weekdays(self): + s = sch.check(_p(days=0b0000001), "lcg") # Mondays only + got = sch.occurrences(s, _ms(2026, 10, 26, 0, 0), _ms(2026, 11, 9, 23, 0), dt.timezone.utc) + self.assertEqual([n for n, _ in got], ["2026-10-26", "2026-11-02", "2026-11-09"]) + + def test_a_time_skipped_in_spring_runs_at_the_first_minute_after(self): + s = sch.check(_p(time="02:30"), "lcg") + (night, ms), = sch.occurrences(s, _ms(2026, 3, 28, 12, 0), _ms(2026, 3, 29, 12, 0), ZURICH) + self.assertEqual((night, dt.datetime.fromtimestamp(ms / 1000, ZURICH).strftime("%H:%M")), + ("2026-03-29", "03:00")) + + def test_a_time_repeated_in_autumn_runs_once(self): + s = sch.check(_p(time="02:30"), "lcg") + got = sch.occurrences(s, _ms(2026, 10, 24, 12, 0), _ms(2026, 10, 25, 12, 0), ZURICH) + self.assertEqual([(n, ms) for n, ms in got], [("2026-10-25", _ms(2026, 10, 25, 0, 30))]) + + def test_the_next_run(self): + s = sch.check(_p(time="02:00"), "lcg") + self.assertEqual(sch.next_run_ms(s, _ms(2026, 10, 28, 1, 0), ZURICH), _ms(2026, 10, 29, 1, 0)) + + +class ApiTest(unittest.TestCase): + def setUp(self): + self.addCleanup(setattr, main, "settings", main.settings) + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + path = os.path.join(d, "orch.db") + self.conn = db.connect(path) + db.migrate(self.conn) + self.addCleanup(self.conn.close) + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": path, + "BITS_ADMINS_POLICY": "lcg @alice\natlas @bob\n* @carol"}) + for p in (patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None), + patch.object(sch, "time_zone", lambda: ZURICH)): + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + + def _do(self, method, user, path, body=None): + return self.client.request(method, path, json=body, headers={"Authorization": "Bearer " + user}) + + def test_admins_edit_a_communitys_schedules_and_anyone_reads_them(self): + self.assertEqual(self._do("POST", "bob", "/ops/schedules", dict(_p(), community="lcg")).status_code, 403) + self.assertEqual(self._do("POST", "alice", "/ops/schedules", dict(_p(time="9"), community="lcg")) + .status_code, 400) + r = self._do("POST", "alice", "/ops/schedules", dict(_p(), community="LCG")) + self.assertEqual(r.status_code, 200, r.text) + s = r.json() + self.assertEqual((s["community"], s["community_dir"], s["revision"], s["approval"]), + ("lcg", "LCG", 1, None)) + self.assertGreater(s["next_run_ms"], db.now_ms()) + self.assertEqual(self._do("POST", "alice", "/ops/schedules", dict(_p(), community="lcg")) + .status_code, 400) # same name + read = self._do("GET", "dave", "/ops/schedules?community=lcg").json() + self.assertEqual((read["time_zone"], [x["id"] for x in read["schedules"]]), ("Europe/Zurich", [s["id"]])) + self.assertNotIn("approval", read["schedules"][0]) # admins only + self.assertEqual(self._do("GET", "dave", "/ops/schedules").json()["schedules"], []) + u = "/ops/schedules/%d" % s["id"] + self.assertEqual(self._do("PUT", "bob", u, dict(_p(), revision=1)).status_code, 403) + r = self._do("PUT", "alice", u, dict(_p(time="03:15"), revision=1)) + self.assertEqual(r.json()["community_dir"], "LCG") + self.assertEqual((r.status_code, r.json()["revision"], r.json()["content_sha256"]), + (200, 2, s["content_sha256"])) # time: same content + self.assertEqual(self._do("PUT", "alice", u, dict(_p(), revision=1)).status_code, 409) + self.assertEqual(self._do("PUT", "alice", u, dict(_p(), revision="x")).status_code, 400) + r = self._do("POST", "alice", u + "/pause") + self.assertEqual((r.json()["enabled"], r.json()["next_run_ms"]), (False, None)) + r = self._do("PUT", "alice", u, dict(_p(time="01:00"), revision=3)) # no enabled: stays paused + self.assertEqual((r.status_code, r.json()["enabled"]), (200, False)) + self.assertEqual(self._do("POST", "alice", "/ops/schedules", + dict(_p(name="x", certify_groups=["atlas"]), community="lcg")).status_code, 403) + self.assertTrue(self._do("POST", "alice", u + "/resume").json()["enabled"]) + self.assertEqual(self._do("POST", "alice", u + "/stop").status_code, 404) + self.assertEqual(self._do("DELETE", "carol", u).json(), {"deleted": s["id"]}) + self.assertEqual(self._do("DELETE", "carol", u).status_code, 404) + kinds = [(r["kind"], sorted(json.loads(r["payload"]))) for r in self.conn.execute( + "SELECT kind, payload FROM events WHERE subject = ? ORDER BY seq", ("schedule:%d" % s["id"],))] + self.assertEqual(kinds, [("schedule.created", ["after"]), ("schedule.updated", ["after", "before"]), + ("schedule.paused", ["after", "before"]), ("schedule.updated", ["after", "before"]), + ("schedule.resumed", ["after", "before"]), ("schedule.deleted", ["before"])]) + self.assertEqual(self.client.get("/ops/schedules").status_code, 401) + + def test_a_change_may_come_from_the_console_and_the_app(self): + main.settings.frontend_origin = "https://pages" # another origin: a preflight + r = self.client.options("/ops/schedules/1", headers={"Origin": "https://pages", + "Access-Control-Request-Method": "PUT"}) + self.assertIn("PUT", r.headers["Access-Control-Allow-Methods"]) + + def test_the_matrix_for_anyone_and_run_now_for_admins(self): + s = self._do("POST", "alice", "/ops/schedules", dict(_p(), community="LCG")).json() + m = self._do("GET", "dave", "/ops/nightlies?community=lcg&nights=3").json() + self.assertEqual((len(m["nights"]), [r["schedule"]["id"] for r in m["rows"]], m["time_zone"]), + (3, [s["id"]], "Europe/Zurich")) + self.assertEqual(self._do("GET", "dave", "/ops/nightlies").json()["rows"], []) + u = "/ops/schedules/%d/run" % s["id"] + self.assertEqual(self._do("POST", "bob", u, {}).status_code, 403) + self.assertEqual(self._do("POST", "alice", u, {}).status_code, 503) # the queue is off + self.assertEqual(self._do("POST", "alice", u, {"platforms": "x"}).status_code, 400) + self.assertEqual(self._do("POST", "alice", u, {"cells": [1]}).status_code, 400) + v = self._do("POST", "alice", "/ops/schedules", dict(_p(name="dev3", variants=["gcc15::opt"]), + community="LCG")).json() + self.assertEqual(v["variants"], ["gcc15::opt"]) + row = [r for r in self._do("GET", "dave", "/ops/nightlies?community=lcg").json()["rows"] + if r["schedule"]["id"] == v["id"]][0] + self.assertEqual([c["key"] for c in row["schedule"]["cells"]], + ["aarch64-el9 gcc15::opt", "x86_64-el9 gcc15::opt"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_shadow.py b/console-backend/tests/test_orch_shadow.py new file mode 100644 index 00000000..008b007e --- /dev/null +++ b/console-backend/tests/test_orch_shadow.py @@ -0,0 +1,580 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The orchestrator's M4, first half: certification in shadow.""" + +import io +import json +import os +import shutil +import tarfile +import tempfile +import unittest +import zipfile +from unittest.mock import patch + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from fastapi.testclient import TestClient + +from bits_helpers import certify, trust +from console_backend import config, forge_ops, main +from console_backend.orch import certified, db, events, shadow, worker + +SETTINGS = {"BITS_ORCH_MANIFESTS_PROJECT": "bits/bits-manifests", + "BITS_ORCH_SHADOW_LOOKBACK_DAYS": "100000"} + + +def _pkg(name, h, arch="el9", sha=None): + return {"package": name, "version": "1", "revision": "1", "hash": h, + "tarball": "%s-1-1.%s.tar.gz" % (name, arch), "tarball_sha256": sha or ("s" + h), + "effective_architecture": arch} + + +def _bom(build_id, arch, *pkgs): + return {"build_id": build_id, "source_pipeline_id": 7, + "packages": [_pkg(n, h, arch) for n, h in pkgs]} + + +def _archive(sha, files): + """GitLab's archive of manifests/ at *sha*: one top directory.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for path, doc in files.items(): + data = json.dumps(doc).encode() if not isinstance(doc, bytes) else doc + info = tarfile.TarInfo("bits-manifests-%s-manifests/%s" % (sha, path)) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +class FakeManifests: + def __init__(self): + self.trees, self.pipelines, self.jobs, self.artifacts, self.parents = {}, {}, {}, {}, {} + self.fail = None + + def add(self, pid, sha, before, status="success", updated="2026-10-11T10:00:00Z", + source="push", artifacts=None): + self.pipelines[pid] = {"id": pid, "sha": sha, "before_sha": before, "ref": "main", + "source": source, "status": status, "updated_at": updated, + "created_at": "2026-10-11T09:00:00Z"} + self.jobs[pid] = [{"id": pid * 10, "name": "certify-and-publish", "status": status}, + {"id": pid * 10 - 1, "name": "certify-and-publish", "status": "failed"}] + if artifacts is not None: + self.artifacts[pid * 10] = artifacts + + def finished_pipelines(self, ref, per_page=50, scope="finished"): + if scope != "finished": + return [{"id": i} for i in getattr(self, "running", ())] + return [dict(p) for p in sorted(self.pipelines.values(), key=lambda p: -p["id"])] + + def get_pipeline(self, pid): + if self.fail: + raise self.fail + return dict(self.pipelines[pid]) + + def pipeline_jobs(self, pid): + return list(self.jobs[pid]) + + def commit(self, sha): + return {"parent_ids": self.parents.get(sha, [])} + + def repository_archive(self, sha, path=None): + assert path == "manifests" + return _archive(sha, self.trees[sha]) if sha in self.trees else None + + def job_artifacts(self, job_id, max_bytes=0): + return self.artifacts.get(job_id) + + +class _Base(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.cache = os.path.join(self.dir, "shadow") + self.settings = config.Settings(env=SETTINGS) + self.gl = FakeManifests() + self.store = {} # (arch, hash) -> sha256 the store holds + priv = Ed25519PrivateKey.generate() + self.key = os.path.join(self.dir, "k.pem") + with open(self.key, "wb") as fh: + fh.write(priv.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + os.chmod(self.key, 0o600) + kid = trust.key_id(priv.public_key()) + self.trusted = {kid: priv.public_key()} + # The shipped key policy is strict; this test key may certify any group. + p = patch.object(trust, "load_key_policy", lambda dirs=None: {kid: ["*"], "default": []}) + p.start() + self.addCleanup(p.stop) + + def probe(self, arch, h, tarball=None): + return self.store.get((arch, h)) + + def ci(self, sha, archs, tamper=None, key=None): + """What the CI job signs for the tree at *sha*: the artifacts zip.""" + root = os.path.join(self.dir, "ci-" + sha) + shutil.rmtree(root, True) + for path, doc in self.gl.trees[sha].items(): + p = os.path.join(root, path) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(doc if isinstance(doc, bytes) else json.dumps(doc).encode()) + out = os.path.join(self.dir, "out-" + sha, "common-manifest.json") + outs = certify.certify_by_arch(os.path.join(root, "manifests"), key or self.key, out, + probe=self.probe, valid_days=30, source_commit=sha, + only_archs=archs) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + for op, sp, arch in outs: + body = open(op, "rb").read() + if tamper: + doc = json.loads(body) + tamper(arch, doc) + body = json.dumps(doc).encode() + sig = json.dumps(trust.sign_bytes(body, trust.load_private_key(key or self.key))) + else: + sig = open(sp).read() + z.writestr(os.path.basename(op), body) + z.writestr(os.path.basename(op) + ".sig", sig) + return buf.getvalue() + + def run_shadow(self, store=("s3", "bucket"), sync=True): + shadow.discover(self.conn, self.gl, self.settings) + if sync: + certified.sync(self.conn, self.gl, self.cache) + while shadow.compare_one(self.conn, self.gl, store, self.settings, self.cache, + trusted=self.trusted, probe_factory=lambda c: self.probe): + pass + + def cert(self, pid): + return self.conn.execute("SELECT * FROM certifications WHERE pipeline_id = ?", + (pid,)).fetchone() + + def run_rows(self, pid, mode="shadow"): + return {r["arch"]: r for r in self.conn.execute( + "SELECT * FROM sign_runs WHERE certification = ? AND mode = ?", (pid, mode))} + + def kinds(self): + return [e["kind"] for e in events.since(self.conn, limit=1000)] + + +class ChangedArchsTest(_Base): + def tree(self, files): + root = os.path.join(tempfile.mkdtemp(dir=self.dir), "manifests") + for path, doc in files.items(): + p = os.path.join(root, path) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as fh: + fh.write(doc if isinstance(doc, str) else json.dumps(doc)) + return root + + def test_the_ci_jobs_reading(self): + a, b = _bom("b1", "el9", ("p", "h1")), _bom("b2", "el10", ("q", "h2")) + old = self.tree({"lcg/a.json": a, "lcg/b.json": b, "lcg/same.json": a}) + new = self.tree({"lcg/a.json": dict(a, extra=1), "lcg/same.json": a, + "alice/c.json": _bom("b3", "shared", ("r", "h3"))}) + changed, archs = shadow.changed_archs(new, old) + self.assertEqual(changed, ["alice/c.json", "lcg/a.json", "lcg/b.json"]) + self.assertEqual(archs, ["el10", "el9", "shared"]) # b.json deleted: its old side + self.assertEqual(shadow.changed_archs(new, new), ([], None)) + mixed = self.tree({"x/m.json": {"packages": [_pkg("p", "h1"), _pkg("q", "h2", "el10")]}}) + self.assertEqual(shadow.changed_archs(mixed, None)[1], None) # every architecture + broken = self.tree({"x/m.json": "{not json"}) + self.assertEqual(shadow.changed_archs(broken, None)[1], None) + labelled = self.tree({"x/m.json": {"effective_architecture": "el8", "packages": []}}) + self.assertEqual(shadow.changed_archs(labelled, None)[1], ["el8"]) + + def test_the_archive_is_extracted_safely(self): + evil = io.BytesIO() + with tarfile.open(fileobj=evil, mode="w:gz") as tar: + for name in ("top/manifests/lcg/ok.json", "top/manifests/../../evil.json", + "/abs/manifests/x.json", "top/README.md"): + info = tarfile.TarInfo(name) + info.size = 2 + tar.addfile(info, io.BytesIO(b"{}")) + link = tarfile.TarInfo("top/manifests/lcg/link.json") + link.type, link.linkname = tarfile.SYMTYPE, "/etc/passwd" + tar.addfile(link) + + class R: + def repository_archive(self, sha, path=None): + return evil.getvalue() + root = shadow.tree(R(), "abcdef1", self.cache) + found = sorted(os.path.relpath(os.path.join(d, f), root) + for d, _s, fs in os.walk(root) for f in fs) + self.assertEqual(found, ["lcg/ok.json"]) + self.assertFalse(os.path.exists(os.path.join(self.dir, "evil.json"))) + self.assertEqual(shadow.tree(None, "abcdef1", self.cache), root) # cached + + def test_trees_in_use_are_kept(self): + import time + for i, sha in enumerate(("a", "b", "c", "d", "e")): + os.makedirs(os.path.join(self.cache, sha)) + t = time.time() - 100 + i + os.utime(os.path.join(self.cache, sha), (t, t)) + shadow.prune(self.cache, keep=2, protect=("a",)) + self.assertEqual(sorted(os.listdir(self.cache)), ["a", "d", "e"]) + + def test_an_unknown_commit_is_a_skip(self): + with self.assertRaises(shadow.Skip): + shadow.tree(FakeManifests(), "nope", self.cache) + + +class CompareTest(_Base): + def setUp(self): + super().setUp() + self.gl.trees["aaaa000"] = {"manifests/lcg/a.json": _bom("b1", "el9", ("p", "h1"))} + self.gl.trees["aaaa111"] = dict(self.gl.trees["aaaa000"], **{ + "manifests/lcg/b.json": _bom("b2", "el9", ("q", "h2")), + "manifests/alice/c.json": _bom("b3", "el10", ("r", "h3"))}) + self.store = {("el9", "h1"): "sh1", ("el9", "h2"): "sh2", ("el10", "h3"): "sh3"} + + def test_identical_manifests_match(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el10", "el9"])) + self.run_shadow() + c = self.cert(5) + self.assertEqual((c["state"], json.loads(c["archs"]), c["job_id"]), + ("compared", ["el10", "el9"], 50)) + runs = self.run_rows(5) + self.assertEqual({a: r["state"] for a, r in runs.items()}, {"el9": "match", "el10": "match"}) + self.assertEqual((runs["el9"]["packages"], runs["el9"]["source"]), (2, "artifacts")) + self.assertEqual(runs["el9"]["manifest_sha256"], runs["el9"]["signed_sha256"]) + self.assertEqual(self.kinds().count("shadow.match"), 2) + self.assertEqual(shadow.summary(self.conn), {"match": 2}) + self.run_shadow() # nothing more to do + self.assertEqual(self.kinds().count("shadow.match"), 2) + + def test_a_difference_is_a_mismatch_with_what_differs(self): + def tamper(arch, doc): + if arch == "el9": + doc["packages"] = doc["packages"][:1] + [dict(_pkg("x", "h9"), group="lcg")] + doc["sources"] = doc["sources"] + ["b-extra"] + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el10", "el9"], tamper=tamper)) + self.run_shadow() + runs = self.run_rows(5) + self.assertEqual(runs["el10"]["state"], "match") + self.assertEqual(runs["el9"]["state"], "mismatch") + d = json.loads(runs["el9"]["diff"]) + self.assertEqual(d["only_ours"]["count"], 1) + self.assertEqual(d["only_signed"]["first"], ["x@1-1 h9"]) + self.assertEqual(d["sources"]["only_signed"], ["b-extra"]) + ev = [e for e in events.since(self.conn, limit=100) if e["kind"] == "shadow.mismatch"] + self.assertEqual(ev[0]["payload"]["arch"], "el9") + self.assertEqual(ev[0]["subject"], "certification:5") + + def test_a_different_scope_shows_in_sources(self): + # CI certified every architecture (no scope), el8 too, which this push + # did not touch: its manifests' sources cover the el8 build as well. + for sha in ("aaaa000", "aaaa111"): + self.gl.trees[sha]["manifests/lcg/old.json"] = _bom("b0", "el8", ("o", "h0")) + self.store[("el8", "h0")] = "sh0" + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", None)) + self.run_shadow() + runs = self.run_rows(5) + self.assertEqual(runs["el9"]["state"], "mismatch") + self.assertEqual(json.loads(runs["el9"]["diff"])["sources"]["only_signed"], ["b0"]) + self.assertIn("not in the worker", json.loads(runs["el8"]["diff"])["error"]) + + def test_an_architecture_signed_but_not_computed_is_a_mismatch(self): + buf = io.BytesIO(self.ci("aaaa111", ["el10", "el9"])) + with zipfile.ZipFile(buf, "a") as z: + z.writestr("common-manifest-el8.json", json.dumps({"packages": []})) + self.gl.add(5, "aaaa111", "aaaa000", artifacts=buf.getvalue()) + self.run_shadow() + self.assertEqual(self.run_rows(5)["el8"]["state"], "mismatch") + + def test_an_untrusted_signature_is_unverified(self): + other = os.path.join(self.dir, "other.pem") + priv = Ed25519PrivateKey.generate() + with open(other, "wb") as fh: + fh.write(priv.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + os.chmod(other, 0o600) + okid = trust.key_id(priv.public_key()) + trust.load_key_policy = lambda dirs=None: {okid: ["*"]} # undone by the patch + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el10", "el9"], key=other)) + self.run_shadow() + self.assertEqual({r["state"] for r in self.run_rows(5).values()}, {"unverified"}) + self.assertNotIn("shadow.match", self.kinds()) + + def test_the_worker_refusing_what_was_signed(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el10", "el9"])) + self.store[("el9", "h2")] = "other-bytes" # the store now disagrees + self.run_shadow() + runs = self.run_rows(5) + self.assertEqual(list(runs), ["*"]) + self.assertIn("refuses", json.loads(runs["*"]["diff"])["error"]) + + def test_without_artifacts_the_stores_copy_while_it_is_current(self): + art = self.ci("aaaa111", ["el10", "el9"]) + objects = {} + with zipfile.ZipFile(io.BytesIO(art)) as z: + for n in z.namelist(): + objects["MANIFESTS/" + n] = z.read(n) + doc = json.loads(objects["MANIFESTS/common-manifest-el10.json"]) + doc["source_commit"] = "aaaa222" # a later certification replaced it + objects["MANIFESTS/common-manifest-el10.json"] = json.dumps(doc).encode() + self.gl.add(5, "aaaa111", "aaaa000") + self.run_shadow(store=(FakeS3(objects), "bucket")) + runs = self.run_rows(5) + self.assertEqual((runs["el9"]["state"], runs["el9"]["source"]), ("match", "store")) + self.assertEqual(runs["el10"]["state"], "superseded") + + def test_a_full_run_from_the_store_skips_architectures_it_did_not_sign(self): + self.gl.trees["aaaa111"]["manifests/lcg/bad.json"] = b"{broken" # scope: every arch + art = self.ci("aaaa111", None) + objects = {"MANIFESTS/common-manifest-el7.json": json.dumps( + {"source_commit": "s-old", "packages": []}).encode(), + "MANIFESTS/common-manifest-el7.json.sig": b"{}"} + with zipfile.ZipFile(io.BytesIO(art)) as z: + for n in z.namelist(): + objects["MANIFESTS/" + n] = z.read(n) + self.gl.add(5, "aaaa111", "aaaa000") + with self.assertLogs("bits", "WARNING"): # bits skips the unreadable BOM + self.run_shadow(store=(FakeS3(objects), "bucket")) + self.assertEqual(json.loads(self.cert(5)["archs"]), []) + self.assertEqual({a: r["state"] for a, r in self.run_rows(5).items()}, + {"el9": "match", "el10": "match"}) + + def test_failed_jobs_are_skipped_until_retried(self): + self.gl.add(5, "aaaa111", "aaaa000", status="failed") + self.run_shadow() + self.assertEqual((self.cert(5)["state"], self.cert(5)["reason"]), + ("skipped", "certify-and-publish failed")) + self.gl.add(5, "aaaa111", "aaaa000", updated="2026-10-11T11:00:00Z", + artifacts=self.ci("aaaa111", ["el10", "el9"])) + self.run_shadow() + self.assertEqual(self.cert(5)["state"], "compared") + + def test_what_is_not_a_certification(self): + self.gl.add(5, "aaaa000", "aaaa000", artifacts=b"") # nothing changed + self.gl.add(6, "aaaa111", "aaaa000", source="merge_request_event") + self.gl.jobs[5] = [{"id": 50, "name": "certify-and-publish", "status": "success"}] + self.run_shadow() + self.assertEqual(self.cert(5)["reason"], "no manifests changed") + self.assertIsNone(self.cert(6)) + self.gl.add(7, "aaaa111", "aaaa000") + self.gl.jobs[7] = [{"id": 70, "name": "validate-and-merge", "status": "success"}] + self.run_shadow() + self.assertEqual(self.cert(7)["reason"], "no certify-and-publish job") + + def test_a_first_push_diffs_against_the_parent(self): + self.gl.parents["aaaa111"] = ["aaaa000"] + self.gl.add(5, "aaaa111", "0" * 40, artifacts=self.ci("aaaa111", ["el10", "el9"])) + self.run_shadow() + self.assertEqual(self.cert(5)["before_sha"], "aaaa000") + self.assertEqual({r["state"] for r in self.run_rows(5).values()}, {"match"}) + + def test_an_outage_fails_the_step_without_counting(self): + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el10", "el9"])) + self.gl.fail = forge_ops.ForgeError(502, "bad gateway") + with self.assertRaises(forge_ops.ForgeError): + self.run_shadow() + self.assertEqual((self.cert(5)["state"], self.cert(5)["attempts"]), ("pending", 0)) + self.gl.fail = None + err = Exception("timeout") + err.response = {"Error": {"Code": "RequestTimeout"}} + + def probe(arch, h, tarball=None): + if h == "h2": + try: + raise err + except Exception as exc: + raise certify.CertifyError("store check failed") from exc + return self.store.get((arch, h)) + with self.assertRaises(certify.CertifyError): + shadow.compare_one(self.conn, self.gl, ("s3", "b"), self.settings, self.cache, + trusted=self.trusted, probe_factory=lambda c: probe) + self.assertEqual((self.cert(5)["state"], self.cert(5)["attempts"]), ("pending", 0)) + self.assertEqual(self.run_rows(5), {}) + + def test_errors_back_off_then_fail(self): + self.gl.add(5, "aaaa111", "aaaa000") + self.gl.fail = forge_ops.ForgeError(403, "forbidden") + with self.assertLogs("bits_orchestrator.shadow", "WARNING"): + self.run_shadow() + c = self.cert(5) + self.assertEqual((c["state"], c["attempts"]), ("pending", 1)) + self.assertGreater(c["next_ms"], db.now_ms()) + with self.assertLogs("bits_orchestrator.shadow", "WARNING"): + for _ in range(shadow.MAX_ATTEMPTS): + self.conn.execute("UPDATE certifications SET next_ms = 0") + self.run_shadow() + self.assertEqual(self.cert(5)["state"], "failed") + self.assertEqual(self.kinds().count("shadow.failed"), 1) + + def test_old_pipelines_are_not_compared(self): + self.settings.orch_shadow_lookback_days = 1 + self.gl.add(5, "aaaa111", "aaaa000") + self.gl.pipelines[5]["created_at"] = "2026-01-01T00:00:00Z" + shadow.discover(self.conn, self.gl, self.settings) + self.assertIsNone(self.cert(5)) + + +class FakeS3: + def __init__(self, objects): + self.objects = dict(objects) + self.gets = [] + + def get_object(self, Bucket, Key): + self.gets.append(Key) + if Key not in self.objects: + e = Exception("NoSuchKey") + e.response = {"Error": {"Code": "NoSuchKey"}} + raise e + return {"Body": io.BytesIO(self.objects[Key])} + + def head_object(self, Bucket, Key): + if Key not in self.objects: + e = Exception("404") + e.response = {"Error": {"Code": "404"}} + raise e + return {"ETag": '"e-%d"' % len(self.objects[Key]), "ContentLength": len(self.objects[Key])} + + def list_objects_v2(self, Bucket, Prefix, ContinuationToken=None): + return {"Contents": [{"Key": k, "ETag": '"e-%d"' % len(v), "Size": len(v)} + for k, v in sorted(self.objects.items()) if k.startswith(Prefix)]} + + +class ProbeCacheTest(_Base): + def test_only_new_objects_are_hashed(self): + key = "TARS/el9/store/h1/h1abc/p-1-1.el9.tar.gz" + s3 = FakeS3({key: b"bytes"}) + probe = certify.s3_probe(s3, "b", "el9", cache=shadow.DigestCache(self.conn)) + first = probe("el9", "h1abc", "p-1-1.el9.tar.gz") + self.assertTrue(first.startswith("sha256:")) + self.assertEqual(len(s3.gets), 1) + self.assertEqual(probe("el9", "h1abc", "p-1-1.el9.tar.gz"), first) + self.assertEqual(len(s3.gets), 1) # from store_digests + s3.objects[key] = b"other bytes" # a changed object is hashed again + self.assertNotEqual(probe("el9", "h1abc", "p-1-1.el9.tar.gz"), first) + self.assertEqual(len(s3.gets), 2) + self.assertEqual(probe("el9", "h1abc"), probe("el9", "h1abc", "p-1-1.el9.tar.gz")) + self.assertIsNone(probe("el9", "nope", "x.tar.gz")) + s3.head_object = lambda Bucket, Key: (_ for _ in ()).throw(TimeoutError("slow")) + with self.assertRaises(certify.CertifyError): # not "absent": the run fails + probe("el9", "h1abc", "p-1-1.el9.tar.gz") + + +class RedirectTest(unittest.TestCase): + def test_a_redirect_is_followed_without_the_token(self): + seen = [] + + class R: + def __init__(self, status, location=None): + self.status_code, self.headers = status, {"Location": location} if location else {} + + def iter_content(self, n): + return [b"zip"] + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + pass + + def get(url, headers, timeout, stream, allow_redirects): + seen.append((url, dict(headers), allow_redirects)) + return R(302, "https://objects.example/bucket/a.zip?sig=x") if len(seen) == 1 else R(200) + f = forge_ops.GitLabForge("https://gl/api/v4", "tok", "bits/bits-manifests") + with patch.object(forge_ops.requests, "get", get): + self.assertEqual(f.job_artifacts(5), b"zip") + self.assertEqual(seen[0][1]["PRIVATE-TOKEN"], "tok") + self.assertEqual(seen[1][0], "https://objects.example/bucket/a.zip?sig=x") + self.assertNotIn("PRIVATE-TOKEN", seen[1][1]) + self.assertEqual({s[2] for s in seen}, {False}) + + +class BuildByArchTest(unittest.TestCase): + def test_it_is_what_certify_by_arch_signs(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + key = os.path.join(d, "k.pem") + with open(key, "wb") as fh: + fh.write(Ed25519PrivateKey.generate().private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + os.chmod(key, 0o600) + p = patch.object(trust, "load_key_policy", lambda dirs=None: None) + p.start() + self.addCleanup(p.stop) + boms = [_bom("b1", "el9", ("p", "h1")), _bom("b2", "el10", ("q", "h2"))] + probe = lambda a, h, t=None: "s" + h # noqa: E731 + built = dict(certify.build_by_arch(boms, probe=probe, source_commit="c")) + signed = certify.certify_by_arch(boms, key, os.path.join(d, "cm.json"), probe=probe, + source_commit="c") + for op, _sp, arch in signed: + self.assertEqual(shadow.digest(json.load(open(op))), shadow.digest(built[arch])) + + +class ApiTest(_Base): + def setUp(self): + super().setUp() + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": self.path, + "BITS_ADMINS_POLICY": "testbed @alice\n* @carol"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + self.client = TestClient(main.app) + self.gl.trees["aaaa000"] = {} + self.gl.trees["aaaa111"] = {"manifests/lcg/b.json": _bom("b2", "el9", ("q", "h2"))} + self.store = {("el9", "h2"): "sh2"} + self.gl.add(5, "aaaa111", "aaaa000", artifacts=self.ci("aaaa111", ["el9"])) + self.run_shadow() + + def _get(self, user, path): + return self.client.get(path, headers={"Authorization": "Bearer %s" % user}) + + def test_overall_admins_read_certifications(self): + r = self._get("carol", "/ops/certifications") + self.assertEqual(r.status_code, 200) + c = r.json()["certifications"][0] + self.assertEqual((c["pipeline_id"], {(x["mode"], x["state"]) for x in c["sign_runs"]}), + (5, {("shadow", "match"), ("set", "match"), ("replay", "missing")})) + summary = self._get("carol", "/ops/certifications/summary").json() + self.assertEqual((summary["sign_runs"], summary["replay"]), ({"match": 1}, {"missing": 1})) + self.assertEqual(self._get("carol", "/ops/certifications?state=bogus").status_code, 400) + self.assertEqual(self._get("alice", "/ops/certifications").status_code, 403) + self.assertEqual(self.client.get("/ops/certifications").status_code, 401) + + +class WorkerTest(_Base): + def test_the_shadow_runs_on_its_own_thread_only_when_configured(self): + w = worker.Worker(self.conn, config.Settings(env={}), path=self.path) + w.start() + self.assertIsNone(w.shadow) + w = worker.Worker(self.conn, self.settings, interval=0.05, path=self.path) + w.deps.manifests_reader = lambda: self.gl + w.deps.store = lambda: None + self.gl.trees["aaaa000"], self.gl.trees["aaaa111"] = {}, {"manifests/l/b.json": _bom("b", "el9")} + self.gl.add(5, "aaaa111", "aaaa000") + w.start() + self.addCleanup(w.stop) + self.assertEqual(w.shadow.name, "shadow") + import time + + def failed(): + return [e for e in events.since(self.conn, limit=1000) + if e["kind"] == "worker.step_failed" and e["payload"]["step"] == "shadow"] + with self.assertLogs("bits_orchestrator.shadow", "ERROR"): + for _ in range(100): + if failed(): + break + time.sleep(0.05) + w.stop() + w.shadow.join(5) + self.assertIn("BITS_ORCH_STORE", failed()[0]["payload"]["error"]) + self.assertEqual(len(failed()), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_trace.py b/console-backend/tests/test_orch_trace.py new file mode 100644 index 00000000..6387b64c --- /dev/null +++ b/console-backend/tests/test_orch_trace.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Which build published a CVMFS path, and who requested and approved it.""" + +import json +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import db, events, trace + +P = "http://prepub:8080" + + +class TraceTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.calls = [] + + def get(self, data): + def get(url): + self.calls.append(url) + value = data.get(url) + if isinstance(value, Exception): + raise value + return value + return get + + def _rec(self, path, job, outcome="published", ts="2026-10-22T22:05:00.123Z", **kw): + return dict({"ts": ts, "build_id": "1", "job_id": job, "repo": "bits.cern.ch", + "path": path, "outcome": outcome}, **kw) + + def _pipeline(self, pid, community="atlas", certifier="alice", variables=None): + with db.transaction(self.conn): + v = dict({"BITS_CERTIFIER": certifier} if certifier else {}, **(variables or {})) + self.conn.execute( + "INSERT INTO pipelines (id, is_build, community, package, version, variables," + " web_url, outcome, first_seen_ms) VALUES (?, 1, ?, 'Athena', '25.0.71', ?," + " ?, 'failed', 1)", (pid, community, json.dumps(v), "https://gl/p/%d" % pid)) + + def _approve(self, subject, approver): + with db.transaction(self.conn): + cur = self.conn.execute( + "INSERT INTO approvals (kind, subject, groups, approver, created_ms, expires_ms)" + " VALUES ('preapproval', ?, '[\"atlas\"]', ?, 1792700000000, 1792800000000)", + (str(subject), approver)) + events.append(self.conn, "approval.granted", "approval:%d" % cur.lastrowid, + events.Actor("person", approver), {"approval": cur.lastrowid}) + + def test_index_reads_new_and_recent_builds_only_published(self): + pkg = "atlas/x86_64-el9-gcc15-dbg/Packages/openloops/2.1.4-1" + data = {P + "/api/v1/measurements": ["latest", "16131386", "15907127"], + P + "/api/v1/measurements/16131386": [ + self._rec("/" + pkg + "/", "j1"), self._rec("x/y", "j2", outcome="failed"), + "junk"], + P + "/api/v1/measurements/15907127": [ + self._rec("old/pkg", "j0", ts="2026-10-01T06:00:00Z", replaced=True)]} + self.assertEqual(trace.step(self.conn, [P + "/"], get=self.get(data)), 2) + rows = self.conn.execute("SELECT * FROM published_paths ORDER BY job_id").fetchall() + self.assertEqual([(r["path"], r["build_id"], r["replaced"]) for r in rows], + [("old/pkg", "15907127", 1), (pkg, "16131386", 0)]) + self.calls.clear() + trace.step(self.conn, [P], get=self.get(data)) # all read just now: only the list + self.assertEqual(self.calls, [P + "/api/v1/measurements"]) + + def test_a_bad_or_empty_build_is_read_again_and_does_not_block_others(self): + import requests + data = {P + "/api/v1/measurements": ["3", "2", "1"], + P + "/api/v1/measurements/3": requests.HTTPError("500"), + P + "/api/v1/measurements/2": [], + P + "/api/v1/measurements/1": [self._rec("a/b", "j", ts="2026-01-01T00:00:00Z")]} + self.assertEqual(trace.step(self.conn, [P], get=self.get(data)), 3) + self.assertEqual(self.conn.execute("SELECT count(*) FROM published_paths").fetchone()[0], 1) + with db.transaction(self.conn): # as if read a while ago + self.conn.execute("UPDATE traced_builds SET checked_ms = 0") + self.calls.clear() + trace.step(self.conn, [P], get=self.get(data)) + # 3 and 2 are recent (no records yet: from when first seen); 1 is old. + self.assertEqual(sorted(self.calls), [P + "/api/v1/measurements", + P + "/api/v1/measurements/2", + P + "/api/v1/measurements/3"]) + + def test_an_unreachable_prepub_does_not_stop_the_others(self): + data = {"http://down/api/v1/measurements": OSError("refused"), + P + "/api/v1/measurements": ["7"], + P + "/api/v1/measurements/7": [self._rec("a/b", "j")]} + self.assertEqual(trace.step(self.conn, ["http://down", P], get=self.get(data)), 1) + + def test_lookup_finds_the_package_above_the_file_and_who_approved(self): + data = {P + "/api/v1/measurements": ["16131386"], + P + "/api/v1/measurements/16131386": [self._rec("atlas/Packages/ROOT/6-1", "j1")]} + trace.step(self.conn, [P], get=self.get(data)) + self._pipeline(16131386) + self._approve(16131386, "bob") + self._pipeline(15000000, community="lhcb", certifier="carol") + with db.transaction(self.conn): + for pid in (15000000, 0): # 0: the same build certified from the CLI + self.conn.execute("INSERT INTO builds (pipeline_id, build_id, recorded_ms)" + " VALUES (?, 'release_gcc15_opt-c193e4dbe60c', 1)", (pid,)) + out = trace.lookup(self.conn, "bits.cern.ch", "/atlas/Packages/ROOT/6-1/lib/libCore.so", + build_id="release_gcc15_opt-c193e4dbe60c", visible={"atlas"}) + self.assertEqual(out["published_path"], "atlas/Packages/ROOT/6-1") + (pub,) = out["publishes"] + self.assertEqual((pub["pipeline"], pub["requested_by"], pub["approval"]["approver"]), + (16131386, "alice", "bob")) + self.assertTrue(pub["approval"]["event"]["hash"]) + self.assertEqual(pub["published_at"], "2026-10-22T22:05:00Z") + (built,) = out["built_by_pipelines"] + self.assertEqual(built, {"pipeline": 15000000, "known": True, "visible": False}) + self.assertEqual(trace.lookup(self.conn, "bits.cern.ch", "atlas/other")["publishes"], []) + + def test_the_approval_in_force_and_what_others_see(self): + self._pipeline(5) + self._approve(5, "first") + with db.transaction(self.conn): + self.conn.execute("UPDATE approvals SET revoked_ms = 1792700100000") + self._approve(5, "second") + self.assertEqual(trace.pipeline(self.conn, 5)["approval"]["approver"], "second") + with db.transaction(self.conn): + self.conn.execute("INSERT INTO published_paths (repo, path, build_id, job_id, ts_ms," + " prepub) VALUES ('r', 'p', '5', 'j', 1, 'http://prepub')") + (pub,) = trace.lookup(self.conn, "r", "p/x", visible={"lhcb"})["publishes"] + self.assertEqual(pub, {"pipeline": 5, "known": True, "visible": False}) + + def test_a_queued_build_names_its_requester(self): + self._pipeline(9, certifier="") + with db.transaction(self.conn): + self.conn.execute( + "INSERT INTO build_queue (batch, community, requested_by, ref, variables, digest," + " target, state, pipeline_id, created_ms, updated_ms) VALUES ('b1', 'atlas'," + " 'dave', 'main', '{}', 'd', 'x', 'dispatched', 9, 1, 1)") + self.assertEqual(trace.pipeline(self.conn, 9)["requested_by"], "dave") + self._pipeline(10, certifier="mallory") # a variable anyone can set + with db.transaction(self.conn): + self.conn.execute("UPDATE build_queue SET pipeline_id = 10") + p = trace.pipeline(self.conn, 10) + self.assertEqual((p["requested_by"], p["requested_by_from"]), ("dave", "queue")) + self.assertIsNone(trace.pipeline(self.conn, "release_x")) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_vuln_triage.py b/console-backend/tests/test_orch_vuln_triage.py new file mode 100644 index 00000000..94ba8d35 --- /dev/null +++ b/console-backend/tests/test_orch_vuln_triage.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""orch.vuln_triage: the admins' triage of vulnerability findings (VEX).""" + +import os +import shutil +import tempfile +import unittest + +from console_backend.orch import db, events, vuln_triage + +DAY = 86400 * 1000 + + +class TriageTest(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + def add(self, vuln="CVE-2024-1", state="false_positive", **kw): + return vuln_triage.add(self.conn, kw.pop("community", "ATLAS"), vuln, state, + kw.pop("reason", "checked"), kw.pop("by", "ann"), **kw) + + def test_a_rule_is_an_event_and_revoked_once(self): + rid = self.add(package="libxml2", version="2.9.13", days=30, now=1000) + r = vuln_triage.get(self.conn, rid) + self.assertEqual((r["community"], r["expires_ms"], r["author"]), ("atlas", 1000 + 30 * DAY, "ann")) + self.assertTrue(vuln_triage.revoke(self.conn, rid, "bob")) + self.assertFalse(vuln_triage.revoke(self.conn, rid, "bob")) + ev = [(e["kind"], e["actor"]["id"]) for e in events.since(self.conn)] + self.assertEqual(ev, [("sbom.triaged", "ann"), ("sbom.triage_revoked", "bob")]) + self.assertEqual(vuln_triage.listing(self.conn), []) + self.assertEqual(len(vuln_triage.listing(self.conn, revoked=True)), 1) + + def test_bad_rules_are_refused(self): + for kw in ({"vuln": "x"}, {"vuln": "CVE 1"}, {"state": "fixed"}, {"state": "not_affected"}, + {"state": "not_affected", "justification": "because"}, {"version": "1.0"}, + {"reason": " "}, {"days": 0.5}, {"days": 1000}, {"days": "x"}, + {"package": "a\nb"}, {"community": ""}, {"vuln": "*"}): # *: a package + with self.assertRaises(vuln_triage.Invalid, msg=kw): + self.add(**kw) + self.assertEqual(vuln_triage.listing(self.conn, revoked=True), []) + rid = self.add(state="not_affected", justification="code_not_reachable") + self.assertEqual(vuln_triage.get(self.conn, rid)["justification"], "code_not_reachable") + + def test_a_package_taken_out_of_the_analysis(self): + cve = self.add(package="perf", reason="this one checked") + anything = self.add(vuln="*", package="perf", reason="a host tool, not shipped") + rules = vuln_triage.Rules.load(self.conn) + rule = lambda vid, name: (rules.rule("atlas", vid, name, "6.1") or {}).get("id") + self.assertEqual((rule("CVE-2024-1", "perf"), rule("CVE-2099-9", "perf")), (cve, anything)) + self.assertIsNone(rule("CVE-2099-9", "zlib")) + self.assertIsNone(rule("licence:unknown", "perf")) # licences stay + self.assertIsNone(rules.rule("lhcb", "CVE-2099-9", "perf", "6.1")) # its community only + (v,) = vuln_triage.annotate([{"id": "CVE-2099-9", "affects": [{"ref": "p"}]}], + {"p": {"name": "perf", "version": "6.1"}}, rules, "atlas") + self.assertIn({"name": "bits:triage:vuln_id", "value": "*"}, v["properties"]) + old = self.add(vuln="*", package="perf", version="5.0", reason="old perf") + every = self.add(vuln="CVE-2099-9", reason="every package") + rules = vuln_triage.Rules.load(self.conn) + self.assertEqual(rules.rule("atlas", "CVE-2099-9", "perf", "6.1")["id"], every) # its own first + self.assertEqual(rules.rule("atlas", "CVE-2099-8", "perf", "5.0")["id"], old) # the version's + self.assertEqual(rules.rule("atlas", "CVE-2099-8", "perf", "6.1")["id"], anything) + + def test_the_most_specific_rule_covers_a_finding_until_it_expires(self): + everything = self.add() + package = self.add(package="LibXML2") + exact = self.add(package="libxml2", version="2.9.13", days=1, now=0) + self.add(vuln="CVE-2024-2", package="libxml2") + rules = vuln_triage.Rules.load(self.conn, now=1) + rule = lambda name, version, alias="": (rules.rule("atlas", "CVE-2024-1", name, version, alias) + or {}).get("id") + self.assertEqual(rule("libxml2", "2.9.13"), exact) + self.assertEqual(rule("libxml2", "2.9.14"), package) + self.assertEqual(rule("zlib1g", "1.3", alias="libxml2"), package) # its bits name + self.assertEqual(rule("zlib", "1.3"), everything) + self.assertIsNone(rules.rule("lhcb", "CVE-2024-1", "zlib", "1.3")) + later = vuln_triage.Rules.load(self.conn, now=2 * DAY) + self.assertEqual(later.rule("atlas", "CVE-2024-1", "libxml2", "2.9.13")["id"], package) + + def test_the_served_sbom_carries_the_rules_as_analysis(self): + self.add(package="zlib", state="not_affected", justification="code_not_present", + reason="we never parse gzip") + self.add(vuln="CVE-2024-2", state="will_not_fix", reason="accepted") + comps = {"a": {"name": "libxml2", "version": "2.9.13"}, + "z": {"name": "zlib1g", "version": "1.3", + "properties": [{"name": "bits:package", "value": "zlib"}]}} + vulns = [{"bom-ref": "vuln-CVE-2024-1", "id": "CVE-2024-1", "affects": [{"ref": "a"}, {"ref": "z"}]}, + {"bom-ref": "vuln-CVE-2024-2", "id": "CVE-2024-2", "affects": [{"ref": "a"}]}, + {"bom-ref": "vuln-CVE-3", "id": "CVE-3", "affects": [{"ref": "a"}]}] + out = vuln_triage.annotate(vulns, comps, vuln_triage.Rules.load(self.conn), "atlas") + self.assertEqual([(v["bom-ref"], [a["ref"] for a in v["affects"]], v.get("analysis", {}).get("state")) + for v in out], [ + ("vuln-CVE-2024-1", ["a"], None), + ("vuln-CVE-2024-1-triage-1", ["z"], "not_affected"), + ("vuln-CVE-2024-2-triage-2", ["a"], "exploitable"), + ("vuln-CVE-3", ["a"], None)]) + self.assertEqual(out[1]["analysis"], {"state": "not_affected", "justification": "code_not_present", + "detail": "we never parse gzip (ann)"}) + self.assertEqual(out[2]["analysis"]["response"], ["will_not_fix"]) + self.assertEqual({p["name"]: p["value"] for p in out[1]["properties"]}["bits:triage:package"], "zlib") + self.assertNotIn("properties", out[0]) + self.assertEqual(vuln_triage.annotate(vulns, comps, vuln_triage.Rules([]), "atlas"), vulns) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_watches.py b/console-backend/tests/test_orch_watches.py new file mode 100644 index 00000000..45b0dbdd --- /dev/null +++ b/console-backend/tests/test_orch_watches.py @@ -0,0 +1,341 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Commit CI: the branches a schedule follows, and its commit runs (orch/watches.py).""" + +import datetime as dt +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from console_backend.orch import db, events, nightlies as nl, schedules as sch, watches as w +from test_orch_nightlies import Reader, _vars + +UTC = dt.timezone.utc +REPO = "https://github.com/bitsorg/lcg.bits.git" +ROOT = "https://github.com/root-project/root.git" +T0 = int(dt.datetime(2026, 10, 28, 10, 0, tzinfo=UTC).timestamp() * 1000) +MIN = 60 * 1000 + + +def _pkt(text): + b = text.encode() + return b"%04x" % (len(b) + 4) + b + + +class Resp: + def __init__(self, status, content=b"", chunks=None): + self.status_code, self.chunks, self.closed = status, chunks or [content], False + + def iter_content(self, n): + return iter(self.chunks) + + def close(self): + self.closed = True + + +def _addr(ip): + return lambda host, port, **kw: [(2, 1, 6, "", (ip, port))] + + +ANSWER = (_pkt("%s refs/heads/main-old\n" % ("1" * 40)) + + _pkt("%s refs/heads/main symref-target:x\n" % ("2" * 40)) + b"0000") + + +@patch("console_backend.orch.watches.socket.getaddrinfo", _addr("140.82.121.3")) +class HeadTest(unittest.TestCase): + @patch("console_backend.orch.watches.requests.post") + def test_the_branchs_own_line_of_a_protocol_v2_answer(self, post): + post.return_value = Resp(200, ANSWER) + self.assertEqual(w.head(REPO, "main"), "2" * 40) + args = post.call_args + self.assertEqual(args.args[0], REPO + "/git-upload-pack") + self.assertIn(b"ref-prefix refs/heads/main\n", args.kwargs["data"]) + self.assertEqual(args.kwargs["headers"]["Git-Protocol"], "version=2") + self.assertEqual((args.kwargs["allow_redirects"], args.kwargs["stream"]), (False, True)) + self.assertIsNotNone(args.kwargs["auth"]) # not ~/.netrc's + self.assertTrue(post.return_value.closed) + post.return_value = Resp(200, b"0000") + self.assertEqual(w.head(REPO, "nope"), "") + w.head("https://gitlab.cern.ch/atlas/athena", "main") # GitLab wants the .git + self.assertEqual(post.call_args.args[0], "https://gitlab.cern.ch/atlas/athena.git/git-upload-pack") + + @patch("console_backend.orch.watches.requests.post") + def test_anything_but_a_whole_answer_is_an_error_not_no_branch(self, post): + for bad in (Resp(401, b"Repository not found."), Resp(302), Resp(200, b"zzzz"), Resp(200, b""), + Resp(200, ANSWER[:-4]), Resp(200, ANSWER[:30]), Resp(200, _pkt("ERR upload-pack: not our ref") + b"0000"), + Resp(200, b"001e# service=git-upload-pack\n0000"), + Resp(200, _pkt("%s refs/heads/main\n" % ("g" * 40)) + b"0000"), + Resp(200, chunks=[b"0" * 8192] * 9)): + post.return_value = bad + with self.assertRaises(w.WatchError, msg=bad.chunks[0][:40]): + w.head(REPO, "main") + with patch("console_backend.orch.watches.time.monotonic", side_effect=[0, 0, 99]): + post.return_value = Resp(200, chunks=[b"0", b"0"]) + with self.assertRaisesRegex(w.WatchError, "no whole answer"): + w.head(REPO, "main") + + @patch("console_backend.orch.watches.requests.post") + def test_only_a_public_https_host(self, post): + post.return_value = Resp(200, ANSWER) + for ip in ("127.0.0.1", "10.1.2.3", "169.254.169.254", "::1", "fd00::1", "192.168.0.1"): + with patch("console_backend.orch.watches.socket.getaddrinfo", _addr(ip)): + with self.assertRaisesRegex(w.WatchError, "not a public address"): + w.head(REPO, "main") + for url in ("http://github.com/x/y.git", "https://github.com:99999/x/y.git"): + with self.assertRaises(w.WatchError): + w.head(url, "main") + post.assert_not_called() + + +class _Watched(unittest.TestCase): + def setUp(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + self.conn = db.connect(os.path.join(d, "orch.db")) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + self.heads = {(REPO, "LCG_110"): "a" * 40, (ROOT, "master"): "b" * 40} + self.reader = Reader() + + def get(self, url, branch): + h = self.heads.get((url, branch)) + if isinstance(h, Exception): + raise h + return h + + def schedule(self, **more): + p = {"name": "dev3", "package": "ROOT", "platforms": ["x86_64-el9"], "time": "02:00", + "variants": ["gcc15::opt", "clang::opt"], "certify": True, "cvmfs": True, + "watch": "%s@LCG_110;%s@master" % (REPO, ROOT)} + p.update(more) + return sch.create(self.conn, sch.check(p, "LCG"), "alice") + + def step(self, now): + return w.step(self.conn, self.reader, "main", UTC, now=now, every_s=120, get_head=self.get) + + def runs(self): + return [(r["variant"], r["fired_by"], r["state"]) for r in self.conn.execute( + "SELECT * FROM schedule_runs ORDER BY id")] + + +class WatchTest(_Watched): + def test_a_branch_first_seen_is_not_built(self): + self.schedule() + self.assertEqual(self.step(T0), 0) + self.assertEqual(self.runs(), []) + self.assertEqual({(h["branch"], h["sha"], h["built_sha"]) for h in w.heads(self.conn, 1)}, + {("LCG_110", "a" * 40, "a" * 40), ("master", "b" * 40, "b" * 40)}) + + def test_a_moved_head_builds_every_cell_once_it_held_still_and_certifies_nothing(self): + self.schedule() + self.step(T0) + self.heads[(ROOT, "master")] = "c" * 40 + self.assertEqual(self.step(T0 + 1 * MIN), 0) # not read again so soon + self.assertEqual(self.step(T0 + 3 * MIN), 0) # read: moved, not settled + self.heads[(ROOT, "master")] = "d" * 40 # another push + self.assertEqual(self.step(T0 + 6 * MIN), 0) + self.assertEqual(self.step(T0 + 10 * MIN), 0) # 4 minutes still: not yet + self.assertEqual(self.step(T0 + 12 * MIN), 1) # 6: the newest heads + self.assertEqual(self.runs(), [("clang::opt", "commit", "queued"), ("gcc15::opt", "commit", "queued")]) + r = self.conn.execute("SELECT * FROM schedule_runs").fetchone() + self.assertEqual(json.loads(r["commits"]), {"%s@LCG_110" % REPO: "a" * 40, "%s@master" % ROOT: "d" * 40}) + for q in self.conn.execute("SELECT * FROM build_queue"): + v = _vars(json.loads(q["variables"])) + self.assertNotIn("CERTIFY", v) + self.assertNotIn("PUBLISH_CVMFS", v) + self.assertEqual(q["certify"], 0) + self.assertEqual(self.step(T0 + 20 * MIN), 0) # built: nothing new + kinds = [e["kind"] for e in events.since(self.conn, limit=1000)] + self.assertEqual(kinds.count("schedule.head_moved"), 2) + + def test_a_run_still_going_holds_the_next_until_it_ends(self): + self.schedule() + self.step(T0) + self.heads[(REPO, "LCG_110")] = "e" * 40 + self.step(T0 + 3 * MIN) + self.step(T0 + 9 * MIN) + self.assertEqual(len(self.runs()), 2) + self.heads[(REPO, "LCG_110")] = "f" * 40 + self.step(T0 + 12 * MIN) + self.assertEqual(self.step(T0 + 20 * MIN), 0) # its first commit run goes on + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed'") + self.assertEqual(self.step(T0 + 21 * MIN), 1) + self.assertEqual(json.loads(self.conn.execute("SELECT commits FROM schedule_runs ORDER BY id DESC") + .fetchone()[0])["%s@LCG_110" % REPO], "f" * 40) + + def test_an_unreadable_repository_is_retried_and_the_others_still_fire(self): + self.schedule() + self.schedule(name="other", watch="%s@LCG_110" % REPO, variants=[]) + self.step(T0) + self.heads[(ROOT, "master")] = w.WatchError("root: HTTP 503") + self.heads[(REPO, "LCG_110")] = "9" * 40 + with self.assertRaises(w.WatchError): + self.step(T0 + 3 * MIN) + with self.assertRaises(w.WatchError): + self.step(T0 + 9 * MIN) + fired = {r["schedule_id"] for r in self.conn.execute("SELECT * FROM schedule_runs")} + self.assertEqual(fired, {1, 2}) # 1: its LCG_110 moved, ROOT as it was + + def test_a_failing_repository_fails_the_step_until_a_read_reads_it(self): + self.schedule() + self.step(T0) + self.heads[(ROOT, "master")] = ValueError("Port out of range") # not a WatchError either + self.heads[(REPO, "LCG_110")] = "9" * 40 + for m in (3, 4): # read at 3; 4 is between reads + with self.assertRaisesRegex(w.WatchError, "Port out of range"): + self.step(T0 + m * MIN) + self.heads[(ROOT, "master")] = "b" * 40 + self.step(T0 + 5 * MIN) # read again: no error + self.step(T0 + 6 * MIN) + self.assertEqual(self.step(T0 + 9 * MIN), 1) # LCG_110, read at 3, still fires + + def test_a_failed_repository_is_not_asked_again_in_the_pass(self): + self.schedule(watch="%s@master;%s@v6-36" % (ROOT, ROOT)) + asked = [] + + def get(url, branch): + asked.append(branch) + raise w.WatchError("root: HTTP 503") + with self.assertRaises(w.WatchError): + w.step(self.conn, self.reader, "main", UTC, now=T0, every_s=120, get_head=get) + self.assertEqual(asked, ["master"]) + + def test_pushed_back_to_what_was_built_waits_anew(self): + self.schedule() + self.step(T0) + self.heads[(REPO, "LCG_110")] = "e" * 40 + self.step(T0 + 3 * MIN) + self.heads[(REPO, "LCG_110")] = "a" * 40 # force-pushed back + self.step(T0 + 6 * MIN) + self.assertEqual(self.step(T0 + 12 * MIN), 0) + self.heads[(REPO, "LCG_110")] = "f" * 40 + for m in range(30, 57, 3): # a push each read: waits 30 from 30 + self.heads[(REPO, "LCG_110")] = "%02d" % m * 20 + self.assertEqual(self.step(T0 + m * MIN), 0, m) + self.heads[(REPO, "LCG_110")] = "57" * 20 + self.assertEqual(self.step(T0 + 60 * MIN), 1) + + def test_a_paused_schedule_or_a_dropped_branch_forgets_its_heads(self): + s = self.schedule() + self.step(T0) + p = dict(name="dev3", package="ROOT", platforms=["x86_64-el9"], time="02:00", watch="%s@LCG_110" % REPO) + sch.update(self.conn, s["id"], sch.check(p, "LCG"), "alice", s["revision"]) + self.step(T0 + 3 * MIN) + self.assertEqual([h["branch"] for h in w.heads(self.conn, s["id"])], ["LCG_110"]) + sch.set_enabled(self.conn, s["id"], False, "alice") + self.step(T0 + 6 * MIN) + self.assertEqual(w.heads(self.conn, s["id"]), []) + + def test_commit_rows_beside_the_nightly_cells(self): + self.schedule(variants=[]) + self.step(T0) + self.heads[(REPO, "LCG_110")] = "7" * 40 + self.step(T0 + 3 * MIN) + self.step(T0 + 9 * MIN) + m = nl.matrix(self.conn, {"lcg"}, 2, UTC, now=T0 + 10 * MIN) + (row,) = m["rows"] + self.assertEqual(row["cells"], {}) + (c,) = row["commits"] + self.assertEqual((c["commits"]["%s@LCG_110" % REPO], list(c["cells"]), c["cells"]["x86_64-el9"]["state"]), + ("7" * 40, ["x86_64-el9"], "queued")) + + def test_a_branch_pushed_to_all_the_time_still_builds(self): + self.schedule() + self.step(T0) + for i in range(1, 16): # a push every 3 minutes + self.heads[(REPO, "LCG_110")] = "%02d" % i * 20 + fired = self.step(T0 + i * 3 * MIN) + if fired: + break + self.assertEqual(i * 3, 33) # the first read MAX_WAIT_MS after the first + + def test_a_branch_gone_or_new_builds_nothing_nor_holds_the_others(self): + self.schedule() + self.heads[(ROOT, "master")] = "" # not there yet + self.step(T0) + self.heads[(ROOT, "master")] = "c" * 40 # created: seen, not built + self.step(T0 + 3 * MIN) + self.assertEqual(self.step(T0 + 9 * MIN), 0) + self.heads[(ROOT, "master")] = "" # deleted + self.heads[(REPO, "LCG_110")] = "e" * 40 + self.step(T0 + 12 * MIN) + self.assertEqual(self.step(T0 + 18 * MIN), 1) + r = self.conn.execute("SELECT commits FROM schedule_runs").fetchone() + self.assertEqual(json.loads(r[0]), {"%s@LCG_110" % REPO: "e" * 40}) + + def test_a_nightly_is_not_skipped_for_a_commit_run_nor_the_other_way(self): + s = self.schedule(variants=[]) + self.step(T0) + self.heads[(REPO, "LCG_110")] = "e" * 40 + self.step(T0 + 3 * MIN) + self.step(T0 + 9 * MIN) # a commit run, queued + cfg = nl.config(self.reader, "LCG", "main") + with db.transaction(self.conn): + self.assertEqual(nl.run_now(self.conn, s["id"], [], cfg, "main", UTC, "alice", now=T0 + 10 * MIN), + {"x86_64-el9": "queued"}) + self.heads[(REPO, "LCG_110")] = "f" * 40 + self.step(T0 + 12 * MIN) + self.assertEqual(self.step(T0 + 18 * MIN), 0) # its own commit run holds it + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed' WHERE fired_by = 'commit'") + self.assertEqual(self.step(T0 + 19 * MIN), 1) # the manual run does not + + def test_an_unreadable_configuration_is_said_after_the_others_fired(self): + self.schedule() + self.step(T0) + self.heads[(REPO, "LCG_110")] = "e" * 40 + self.step(T0 + 3 * MIN) + with patch.object(nl, "config", side_effect=RuntimeError("no build.yaml")): + with self.assertRaisesRegex(w.WatchError, "no build.yaml"): + self.step(T0 + 9 * MIN) + self.assertEqual(self.runs(), []) + self.assertEqual(self.step(T0 + 10 * MIN), 1) + + def test_a_deleted_schedule_leaves_no_heads(self): + s = self.schedule() + self.step(T0) + sch.delete(self.conn, s["id"], "alice") + self.step(T0 + 3 * MIN) + self.assertEqual(self.conn.execute("SELECT COUNT(*) FROM schedule_watches").fetchone()[0], 0) + + def test_each_fire_its_own_row_even_with_the_same_heads(self): + s = self.schedule(variants=[]) + self.step(T0) + for k, sha in enumerate(("e", "f", "e"), 1): # e, then f, then back to e + self.heads[(REPO, "LCG_110")] = sha * 40 + self.step(T0 + k * 20 * MIN) + self.assertEqual(self.step(T0 + k * 20 * MIN + 6 * MIN), 1) + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed'") + rows = nl._commit_rows(self.conn, s["id"], T0 + 90 * MIN) + self.assertEqual([r["commits"]["%s@LCG_110" % REPO][0] for r in rows], ["e", "f", "e"]) + self.assertEqual(nl._commit_rows(self.conn, s["id"], T0 + 31 * 86400 * 1000), []) + for k in range(4, 14): + self.heads[(REPO, "LCG_110")] = "%02d" % k * 20 + self.step(T0 + k * 20 * MIN) + self.assertEqual(self.step(T0 + k * 20 * MIN + 6 * MIN), 1) + with db.transaction(self.conn): + self.conn.execute("UPDATE schedule_runs SET state = 'passed'") + rows = nl._commit_rows(self.conn, s["id"], T0 + 300 * MIN) + self.assertEqual([r["commits"]["%s@LCG_110" % REPO][:2] for r in rows], ["%02d" % k for k in range(13, 3, -1)]) + + +class WatchListTest(unittest.TestCase): + def test_what_a_schedule_may_follow_and_its_hash(self): + p = {"name": "n", "package": "ROOT", "platforms": ["x86_64-el9"], "time": "02:00"} + plain = sch.check(p, "lcg") + self.assertEqual(plain["watch"], "") + s = sch.check(dict(p, watch="%s@master;%s@LCG_110" % (ROOT, REPO)), "lcg") + self.assertEqual(sch.watches(s), [(REPO, "LCG_110"), (ROOT, "master")]) + self.assertEqual(sch.content_sha256(s), sch.content_sha256(plain)) # when, not what: no new approval + for bad in (REPO, "http://x/y@main", "https://h.org:99999/x@main", "%s@a;%s@a" % (REPO, REPO), ";".join("%s@b%d" % (REPO, i) for i in range(9))): + with self.assertRaises(sch.Invalid, msg=bad): + sch.check(dict(p, watch=bad), "lcg") + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_orch_worker_state.py b/console-backend/tests/test_orch_worker_state.py new file mode 100644 index 00000000..c76e6e2a --- /dev/null +++ b/console-backend/tests/test_orch_worker_state.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The worker's failing steps (events.failing_steps) and GET /ops/worker.""" + +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from console_backend import config, main +from console_backend.orch import db, events + + +def _ev(conn, kind, payload=None, subject="worker"): + with db.transaction(conn): + events.append(conn, kind, subject, events.WORKER, payload) + + +class WorkerStateTest(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + self.path = os.path.join(self.dir, "orch.db") + self.conn = db.connect(self.path) + self.addCleanup(self.conn.close) + db.migrate(self.conn) + + def test_a_step_fails_from_its_first_error_until_it_recovers_or_the_worker_restarts(self): + _ev(self.conn, "worker.started") + _ev(self.conn, "worker.step_failed", {"step": "scan", "error": "a"}) + first = events.failing_steps(self.conn)["scan"]["since_ms"] + _ev(self.conn, "worker.step_failed", {"step": "scan", "error": "b"}) + _ev(self.conn, "worker.step_failed", {"step": "dispatch", "error": "c"}) + _ev(self.conn, "worker.step_failed", {"step": "x", "error": "y"}, subject="other") + got = events.failing_steps(self.conn) + self.assertEqual(sorted(got), ["dispatch", "scan"]) + self.assertEqual(got["scan"], {"error": "b", "since_ms": first}) + _ev(self.conn, "worker.step_recovered", {"step": "dispatch", "error": None}) + self.assertEqual(sorted(events.failing_steps(self.conn)), ["scan"]) + _ev(self.conn, "worker.started") + self.assertEqual(events.failing_steps(self.conn), {}) + + def test_the_api_shows_errors_to_overall_admins_only(self): + main.settings = config.Settings(env={ + "GITLAB_API_URL": "https://gl/api/v4", "BITS_ORCH_DB": self.path, + "BITS_ADMINS_POLICY": "testbed @alice\n* @carol"}) + p = patch.object(main.identity, "verify_gitlab_token", lambda a, t, *x, **k: t or None) + p.start() + self.addCleanup(p.stop) + client = TestClient(main.app) + h = lambda u: {"Authorization": "Bearer " + u} + _ev(self.conn, "worker.started") + _ev(self.conn, "worker.step_failed", {"step": "scan", "error": "scanner: 503"}) + self.assertEqual(client.get("/ops/worker").status_code, 401) + self.assertEqual(client.get("/ops/worker", headers=h("dave")).status_code, 403) + self.assertEqual(client.get("/ops/worker?community=other", headers=h("alice")).status_code, 403) + mine = client.get("/ops/worker?community=testbed", headers=h("alice")).json() + self.assertEqual(([f["step"] for f in mine["failing"]], "error" in mine["failing"][0], + mine["last_pass_ms"]), (["scan"], False, None)) + open(self.path + ".heartbeat", "w").close() + all_ = client.get("/ops/worker", headers=h("carol")).json() + self.assertEqual(all_["failing"][0]["error"], "scanner: 503") + self.assertIsInstance(all_["last_pass_ms"], int) + self.assertEqual(all_["interval_s"], 30.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/console-backend/tests/test_sign_retired.py b/console-backend/tests/test_sign_retired.py new file mode 100644 index 00000000..c8f5cc05 --- /dev/null +++ b/console-backend/tests/test_sign_retired.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""M8b: once the orchestrator owns the set, the backend signs nothing.""" + +import unittest + +from fastapi.testclient import TestClient + +from console_backend import config, main + + +class SignRetiredTest(unittest.TestCase): + def setUp(self): + self.old = main.settings + self.addCleanup(setattr, main, "settings", self.old) + self.client = TestClient(main.app, follow_redirects=False) + + def test_the_sign_routes_are_gone_once_the_orchestrator_owns_the_set(self): + main.settings = config.Settings(env={"BITS_ORCH_OWNS_SET": "1"}) + for method, path in (("post", "/sign"), ("post", "/sign/preapproved?build_id=1"), + ("post", "/sign/cli/request"), ("get", "/sign/cli/x/result")): + r = getattr(self.client, method)(path) + self.assertEqual(r.status_code, 410, path) + self.assertNotEqual(self.client.get("/healthz").status_code, 410) + + def test_before_the_switch_they_answer_as_before(self): + main.settings = config.Settings(env={}) + self.assertNotEqual(self.client.post("/sign/preapproved?build_id=1").status_code, 410) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index c56a920f..e39fed0e 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -161,6 +161,44 @@ update source mirrors ← refresh the reference mirror of each source r upload the tarball to the write store (if configured) ``` +The console shows each package's position in the selected plan and its current +stage. Source fetching includes the URL or repository and ref, or the local cache +or remote store being used. Configuration, compilation, and installation are +reported separately. For example: + +```text +[3/16] Python@3.13.11: Fetching sources from https://www.python.org/ftp/python/3.13.11/Python-3.13.11.tar.xz +[3/16] Python@3.13.11: Configuring +[3/16] Python@3.13.11: Compiling +[3/16] Python@3.13.11: Installing +[3/16] Python@3.13.11: done +``` + +The count includes packages reused from an existing installation or cache and +excludes the internal `defaults-release` package. `--only-deps` counts only the +selected dependencies. With multiple builders, each package keeps its position +from the plan, so messages can arrive out of numerical order. Stage changes also +appear in redirected output and CI logs. Terminal builds retain compiler +percentages within a stage; `--debug` shows the full command output. Stage +transitions are detected from the executed shell commands for Autotools, CMake, +Make, Ninja, Meson, Python builds, and the `Configure`, `Make`, and `MakeInstall` +recipe hooks. Post-install hooks, packaging, and relocation belong to the +installation stage. Recipes using custom tools can mark their own transitions: + +```bash +echo bits:stage:configure +# Custom configuration commands +echo bits:stage:compile +# Custom compilation commands +echo bits:stage:install +# Custom installation commands +``` + +A recipe that does not expose a recognizable command or marker retains the +`Running build recipe` label until installation. Incremental builds use the +same stage detection. Only steps that actually run are shown; cached binaries +report archive unpacking and installation. + ### Artifact resolution order (trust-tiered reuse) Before compiling a package, bits looks for an existing, trusted build of the @@ -208,13 +246,23 @@ Reuse from the remote archive (tier 3) is controlled by these `bits build` optio `~/.config/bits/keys`). - `--require-signed-reuse` (the default) — fail closed: a tarball fetched from the remote store is reused only when a trusted signed manifest lists its hash - **and** its sha256 matches. Unlisted → discard and rebuild; sha256 mismatch → fatal (tampering). - Local build-node and CVMFS artifacts are unaffected. + **and** its sha256 matches. Unlisted or sha256 mismatch → discard and rebuild, + with a warning. Local build-node and CVMFS artifacts are unaffected. Maximal reuse is the default: with no `--trust-manifest`, a build trusts - every signed manifest present in the store (content hashes do not include the - architecture, so a hash vouched for by any trusted manifest counts, whichever - platform or community certified it); signature and expiry are the only checks. Scope it - with `--trust-groups`, or turn verification off with `--no-require-signed-reuse`. + every signed manifest present in the store, under `system: trust_manifest_prefix` + (default `MANIFESTS/common-manifest`; several prefixes, comma-separated, are + tried in order, and for each architecture the first that has a manifest is + used). An entry names an architecture and a hash, as the store keys its + objects; an object two verified manifests certify with different sha256 is + trusted by neither. Scope it with `--trust-groups`, or turn verification off + with `--no-require-signed-reuse`. + Every build fetches the manifests afresh (an unchanged one is revalidated with + its ETag, not downloaded again) and verifies and parses the same bytes; a + manifest that cannot be fetched trusts nothing. Besides signature and expiry, + a numbered manifest (`sequence`, as the orchestrator writes them) is checked + against the highest number this node has verified for its issuer and + architecture (`~/.bits/trust-sequence.json`, or `$BITS_TRUST_SEQUENCE_STATE`): + an older one put back is refused. - `--trust-groups G1,G2,…` — scope reuse by group. The signed common manifest may tag each entry with a `group`; with `--trust-groups` a consumer trusts only those groups plus the always-trusted `common` base (untagged entries count as @@ -252,8 +300,25 @@ Three commands, one per step: The manifests CI then checks the MR author and the approval, and signs (`bits sign`). If the approval is refused or times out, no MR is opened. +Once bits-orchestrator owns the store's certified set, it puts +`MANIFESTS/orchestrator.json` in the store and bits changes accordingly: +`bits certify` stops after step 2 (the orchestrator certifies the build from its +pre-approval, as soon as the build is in the store, so no token, manifests repo +or MR is needed); `bits sign` refuses the store, as it does one whose marker +cannot be read; `bits store rm --stale-boms --manifests-dir` and `bits compliance +--enforce` request decertify actions from bits-console (`--console`, your +GitLab token) instead of removing BOM files or re-signing with `--key`; and +`bits store rm` never deletes the orchestrator's signed manifests, nor its marker +unless named exactly with `--raw` and `--force`. `--enforce` asks for each restricted package to be taken out of the +BOMs of every group you administer (`decertify-package`: each BOM is replaced by +itself without it, so the builds' other packages stay certified). Going back to the manifests CI +removes the marker, once the worker no longer owns the set (it would write it +again): `bits store rm --raw MANIFESTS/orchestrator.json --force`. + `--group`, `--manifests-remote`, `--ref` and `--console` default from the active -defaults' `system:` block, so a configured community just runs `bits certify`: +defaults' `system:` block (`--console` after `$BITS_CONSOLE_URL`), and `--console` +last from the site configuration's `console.backend_url` (`https://bits.cern.ch` +at CERN), so a configured community just runs `bits certify`: ```yaml system: @@ -479,7 +544,6 @@ but are not enforced by CI. | `bitsModules` | Module listing on CVMFS through the serving catalog (used by `bits` with `BITS_CATALOG_LISTING=1`) | | `aliBuild` | Backward-compatible wrapper: sets the ALICE defaults and execs `bits` | | `pb` | Thin wrapper calling `bitsBuild` | -| `bitsenv` | Legacy environment manager | | `bits_helpers/args.py` | Argument parsing for all sub-commands | | `bits_helpers/build.py` | Core build orchestration (~4 000 lines); `doBuild` | | `bits_helpers/hashing.py` | Build-hash computation (`storeHashes`) and the hash-excluded recipe keys | @@ -501,7 +565,9 @@ but are not enforced by CI. | `bits_helpers/verify.py` | `bits verify` — check a deployment against a build manifest | | `bits_helpers/cvmfs_inspect.py` | `bits cvmfs` group — `platforms`/`show`/`summary`; dispatches `stage`/`publish` | | `bits_helpers/cvmfs_publish.py` | `bits cvmfs publish` — place a build's packages, release view and merged view on CVMFS | -| `bits_helpers/plan.py` | `bits build --dry-run` per-package reuse plan | +| `bits_helpers/modulefile_env.py` | What a release's bits modulefiles set, read without Tcl, for the merged view's `setup.sh` | +| `bits_helpers/upgrade_cmd.py` | `bits upgrade`: new package versions set in the recipes | +| `bits_helpers/plan.py` | The build plan (`bits build`, `--dry-run`): per-package reuse and what is not needed | | `bits_helpers/brew.py` | `bits brew` — macOS Brewfile from the recipes | | `bits_helpers/overlay/lcg.py` | `bits overlay lcg` — LCG release view over a built closure | | `bits_helpers/bits_use.py` | `bits use` — per-directory saved-argument profiles | @@ -946,17 +1012,33 @@ The project is licensed under the terms in `LICENSE.md`. ## 16. Command-Line Reference -All sub-commands are accessed through the unified `bits` entry point: +All commands go through the `bits` entry point. `bits help` (or `bits -h`) lists +them by group; `bits help COMMAND...`, the same as `bits COMMAND... --help`, prints +one command's options (`bits help cvmfs stage`). ``` -bits [--debug|-d] [--dry-run|-n] [options] +bits [--community NAME] COMMAND [OPTIONS] +bits --version ``` -| Global option | Description | -|---------------|-------------| -| `-d`, `--debug` | Enable verbose debug output | -| `-n`, `--dry-run` | Print what would happen without executing. For `bits build`: a per-package plan (installed, local tarball, from the remote store, from the reuse overlay, or build) and a summary. Nothing is downloaded: the remote store is only listed, which is possible for `http(s)://` and `b3://` stores. | -| `-a ARCH`, `-w DIR`, `--no-refresh` | Module commands (`enter`, `load`, `q`, …) only: architecture, work directory, and skip refreshing the modules directory. | +| Option | Applies to | Description | +|--------|------------|-------------| +| `--community NAME` | every command, first | The same as `BITS_COMMUNITY=NAME`. Through the CVMFS entry point it also selects the community's published modules, e.g. `/cvmfs/bits.cern.ch/bits/bin/bits --community key4hep setenv key4hep/2026-10-01 -c ./run.sh` in a grid job. | +| `-h`, `--help` | every command | The commands; after a command, that command's help. | +| `--version` | alone | The bits version. | +| `-d`, `--debug` | all but the module commands, `cvmfs`, `store`, `use`, `overlay`, `preload` | Verbose debug output. | +| `-n`, `--dry-run` | `build`, `clean`, `init`, `publish`, `prune` (`compliance`: `--dry-run`) | Print what would happen without executing. For `bits build`: the build plan a real build prints before it starts: per package where it comes from (installed, local tarball, from the remote store, from the reuse overlay, build, or not needed), its recipe source, and a summary. Nothing is downloaded: the remote store is only listed, which is possible for `http(s)://`, `s3://` and `b3://` stores. A tarball found there that a signed manifest lists is remembered in `/.store-cache/` and not asked for again while the manifest lists it (with signed reuse on, the default); deleting that directory only means asking again. The summary says when there is no remote store, and how many packages to build are in the store but not reused, and why (no signed manifest vouches for them, or the Python package `cryptography`, which checks the signatures, is missing). | +| `-a ARCH`, `-w DIR`, `--no-refresh`, `-q`, `--view`, `--archive`, `--dev`, `--shellrc` | module commands | See [bits enter](#bits-enter) and `bits enter --help`. | + +A command's own options come after it (`bits build -a ARCH PKG`, not `bits -a ARCH build PKG`). + +| Group | Commands | +|-------|----------| +| Using software (the module commands) | [`q`, `query`, `list`, `avail`](#bits-query--list--avail), [`enter`](#bits-enter), [`setenv`](#bits-setenv), [`load`, `printenv`](#bits-load--printenv), [`unload`](#bits-unload), [`modulecmd`](#bits-modulecmd), [`shell-helper`](#bits-shell-helper) | +| Building | [`build`](#bits-build), [`status`](#bits-status), [`deps`](#bits-deps), [`doctor`](#bits-doctor), [`init`](#bits-init), `use` (in [bits init](#bits-init)), [`checksums`](#bits-checksums), [`upgrade`](#bits-upgrade), [`stats`](#bits-stats), [`brew`](#bits-brew), [`clean`](#bits-clean), [`prune`](#bits-prune), [`architecture`, `version`](#bits-version--architecture) | +| CVMFS | [`cvmfs`](#bits-store--bits-cvmfs-admin--ci-groups), [`publish`](#bits-publish), [`cvmfs-path`](#bits-cvmfs-path), [`import`](#bits-import), [`overlay`](#bits-overlay-lcg), `preload` (`bits preload --help`) | +| Trust and compliance | [`certify`, `sign`](#bits-certify--bits-sign), [`verify`](#bits-verify), [`sbom`](#bits-sbom), [`compliance`](#bits-compliance) | +| Binary store | [`store`](#bits-store--bits-cvmfs-admin--ci-groups) | Saved arguments from a `bits use` profile are inserted right after the sub-command, before your own (see [bits init](#bits-init)); `bits.rc` is no longer read. @@ -973,10 +1055,12 @@ bits build [options] PACKAGE [PACKAGE ...] | Option | Description | |--------|-------------| | `--defaults PROFILE` | Defaults profile(s); use `::` to combine (e.g. `release::myproject`). Default: `release`. | -| `--flavour NAME[=VALUE]` (aliases `--flavor`, `--set`) | Set a build-wide flavour variable (repeatable, comma-separated). `NAME`→`true`, `NAME=VALUE`→`VALUE`, `!NAME`→`false`. Gates `(?NAME)` conditional requires/sources/patches and is exported into the build environment; overrides a defaults `variables:` entry of the same name. See [Flavours](#flavours). | +| `--flavour NAME[=VALUE]` (aliases `--flavor`, `--set`) | Set a build-wide flavour variable (repeatable, comma-separated). `NAME`→`true`, `NAME=VALUE`→`VALUE`, `!NAME`→`false`. Gates `(?NAME)` conditional requires/sources/patches and is exported into the build environment (not for a variable declared `export: false`); overrides a defaults `variables:` entry of the same name. See [Flavours](#flavours). | | `--reuse-from PATH\|cvmfs` | Reuse components already deployed on CVMFS, found through their published modulefiles at this absolute modules-tree path (unlike `--remote-store`, which is a tarball store). The literal `cvmfs` takes the location from the defaults `system:` layout (`module_dir` under `cvmfs_dir`) or, failing that, from `cvmfs_modules_template`. A trailing `::relaxed` or `::strict` also sets the reuse policy (e.g. `cvmfs::relaxed`); it must agree with `--reuse-policy` if both are given. Defaults to `$BITS_REUSE_FROM`. See [Reusing deployed components](#relaxed-cvmfs-reuse). | | `--reuse-policy {strict,relaxed}` | How a `--reuse-from` component is matched. `strict` (default): only on an exact content-hash match; the result is publishable. `relaxed`: any deployed build of the **same version** (revision and hash may differ), for fast local development on top of e.g. an LCG release, so only the top of the stack is built. Relaxed builds have *loose* provenance and cannot be published: combining them with `--write-store` is refused. Falls back to the defaults `reuse_policy:` value. | | `--build-local PKG[,PKG…]` | Packages to always build locally even when they could be reused (e.g. one you need patched), instead of taking them from `--reuse-from`. | +| `--symlink-packages-from ROOT` | Link matching installed packages into the local work directory with private relocated initialization files. ROOT contains `ROOT//[family/]/`; shared packages are found under `ROOT/share`, and a family-free layout is supported. No modulefiles are required. An existing local installation with the same build hash is kept as is. Selection skips development packages, virtual packages and `--build-local` entries. Unmatched packages proceed through normal store/build handling. The source installation must remain accessible. | +| `--symlink-policy {strict,relaxed}` | Independent of `--reuse-policy`. `strict` (default) requires `.build-hash` to match the computed remote package hash and, with signed reuse on, the signed manifest to certify that hash. `relaxed` matches the architecture, family and version path without requiring a matching hash. Explicit `force_revision` values, including an empty revision, are respected. Otherwise the expected directory label is preferred, followed by sorted versions of the same package version. Relaxed builds have loose provenance and cannot use `--write-store`. | | `-a ARCH`, `--architecture ARCH` | Target architecture. Default: auto-detected, or the `architecture:` template from defaults (see [The `architecture:` template](#architecture-string-and-the-architecture-template)). An explicit value here overrides the template. | | `--force-unknown-architecture` | Proceed even if architecture is unrecognised. | | `--day DAY` | Value of the `{day}` nightly CVMFS path slot (e.g. `Fri`). Default: the UTC weekday, filled in only when a template uses `{day}`; `''` collapses the slot. Layout only — never hashed. See [bits store / bits cvmfs](#bits-store--bits-cvmfs-admin--ci-groups). | @@ -991,6 +1075,7 @@ bits build [options] PACKAGE [PACKAGE ...] | `--build-nice-step N` | Nice increment between concurrent build slots when `--build-nice` is set: slot *k* → nice `min(k×N, 19)`. `N=1` gives a gentle `0,1,2,3` ladder; larger values separate the slots more aggressively. Default: 5. | | `--build-nice-boost-after SECONDS` | With `--build-nice`, a watchdog raises the priority of one long-running straggler at a time, so a single heavy compile does not drag out the end of the build. Native builds: the longest-running lowered build is reniced towards 0 (needs root or `CAP_SYS_NICE`; otherwise a logged no-op). `--docker`/podman builds: a compiler process (e.g. `cc1plus`, `f951`) running longer than this is reniced as root inside the container; this **needs `ps` (package `procps`) in the build image**, see the note below. `0` disables. Default: 600. | | `--prefetch-workers N` | Spawn *N* background threads to fetch remote tarballs and source archives ahead of the main build loop, so downloads overlap with compilation instead of blocking the serial preparation pass. Default: `-1` (auto — scales with `--parallel`, capped at 4); `0` disables. No effect without `--remote-store`. | +| `--fetch-build-requires` | Also fetch or build the build-only dependencies (`build_requires`) of packages reused from a store. By default they are **not needed** and skipped: the build plan printed before the build lists them as `not needed (build-only, for reused X)`. If a package planned as reused is built after all (e.g. its tarball is not vouched for by the signed manifest), its skipped dependencies are fetched or built first. | | `--parallel-downloads N` | Maximum concurrent source/tarball downloads the `--parallel` scheduler runs as standalone download tasks (so a checkout overlaps the previous package's build). Default: 2. | | `--auto-resources` | Opt-in for `--parallel` > 1: load the per-package CPU and memory statistics recorded by a previous run, and turn on monitoring to refresh them, so a new build starts only while the machine still has CPU and memory to spare. Off by default (concurrency is then limited only by `--parallel`); explicit `--resources` / `--resource-monitoring` still apply. | | `--brew` | **macOS only.** Let a recipe that sources a system library from Homebrew run `brew install ` on demand (during dependency resolution) when the formula is missing. Without it, such a recipe fails with a message naming the formula to install. Exported to recipe `prefer_system_check` scripts as `BITS_BREW=1`. See [macOS Homebrew system layer](#macos-homebrew-system-layer). | @@ -1010,9 +1095,9 @@ bits build [options] PACKAGE [PACKAGE ...] | `-c DIR`, `--config-dir DIR` | Directory containing recipe files. Default: `.`. | | `--search-path NAMES` | Comma-separated recipe sub-repos to search besides the config dir. A relative NAME resolves to `/NAME.bits`; absolute paths are used as-is. Seeds `BITS_PATH` (an explicit `$BITS_PATH` wins). | | `--reference-sources DIR` | Directory for the local mirrors of git repositories. Default: `/MIRROR`. | -| `--remote-store URL` | Binary store to fetch pre-built tarballs from. Append `::rw` to also upload to it (cannot be combined with `--write-store`). On supported architectures the public CERN store is the default; a `remote_store:` entry in the defaults `system:` block replaces that default. | -| `--write-store URL` | Binary store to upload built tarballs to. Given alone (no `--remote-store`), it is also the read store, except where a default remote store applies (then use `--remote-store URL::rw`). | -| `--no-remote-store` | Disable the default remote store; a `--write-store` is still read from. | +| `--remote-store URL` | Binary store to fetch pre-built tarballs from; several, comma-separated, are tried in order (e.g. production, then the PR store). Append `::rw` to also upload to a single store (cannot be combined with `--write-store`). Without it, the [site configuration](#site-configuration)'s `store.read_by_arch` / `store.read` is the default (at CERN `https://s3.cern.ch/lcgapp-bits-testing`, where the bits CI builds publish, and ALICE's public store for ALICE's architectures; with `--write-store` alone, or under the aliBuild wrapper, only the latter); a `remote_store:` entry in the defaults `system:` block replaces that default. | +| `--write-store URL` | Binary store to upload built tarballs to; several, comma-separated, all receive every built package (an upload failure to any fails the package). Revision numbers are chosen from the markers of every read and write store, so the stores never disagree on one. Given alone (no `--remote-store`), it is also the read store, except where a default remote store applies (then use `--remote-store URL::rw`). | +| `--no-remote-store` | Disable the default remote store; a `--write-store` is still read from. Without it, an http(s):// or b3:// read store that does not answer (an offline laptop) is noticed once at the start, and the build goes on without it, with a warning. Not when a write store is set. | | `--insecure` | Do not validate TLS certificates of an `https://` store. | | `--sign-manifest KEY.pem`, `--trust-manifest URL\|PATH`, `--require-signed-reuse` / `--no-require-signed-reuse`, `--trust-groups G1,G2` | Signed reuse of remote tarballs: on by default (fail-closed), trust derived from the store's signed manifests. See [Signing and verifying the archive tier](#signing-and-verifying-the-archive-tier). | | `--reuse-beacon URL` | Report reused-from-store hashes to `/api/reuse` (best-effort; falls back to `$BITS_REUSE_BEACON`). | @@ -1033,6 +1118,7 @@ bits build [options] PACKAGE [PACKAGE ...] | `--container-use-workdir` | Mount the work directory at the same absolute path inside the container (without this flag it is mounted at `/container/bits/sw`). Ignored when `--cvmfs-prefix` is set. | | `--docker-platform PLATFORM` | Docker `--platform` for cross-compilation (e.g. `linux/arm64`). Inferred automatically from `--architecture`; pass `native` to suppress. Requires QEMU binfmt handlers. See [§22.2 Cross-compilation via QEMU](#222-cross-compilation-via-qemu). | | `--sandbox MODE` | Sandbox recipe builds: `off` (default), `auto`, `podman`, or `sandbox-exec` (macOS). See [§22.1 Recipe Sandbox](#221-recipe-sandbox). | +| `--sandbox-allow-write PATH` | With `sandbox-exec`, also let builds write under PATH (repeatable). | | `--sandbox-image IMAGE` | Container image for `--sandbox=podman` when not using `--docker`. Implies `--sandbox=podman`. | | `--sandbox-network MODE` | Default network restriction inside the sandbox: `on` blocks network access, `off` allows it. A recipe's `sandbox_network:` always wins; falls back to `sandbox_network:` in the active defaults, then `on`. No effect where sandboxing is off. | | `--no-auto-cleanup` | Keep the build directories after a successful build. | @@ -1082,7 +1168,34 @@ S3 credentials, endpoint and region not set by 1–3 are also read from | Addressing style | `--s3-addressing-style` | `S3_ADDRESSING_STYLE` | `BITS_S3_ADDRESSING_STYLE` | A store set as `REMOTE_STORE=b3://mybucket::rw` (or the flag) reads from and -uploads to the same bucket. With no store and no env vars, behaviour is +uploads to the same bucket. + +**Several buckets, each with its own credentials.** Each `b3://` bucket may have +its own endpoint and credentials: `BITS_S3__ACCESS_KEY_ID`, +`BITS_S3__SECRET_ACCESS_KEY`, and optionally `…_SESSION_TOKEN`, +`…_ENDPOINT_URL`, `…_REGION` and `…_ADDRESSING_STYLE`, where `` is the +bucket's name upper-cased with other characters as `_` (`cms-pr` → +`BITS_S3_CMS_PR_…`), or a `[bucket cms-pr]` section in `~/.bits/s3keys` with the +same keys (other sections still count for every bucket, as before). The key, secret and token are taken together from one of these. A bucket +with none of its own uses the settings above. For example, a PR job that reuses +production packages and uploads to the PR bucket: + +```bash +export BITS_S3_CMS_PROD_ACCESS_KEY_ID=… BITS_S3_CMS_PROD_SECRET_ACCESS_KEY=… +export BITS_S3_CMS_PR_ACCESS_KEY_ID=… BITS_S3_CMS_PR_SECRET_ACCESS_KEY=… +bits build --remote-store b3://cms-prod,b3://cms-pr --write-store b3://cms-pr … +``` + +A package recalled from one store is not copied to the write stores; only +packages built in the run are uploaded. If the write stores already hold +different tarballs for a hash, the build manifest records the first store's. +Signed reuse applies to every read store, with the signed manifests of each. +Recipes never see any of these credentials. `bits publish`, `bits store upload` +and `bits gc` take a list too and work on every store in turn, `bits compliance` +audits (and with `--enforce` cleans) each, and `bits certify` publishes to each +but certifies the build once, as the first store holds it (it warns when another +store holds other tarballs, which signed reuse then rejects). `bits store-stats`, +`bits store ls/rm/verify/cat` and `bits cvmfs release` work on one store. With no store and no env vars, behaviour is unchanged: the public CERN read store on supported architectures, no upload. **Client requirement — install on the build host, not in the container.** bits @@ -1224,8 +1337,8 @@ bits doctor --check-store — architecture: slc9_x86-64 | `--defaults PROFILE` | `release` | Defaults profile for dependency resolution. | | `-w DIR`, `--work-dir DIR` | `sw` | Work directory checked for disk space (setup and `--runner` modes). | | `--docker` | off | Run recipe checks inside a Docker container, as `bits build --docker` would. | -| `--remote-store URL` | _(none)_ | Remote binary store URL; checked for reachability in setup and `--runner` mode and probed per-package in `--check-store` mode. | -| `--write-store URL` | _(none)_ | Write store; checked in setup mode like `--remote-store`. | +| `--remote-store URL` | _(none)_ | Remote binary store URL(s), comma-separated; each is checked for reachability (and, for `b3://`, its bucket's credentials) in setup and `--runner` mode, and packages are looked up in all of them in `--check-store` mode. | +| `--write-store URL` | _(none)_ | Write store(s); each checked in setup mode like `--remote-store`. | | `--no-remote-store` | off | Disable the default remote store. | | `--insecure` | off | Skip TLS certificate validation when probing an `https://` store. | | `--s3-endpoint`, `--s3-access-key`, `--s3-secret-key`, `--s3-region`, `--s3-addressing-style` | — | S3 connection for `b3://` stores, as for `bits build`. | @@ -1290,6 +1403,18 @@ Show what `bits build` would do for each package in the dependency tree, without bits status [options] PACKAGE [PACKAGE...] ``` +To inspect the same deployment selection as a symlink build, use matching +architecture, defaults, deployment root and remote store options: + +```bash +bits status -c cms.bits -a el9_amd64_gcc14 -w temp \ + --symlink-packages-from /path/to/sw \ + --remote-store b3://bits --check-store Python +``` + +`--check-store` checks remote dependencies before status chooses their hashes; +this matters when some dependencies are absent from the deployment root. + | Option | Default | Description | |--------|---------|-------------| | `--defaults PROFILE` | `release` | Defaults profile(s); use `::` to combine. | @@ -1297,6 +1422,7 @@ bits status [options] PACKAGE [PACKAGE...] | `-w DIR`, `--work-dir DIR` | `sw` | bits work directory to inspect. | | `-c DIR`, `--config-dir DIR` | `.` | Recipe directory. | | `--search-path NAMES` | _(none)_ | Extra recipe sub-repos, as for `bits build`. | +| `--set NAME=VALUE` | _(none)_ | As for `bits build` (e.g. `release=LCG_110`), so the packages get the build's hashes. | | `-C DIR`, `--chdir DIR` | `.` | Change to DIR first. | | `--reference-sources DIR` | `/MIRROR` | Mirror directory for git ref cache. | | `--no-local PACKAGE` | _(none)_ | Exclude a package from local-checkout detection. May be repeated. | @@ -1305,10 +1431,14 @@ bits status [options] PACKAGE [PACKAGE...] | `--force-rebuild PACKAGE` | _(none)_ | Report the named package as needing a rebuild regardless of its hash. | | `-u`, `--fetch-repos` | on | Clone / fetch reference repos to populate the git ref cache before computing hashes. Requires network access. | | `--no-fetch-repos` | off | Use only the refs already cached (offline). | -| `--remote-store URL` | _(none)_ | Remote binary store URL. Only consulted when `--check-store` is given. | +| `--remote-store URL` | _(none)_ | Remote binary store URL(s), comma-separated, looked up in order. Only consulted when `--check-store` is given. | | `--no-remote-store` | off | Disable any remote store, even a configured default. | | `--check-store` | off | Probe the remote store to detect tarballs not mirrored locally. Adds a network round-trip per uncached package. | +| `--symlink-packages-from ROOT` | off | Report packages that `bits build` would link from `ROOT`. Uses the same architecture, family, version and hash selection. | +| `--symlink-policy {strict,relaxed}` | `strict` | Strict requires the deployed `.build-hash` to match the computed remote hash; relaxed matches the package path. The signed manifest is not checked here. | +| `--build-local PKG[,PKG...]` | _(none)_ | Exclude these packages from symlink selection. | | `--json` | off | Emit a machine-readable JSON report. | +| `--docker`, `--docker-image IMAGE`, `--docker-extra-args ARGS` | off | As for `bits build --docker`: the system packages are checked in the image, and `own_hash` packages (the toolchain) hash its container fingerprint. Use the build's image to get the hashes of a `--docker` build; without it, every package is built. | **Package states:** @@ -1317,6 +1447,7 @@ bits status [options] PACKAGE [PACKAGE...] | `already_installed` | Hash matches the installed package; nothing to do. | | `from_store` | Matching tarball found in the local TARS store; will be unpacked. | | `from_remote_store` | Tarball only in remote store; will be downloaded then unpacked. Detected only with `--check-store`. | +| `symlink_from_deployment` | Matching deployed package will be linked into the work directory. JSON output includes its `source` path. | | `local_checkout` | A directory matching the package name exists in cwd; will be compiled from local sources. | | `local_checkout_unchanged` | Devel package whose content hash has not changed; rebuild would be skipped. | | `build_from_source` | No cached result found; will be compiled from scratch. | @@ -1357,6 +1488,26 @@ bits verify --from-manifest FILE [options] --- +### bits trace + +Who built, published and approved a file on CVMFS. + +```bash +bits trace /cvmfs/bits.cern.ch/atlas/x86_64-el9-gcc15-opt/Packages/ROOT/v6.40.02-2/lib/libCore.so +``` + +From the file (symlinks of a release view resolved) it finds its package, the `.meta.json` above it: package, version, content hash, the build that compiled it, the recipes' commit and the bits version. The console backend's orchestrator then names the pipeline that published the package (from prepub's measurement records, `BITS_ORCH_PREPUB_URLS`) and the pipelines of the build that compiled it, and for each who requested it and who approved it with their passkey, with the approval's event in the orchestrator's audit chain. Pipelines of communities you do not administer are named by id only. The path need not be mounted: without a `.meta.json` only the publish is shown. + +| Option | Default | Description | +|--------|---------|-------------| +| `--backend URL` | `$BITS_BACKEND_URL`, else site `console.backend_url` | Console backend to ask. | +| `--cafile FILE` | system | CA bundle for the backend's certificate. | +| `--json` | off | The answer as JSON. | + +It authenticates with your GitLab token, as `bits certify` does (`~/.bits/gitlab-token` or `$BITS_GITLAB_TOKEN`); without one it shows the package only. + +--- + ### bits sbom Export a build manifest as a Software Bill of Materials: CycloneDX 1.6 JSON @@ -1375,16 +1526,23 @@ bits sbom MANIFEST [--format cyclonedx|spdx|both] [-o DIR|-] [--build-id ID] | `-o DIR`, `--output-dir DIR` | `.` | Where to write the files; `-` prints one format to stdout. | | `--build-id ID` | the manifest's build id | Release name in the SBOM. | -Each built package is a component: version `-`, the tarball -SHA-256, the SPDX licence, a purl (`pkg:github/…@` for GitHub code, -else `pkg:generic/…`), source archives with their checksums, the git origin, -and bits properties (build hash, architecture, patches, redistributable). +Each built package is a component: the upstream version (the bits revision is +the `bits:revision` property), the tarball SHA-256, the SPDX licence, a purl and, +when the recipe gives one, a CPE (see the recipe field `references`), source +archives with their checksums, the git origin, and bits properties (build hash, +architecture, patches, redistributable, and `bits:pinned_by` when a defaults override or another package pinned its version). The purl is the recipe's own, else +`pkg:pypi/@` for a `PythonPipRecipe`, else +`pkg:github/…@` for GitHub code, else `pkg:generic/…`. Dependencies come from the manifest (schema v4 and later): runtime and untracked `requires` become dependency edges (CycloneDX `dependencies`, SPDX `DEPENDS_ON`); `build_requires` become a `bits:build_requires` property in CycloneDX and `BUILD_DEPENDENCY_OF` in SPDX. Packages taken from the system (`system_packages`) appear as components marked -`bits:provided_by = system`. Older (v3) manifests have no dependency edges: +`bits:provided_by = system`. A recipe with `sbom: tool` (hash-excluded) is a host +tool, run by the stack but never shipped with it (perf, git): its component is +marked `bits:sbom = tool`, and bits-console's scans leave it out of the +vulnerability counts and upgrade plans, as they do a system package only +`build_requires` reach. Older (v3) manifests have no dependency edges: their SBOM lists components only. A recipe `license:` that is not a valid SPDX expression is kept verbatim (a licence name in CycloneDX, a declared `LicenseRef-bits-…` in SPDX), so both files stay valid. URLs lose any credentials, and a local (non-URL) source @@ -1392,6 +1550,77 @@ is not exported. The output is deterministic: the same manifest and bits version give byte-identical files. The SBOMs `bits publish` uploads carry the stored tarballs' sha256, as the BOM does. +The console backend serves a published build's SBOMs to any signed-in user +(`GET /ops/builds/{pipeline}/sbom?format=cyclonedx|spdx`); the console draws the +CycloneDX one as a report of its components and dependencies. With +`BITS_ORCH_SCANNER_URL` (the sbom-scanner service of bits-services, grype), the +worker scans every published build's SBOM, and again whenever the vulnerability +database is newer or a newer pipeline publishes the build (a re-run, which writes +its SBOM anew; a re-publish replays the old build manifest, and so the old SBOM); the report then shows the build's vulnerabilities, and +`GET /ops/sbom/scans?community=&flagged=` lists the scanned builds, most severe +first (the console's Vulnerabilities page). A package is checked only through +its purl or CPE (the recipe field `references`, or the distribution's package for +one taken from the system); each scan counts the components a scanner can +identify, and a build with none is not scannable: its zero findings say nothing. +`GET /ops/sbom/history?community=&days=` gives each release's open findings by +severity and their risk over time: a release is a community's top-level package +(its compiler and build-type variants share their sources), recorded once a day +from its newest scanned build (earlier builds at their publish days, with their +counts when recorded). + +A scan also indexes the SBOM, for the admins of the build's community: +`GET /ops/sbom/packages?name=&version=` lists the scanned builds that contain a +package (by its SBOM name or its bits name; `2.9` or `2.9.x` covers 2.9.13, not +2.90), with its findings, and `GET /ops/sbom/vulnerabilities/{id}` the builds a +vulnerability was found in. They triage findings (`POST /ops/sbom/triage`): a +vulnerability is `not_affected` (with a CycloneDX justification), a +`false_positive` or `will_not_fix` (an accepted risk) in their community, for +one version of a package, every version of it or every package, or for every +vulnerability of a package, now and later (`vuln_id` `*`: one taken out of the +analysis, such as a host tool; its licence findings stay), with a reason +and an optional expiry in days; revoked with `POST /ops/sbom/triage/{id}/revoke`. +Both are events. The findings a rule covers are left out of the effective counts +of `/ops/sbom/scans`, and `GET /ops/builds/{pipeline}/sbom?vulnerabilities=1` +serves the SBOM with its findings and the rules as CycloneDX `analysis` (VEX). +So are, with no rule, the findings of components the stack does not ship +(`not_shipped`, counted apart): host tools (`bits:sbom = tool`), what only host +tools use at run time, and system packages outside the runtime closure of what +the release asks for (only `build_requires` reach them); the served SBOM marks them +`not_affected` with `bits:not_shipped`, and the upgrade plans skip them. + +`GET /ops/sbom/plan?community=` plans upgrades from them. For each release line +(a community's release on one platform, from its newest scanned build), each +package with open findings gets a target: the smallest stable version newer than +its own that fixes each of them. The packages fall into waves, each one set of +version bumps and one rebuild: known exploited (CISA KEV), upgrades within the +major version, major upgrades, ordered by the risk they remove (grype's score: +severity, EPSS, KEV) per package rebuilt; packages taken from the system and +findings no version fixes are listed apart. The converged plan then picks one +version per package for all the release lines in scope, the highest any needs or +a higher one already built and scanned clean; a line whose version is pinned +(`bits:pinned_by` in its SBOM: a defaults `overrides:` entry or another +package's `name = version` requires) is an exception. + +`GET /ops/sbom/licences?community=` lists the release lines with their open +licence findings, and `GET /ops/sbom/licences/{build}` analyses a line's newest +build, from the licences its SBOM gives (the recipes' `license:`, as SPDX +expressions). Its findings are licence conflicts along runtime dependency paths +(GPL-2.0-only with GPL-3.0, AGPL or Apache-2.0; GPL-3.0 or AGPL with +GPL-2.0-only), each with its paths, and packages with no licence, a +`LicenseRef-` or a name that is no SPDX id. Build-only +dependencies, standalone build tools listed in `requires:` (CMake, autotools, +...) and packages from the host form no combined work and are not followed. An +admin accepts a finding as a triage rule on `licence:conflict` or +`licence:unknown`. It also lists the GPL and AGPL code each package the release +asks for uses at run time, and the packages whose binaries are not +redistributed. `GET /ops/sbom/dependencies?community=` gives the release lines +and the packages their newest builds have in several versions, and +`GET /ops/sbom/dependencies/{build}` a line's newest build: the packages most +others depend on (how many rebuild when one changes, how many use it at run +time), those needed only to build, those nothing needs, and what changed since +the line's previous build (packages added, removed, in another version, under +another licence). + --- ### bits checksums @@ -1414,7 +1643,7 @@ bits checksums [PACKAGE ...] [-c DIR] [--recipes DIR ...] [--defaults all|P1,P2] | `--write` | off | Record new entries. Without it the command only reports. | | `-w WORKDIR` | `sw` | Holds the download cache (`SOURCES/cache/`), shared with builds. | | `-a ARCH` | detected | Architecture for `$(...)` sources. | -| `--remote-store URL` | upstream only | Store whose source mirror is tried before upstream (read only). | +| `--remote-store URL` | upstream only | Store(s), comma-separated, whose source mirrors are tried in order before upstream (read only). | | `--fresh` | off | Download every source again from upstream into a private cache, ignoring the download cache and the remote store. | | `-j N`, `--jobs N` | 8 | Parallel downloads and git queries. | @@ -1443,6 +1672,73 @@ build would stop there too. Exit status 1 on any `MISMATCH` or `failed`. --- +### bits upgrade + +Set new versions of packages in the recipes, in place, to build and fix them +from there. + +```bash +bits upgrade PACKAGE ... [plan.csv] [--defaults D] [-a ARCH] [-c DIR] [-w WORKDIR] + [--set NAME=VALUE] [--wave N ... | --line LINE] [--to PKG=VER ...] [--no-check] +bits -n upgrade ... # report what it would change, write nothing + +bits upgrade LCG plan.csv --wave 0 --wave 1 -a x86_64-el9-gcc14-dbg +``` + +`PACKAGE`, `--defaults`, `-a`, `-c`, `-w`, `--set` and `--search-path` are the +build's: the packages are resolved as `bits build` resolves them (as `bits +status` does). The new versions come from a plan exported as CSV from the +console's Plan view (Vulnerabilities, made from the SBOM scans of the builds): +the `.csv` file given after the packages (or with `--plan`), by default the +newest `upgrade-plan-*.csv` in the current directory. +From a release line's plan, every wave is taken, each package at its target +version, or only the waves given with `--wave` (repeatable; 0 known exploited, +1 within the major version, 2 major upgrades); from the converged plan, the +release line given with `--line`. `--to` (repeatable) adds or overrides +versions; given alone, no plan is read. + +For each package, the file that sets its version is edited: + +| The version comes from | Edited | +|---|---| +| the recipe | its `version:`, and its `tag:` when the tag spells the version (`v1.2.3`, `curl-8_18_0`); a tag built from `%(version)s` is left alone | +| a defaults profile's `overrides:` | that profile's entry for the package (with a matcher, only the entry active for the architecture) | +| another recipe's `- pkg = version` requires | that line of that recipe | + +Only the YAML header is edited, keeping its layout and comments. A recipe of a +repository provider's cached checkout (`WORKDIR/REPOS/`) is not edited there: +it is copied, with the patches and the checksum file it uses, into the recipe +directory `-c`, which the build searches first; a copy never replaces a file +there. A defaults profile in such a checkout is left for a hand edit: check that +repository out and build with it. Anything else (a tag that does not spell +the version, an override of another version, a package not in the build) is +reported for a hand edit, and the exit status is 1. Unless `--no-check`, the +new tarball URLs are tried (HTTP HEAD), a git source's new tag is looked up (`git +ls-remote`) and a `PythonPipRecipe` version on PyPI: a missing one is reported, +since the build would stop there, and the exit status is 1. + +When none of a recipe's new tarballs exists, the version is taken from upstream +instead: from the recipe's own `source:`, else the GitHub projects PyPI names for +its `pkg:pypi` references, else a GitHub search by the package's name (reported +as a guess to check; `$GITHUB_TOKEN` raises GitHub's rate limits). The first of +these repositories with the version wins. Its release tarball comes first: a file +of the GitHub release of the version's tag, or GNOME's download site for +`gitlab.gnome.org`; it replaces the recipe's one `sources:` entry, patches kept. +Otherwise its git tag (the recipe's `tag:` spelling, else `1.2.3`, `v1.2.3`, +`name-1.2.3`, `name_1_2_3` and similar): the recipe gets `source:` and `tag:` +(as `v%(version)s` when it spells the version), and its `sources:` and +`patches:` are commented out, under a line saying why. A git checkout has no generated +`configure` (release tarballs ship it), so an autotools recipe also gets +`autoreconf -fi` (when `configure` is missing) first in its `Configure()`, or +in a `Configure()` added for `AutoToolsRecipe`'s default; a recipe that runs a +configure otherwise is reported to check. So is a repository found by the +search, and the exit status is then 1. + +Then build, fix what is left (recipe bodies, patches), and commit the edited and +copied files to a branch of the repositories they belong to. + +--- + ### bits stats Summarise the resource data recorded when a build ran with `--resource-monitoring` @@ -1503,7 +1799,7 @@ bits compliance --enforce [--dry-run] [--key PEM] # admin | `PACKAGE ...` | _(none)_ | Group mode: audit exactly the resolved dependency closure of these roots (e.g. the group's meta-packages), discovering recipe repositories as `bits build` does (config dir, defaults profile, repository providers). | | `-c DIR`, `--search-path NAMES`, `-a ARCH`, `--defaults PROFILE`, `--disable PACKAGE` | as `bits build` | Group mode: how the closure is resolved. | | `--recipes DIR` | `.` | Without `PACKAGE`: recipe repository to audit (a directory of `*.sh` recipes). | -| `--remote-store URL` | `https://s3.cern.ch/lcgapp-bits-testing` | S3 store to audit against the recipe flags (`https`, `b3://`, `s3://`). `$BITS_S3_STORE` changes the default. `--store` is the deprecated spelling. | +| `--remote-store URL` | the site's `store.operations` (CERN: `lcgapp-bits-testing`) | S3 store to audit against the recipe flags (`https`, `b3://`, `s3://`). `$BITS_S3_STORE` changes the default. `--store` is the deprecated spelling. | | `--no-store-check` | off | Audit the recipes only. | | `--enforce` | off | ADMIN: purge non-compliant packages from the store (see below). | | `--dry-run` | off | With `--enforce`: print every action, touch nothing. | @@ -1580,7 +1876,7 @@ bits init --dry-run --remote-store https://store.example.com/store | `-c DIR`, `--config-dir DIR` | `[build]` | Default recipe directory. | | `-w DIR`, `--work-dir DIR` | `[build]` | Default work/output directory. | | `--reference-sources DIR` | `[build]` | Default mirror directory. | -| `--community NAME` | env only | No build-time flag; set `$BITS_COMMUNITY` (the `aliBuild` wrapper sets it). `--organisation` is its former name. | +| `--community NAME` | env only | Not saved; set `$BITS_COMMUNITY` or use `bits --community NAME build …` (the `aliBuild` wrapper sets it). `--organisation` is its former name. | | `--providers URL` | env only | No build-time flag; set `$BITS_PROVIDERS`. | > **`bits.rc` has been retired.** Earlier versions read a `bits.rc` / `.bitsrc` / `~/.bitsrc` file; it is no longer read. Per-directory settings now live in a `bits use` profile (above); global settings come from environment variables: `$BITS_WORK_DIR`, `$BITS_REPO_DIR` (config dir), `$BITS_COMMUNITY`, `$BITS_PROVIDERS`, `$BITS_PATH` (recipe search path — see `--search-path`), `$BITS_S3_STORE`, `$BITS_PREREQUISITES_URL`, `$BITS_CVMFS_REPOS`. Display prefix and branding (`$BITS_PKG_PREFIX`, `$BITS_BRANDING`) are set by the `aliBuild` wrapper. @@ -1647,9 +1943,18 @@ bits enter [-q] [--shellrc] [--dev] [--view] MODULE1[,MODULE2,...] | `-q` | Silence the module-load messages. | | `--view` | Collapse `PATH`, `LD_LIBRARY_PATH`, `CMAKE_PREFIX_PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` and `ROOT_INCLUDE_PATH` onto a merged view of the loaded closure (cached under `$WORK_DIR/VIEWS/`), so the environment stays small on big stacks. `_ROOT` and other `setenv`s are untouched. Turned on automatically when a loaded package's recipe sets `view: true`. | +**A release with a complete view.** When the request is one CVMFS module `/` +of a release whose view is complete (its modules directory has `..view`, see +[the merged view](#bits-store--bits-cvmfs-admin--ci-groups)) and the view is to be used +(`--view`; for `enter` and `setenv` also `view: true`), `enter`, `setenv` and +`load`/`printenv` source the view's `setup.sh` (`setup.csh` for csh) instead of loading +the modules: no `modulecmd`, no Python, one file read. `bits list` then shows no modules, and `unload` does not undo it. +A local build of the module, several modules, a bare `` or `--dev` take the modules +path as before. + The shell type is auto-detected from the parent process (`bash`, `zsh`, `ksh`, `csh`/`tcsh`, `sh`). Override with the `MODULES_SHELL` environment variable. The prompt is set to `[MODULE_LIST] \w $>` (or the zsh/ksh equivalent) for the duration of the session. Nesting `bits enter` inside another bits environment is blocked. -All module commands (`enter`, `load`, `unload`, `setenv`, `q`, `avail`, `modulecmd`) also accept `-w DIR` (default: `$BITS_WORK_DIR`, else `./sw` or `../sw`), `-a ARCH` (default: the installed architecture, preferring the host's when several are present) and `--no-refresh` (do not refresh the modules directory). +All module commands (`enter`, `load`, `unload`, `setenv`, `q`, `avail`, `modulecmd`) also accept `-w DIR` (default: `$BITS_WORK_DIR`, else `./sw` or `../sw`; with none of them, `q`, `enter`, `load`, `printenv`, `unload` and `setenv` use the community's CVMFS modules alone, e.g. in a grid job; `--dev` needs a work dir), `-a ARCH` (default: the installed architecture, preferring the host's CPU and OS when several are present; with none installed, the one the community's CVMFS module trees publish for this host, if exactly one) and `--no-refresh` (do not refresh the modules directory). --- @@ -1715,16 +2020,21 @@ with the same result. **A community's modules on CVMFS.** With `BITS_CVMFS_PREFIX` set to a community's CVMFS prefix (the bits entry point on CVMFS sets it), `bits q`, `enter`, `load`, `printenv`, `unload` and `setenv` also use the modules the community published there, -after the local ones. bits uses the trees `//Modules/modulefiles` whose -`` is `ARCHITECTURE` but for a trailing `-opt`/`-dbg`, in this order: the exact -one, the one without build type (the toolchain), `-opt`, `-dbg`. So -`-a x86_64-el9-gcc14-opt` uses `x86_64-el9-gcc14-opt`, `x86_64-el9-gcc14` and -`x86_64-el9-gcc14-dbg`, never another compiler or OS. A module comes from the first +after the local ones. bits uses the trees `//Modules/modulefiles` of +`ARCHITECTURE` and of `ARCHITECTURE` without its `-opt`/`-dbg` (the toolchain, and +communities published without a build type); an `ARCHITECTURE` without one is `-opt`. +So `-a x86_64-el9-gcc14-opt` uses `x86_64-el9-gcc14-opt` and `x86_64-el9-gcc14`, never +the `-dbg` tree, another compiler or OS; `bits q` notes when the other build type is +published. A module comes from the first tree that has it, local builds first; `q` lists each name once. Each tree's modules are loaded against that tree alone, with its own `BASEDIR`, so modules from several trees can be loaded together; `unload` uses the tree each module was loaded from. The CVMFS trees are not put on `MODULEPATH`: inside `bits enter`, a `module load` by hand -sees only the local modules. `--dev` works with local modules only. +sees only the local modules. `--dev` works with local modules only. A tree's retired +modulefiles, in `Modules/archive` beside `Modules/modulefiles` (ALICE's convention, +which `alienv --archive` searches), are found only with `--archive` (`-archive`; not +`-a`, the architecture): after the tree's own, against the same packages. `unload` +always looks there too, for a module loaded from it. Without `BITS_CVMFS_PREFIX` (unset; set but empty turns CVMFS modules off), a recipe repository's [`cvmfs.yaml`](#the-cvmfs-layout-file-cvmfsyaml) gives the trees: in the @@ -1733,7 +2043,13 @@ current directory, or beside the work directory (`sw/`). They are the directory `` order above. Without `-a` and with no tree for the local architecture, `` is the one published for this host's CPU and OS (`el` or `ubuntu`, as the CVMFS entry point picks its view) when there is exactly one compiler for it; otherwise bits uses none and says which there are, so -`bits -a q` picks one. +`bits -a q` picks one. A template with `{install_dir}` instead of `{arch}`, as +alice.bits' `{prefix}/{install_dir}/Modules/modulefiles/{pkg}`, has a tree per install +dir (`el9-x86_64`, as on `/cvmfs/alice.cern.ch`): bits uses the one published with the +architecture's CPU and OS (`slc9_x86-64` → `el9-x86_64`), or, without a work +directory and without `-a`, the host's. `-a` can also name the install dir itself; +that is the only way to use one without an OS (`x86_64-2.6-gnu-4.8.3`) or one of +several for the same CPU and OS. --- @@ -1861,7 +2177,7 @@ lists the available formats. ```bash bits overlay lcg -a ARCH --platform PLATFORM --version-number N \ - [--postfix P] [--out DIR] [--build-view DIR] [-w DIR] + [--postfix P] [--out DIR] [--build-view DIR] [-w DIR] [--closure META] ``` | Option | Description | @@ -1872,6 +2188,7 @@ bits overlay lcg -a ARCH --platform PLATFORM --version-number N \ | `--version-number N` | LCG version number of the release directory (e.g. `110`). Required. | | `--postfix P` | LCG version postfix (e.g. `_ATLAS_5`). Default: none. | | `--out DIR` | `LCG_RELEASE_BASE` root to write `LCG_/` under (default: current directory). | +| `--closure META` | List only the packages of the dependency closure in META, a `.meta.json` (the view package passes its own), at their versions; without it every package of the arch subtree is listed. | | `--build-view DIR` | Also build the merged symlink view of the closure in DIR, with a `setup.sh` that collapses `PATH`/`LD_LIBRARY_PATH`/… onto it (no lcgcmake `create_lcg_view` needed). | | `--lib-path-var VAR` | Loader path variable for that `setup.sh` (default `LD_LIBRARY_PATH`; use `DYLD_LIBRARY_PATH` on macOS). | @@ -1986,9 +2303,8 @@ cvmfs_views_template: "{prefix}/views/{release}/{arch}" ``` The shell reads `prefix` and `cvmfs_modules_template` as plain `key: value` lines; a -modules template that is not `/{pkg}` (e.g. one with -`{install_dir}`), or a file without `prefix`, gives `bits q`, `enter` and `load` no -CVMFS modules. +modules template that is not `/{pkg}`, +or a file without `prefix`, gives `bits q`, `enter` and `load` no CVMFS modules. `bits cvmfs publish` places every package of the build with the build's own CVMFS templates (recorded in its manifest). A group that sets `cvmfs_packages_template` @@ -1997,7 +2313,8 @@ path: one already published by the same build hash is skipped, one published by different build is refused. Its `cvmfs_releases_template` is then the **release view** (the release links), created only with `--release-view` (bits-console: *Create release view*) or by `bits cvmfs release`, and only after every package -published: one relative symlink per package, sent +published (never for a build whose top-level packages all live outside its arch, +such as the toolchain alone): one relative symlink per package, sent via prepub's ingest path so it merges into a release directory other platforms already use (without ingest, only a new release directory works). Such a publish also adds, once per arch, a `BASE/1.0` modulefile in the modules directory that @@ -2006,7 +2323,7 @@ modulefiles resolve against ($BASEDIR//). Packages published elsewhere (noarch packages, a toolchain with [`own_hash: true`](#shared-toolchains-own_hash)) get a relative symlink there. With the ALICE-style templates `{prefix}/{arch}/Packages/…` and `{prefix}/{arch}/Modules/modulefiles/{pkg}`, -`BITS_MODULEDIR= BITS_PLATFORM= bitsenv …` then works unchanged. +`BITS_CVMFS_PREFIX= bits q`, `enter`, `setenv` … then use them. A package's modulefile is published inside it (`etc/modulefiles/`); after the packages, one job adds a relative symlink `//` to it for each package that has none yet, instead of a commit per modulefile. Like @@ -2051,7 +2368,17 @@ group can have views without `cvmfs_releases_template`; each is made when its template is set. A subdirectory only one package fills, with nothing of it excluded, is linked whole (e.g. `include/boost`); the top-level directories, `lib*/pkgconfig` and `lib*/python*/site-packages` stay real directories. The view also has -`setup.sh` (locates itself under bash/zsh) and `setup.csh`. +`setup.sh` (locates itself under bash/zsh) and `setup.csh`. They also set what +loading the release's modules sets: each package's `_ROOT` and the +recipes' other `setenv`s (pointing at the packages, not the view), and path +entries outside the view. The modulefiles are read without Tcl, in the forms bits +writes; the view's `.meta.json` records under `setup` whether that covers all of +them (`modules`) and, if not, what was not understood (`problems`). When it covers them and +the release has one top-level module, the view also has +`.setup-complete..`, and that module a link to it beside its +modulefile, `//..view` (a file, so modulecmd passes it +over). A view later made for another revision has no such file for this one: the +link then leads nowhere. `system.cvmfs_view_exclude: [pkg, …]` leaves packages out. The file lists come from `.bits-view.json`, which every build writes into the package just before packing (so later changes to the local tree don't leak in); for a tarball built before that, from the tarball itself. A recipe shapes its own part of the @@ -2241,6 +2568,8 @@ rebuilds anything; see [Licence compliance and redistribution policy](#licence-c |-------|-------------| | `license` | SPDX licence identifier (`LicenseRef-*` for custom licences; `NOASSERTION` for system shims). Recorded in the build manifest, the publish BOM and the signed manifest; feeds the per-package and per-release `NOTICE` files and the `bits compliance` audit. | | `acknowledgment` | Attribution text required by the licence; written into the per-package `NOTICE`. | +| `references` | Names of the package for vulnerability scanners: CPEs (`cpe:2.3:a::::…`) and purls (`pkg:…`), with `%(version)s`, `%(name)s` and `%(tag)s` filled in. A leading `v` is dropped from a CPE's version. Written into the SBOMs: the first CPE and the purl on the CycloneDX component, further CPEs as `syft:cpe23` properties (as syft writes them, which grype reads), every CPE in SPDX. Hash-excluded. A `PythonPipRecipe` without a purl gets `pkg:pypi/…` automatically. | +| `fetch_cache` | `true` for a recipe whose build downloads sources itself (Bazel externals, OpenLoops process lists, ATLAS externals). bits fills `$BITS_FETCH_CACHE` from the store before the build and saves what the build left there after a successful build, so a later build of the same version keeps working when upstream changes or disappears. Only honoured when the defaults name the cache under `system: fetch_cache: `. The recipe points its tool at the directory, e.g. `bazel build ${BITS_FETCH_CACHE:+--repository_cache=$BITS_FETCH_CACHE}`. Each file is stored once by sha256 (under `SOURCES/cache/`, shared by all versions). The list of files of a version on an architecture is written once, by the first build that saves it; a later build restores exactly that list and warns about files it downloaded that are not in it. Not part of the package hash; nothing is saved without a write store or when `redistributable` forbids sources. File names with a newline or a backslash are not saved. | | `redistributable` | Which forms may be redistributed: `all` (default), `binaries`, `sources`, `none`. Restricted binaries are never uploaded to the store nor published to CVMFS; restricted sources are never mirrored to `SOURCES/cache/`. Legacy `true`/`false` = `all`/`none`; unknown values fail closed as `none`. | | `description`, `url`, `homepage`, `source_url` | Free-text metadata, also hash-excluded. | @@ -2378,8 +2707,9 @@ Grammar (repeatable, comma-separated): `NAME` → `true`, `NAME=VALUE` → `VALU `!NAME` → `false`. A value is *truthy* unless it is empty, `0`, `false`, `off`, or `no`. Each flavour is merged into the defaults' `variables:` map (so `(?NAME)` sees it) **and** the `env:` map (so it is exported as `$NAME` in every recipe's -build and contributes to the package hash). A `--flavour` overrides a defaults -`variables:`/`env:` entry of the same name. +build and contributes to the package hash), unless the variable is declared +`export: false` (below). A `--flavour` overrides a defaults `variables:`/`env:` +entry of the same name. `NAME` must be a plain identifier — `[A-Za-z_][A-Za-z0-9_]*` (a letter or underscore, then letters/digits/underscores). **Hyphens are not allowed:** use @@ -2420,6 +2750,21 @@ variables: A truthy value is anything except empty, `0`, `false`, `off`, or `no`. A `--flavour` of the same name overrides a defaults `variables:` value. +A `--flavour`/`--set` normally also goes into the build environment, and so into +every package's hash. For a variable declared `export: false` it sets only the +variable, which then enters only the hashes of the recipes that use it: + +```yaml +variables: + athena_release: + value: main + export: false # --set athena_release=release/25.0.75 rebuilds only what uses it +``` + +Defaults variables expand in a recipe's `tag:` and body (not in `version:`, which can +use `%(tag_basename)s`). A later profile in the chain that sets the same name to a +plain value replaces the whole entry, `export: false` included. + ##### `variables` vs `env` vs `flavours` — quick comparison These three are easy to confuse. `variables:` is **text templating only** (Python @@ -2432,8 +2777,8 @@ the name **verbatim** — none of them upper-cases it. | Defined in | defaults / recipe `variables:` | defaults `env:` | CLI (repeatable) | | Surface in recipe | `%(NAME)s` (text) | `$NAME` (shell) | both `%(NAME)s` **and** `$NAME` | | Gates `(?NAME)` in requires/patches/overrides | defaults `variables:` only | no | yes | -| Exported into build shell | no | yes (via `defaults-release`) | yes | -| In the package hash | only through the text they expand into: recipe `variables:` in `version`/`tag`/`source`/`sources`/`patches`/body; defaults `variables:` in `tag` and the body | yes (via the `defaults-release` environment) | yes (both paths) | +| Exported into build shell | no | yes (via `defaults-release`) | yes, unless the variable is `export: false` | +| In the package hash | only through the text they expand into: recipe `variables:` in `version`/`tag`/`source`/`sources`/`patches`/body; defaults `variables:` in `tag` and the body | yes (via the `defaults-release` environment) | yes (both paths; for an `export: false` variable only through the text) | | When evaluated | build-time text substitution, **before** hashing | exported into the shell before the recipe body runs | both | | Name case | verbatim | verbatim | verbatim | @@ -2483,6 +2828,7 @@ Only one version pin per dependency is allowed across the whole graph; conflicti |-------|-------------| | `prefer_system` | Architecture regex. When it matches (or with `--always-prefer-system`), bits runs `prefer_system_check`. | | `prefer_system_check` | Bash snippet (sees `$REQUESTED_VERSION`). Exit 0 to use the system package instead of building it; non-zero to build it. | +| `system_package` | Name (or list of names) of the distribution package that provides a package taken from the system, for the SBOM; default: the recipe's name, lower-case, `-libs` and `lib` spellings. bits asks rpm and dpkg (once per build, for all such packages) for the installed package and records it in the SBOM under the distribution's name and version, with a `pkg:rpm/...` or `pkg:deb/...` purl, which vulnerability scanners match against the distribution's advisories. If none is installed, the version comes from the check's output: a `bits_system_version: X` line, else the first line that is only a version (as `xml2-config --version` prints); the recipe's `references:` are then filled with it. Also applies to a passing `system_requirement_check`. | | `system_requirement` | Architecture regex. When it matches, bits runs `system_requirement_check`; a non-zero exit aborts the build with a missing-requirement error. Such a recipe must have an empty body. | | `system_requirement_check` | Bash snippet that checks for the required system package. | | `system_requirement_missing` | Message (e.g. install instructions) printed by `bits doctor` when `system_requirement_check` fails. | @@ -2675,6 +3021,8 @@ A **defaults profile** is a special recipe file named `defaults-.sh` that The active profile is selected with `--defaults PROFILE`. If the flag is omitted, bits uses `release`, loading `defaults-release.sh`. Any other selection is layered on top of `release`: `--defaults dev` behaves like `release::dev` (a missing `defaults-release.sh` is skipped). +A chain that qualifies the architecture (a compiler profile's `append_arch`) but names no build type gets `opt` when the repository has `defaults-opt.sh`: `--defaults key4hep::gcc15` builds as `key4hep::gcc15::opt`, so the same binaries always land under one architecture (`…-gcc15-opt`) and match its certified tarballs. The implied `opt` is placed before any trailing overlay and is not held against a recipe's `valid_defaults`. + `defaults-release.sh` occupies a privileged position: every package in the build graph automatically depends on a pseudo-package named `defaults-release`, which is fulfilled by whatever profile(s) are loaded. This is the mechanism that injects the global `env:` block into every package's `init.sh`. @@ -2764,14 +3112,14 @@ package_family: | `qualify_arch` | Set to `true` to append **all** non-`release` default names to the install architecture string; see [Qualifying the install architecture](#qualifying-the-install-architecture) below. | | `append_arch` | String appended **verbatim** to the install architecture string, **only for this defaults file** (include the separator, e.g. `-gcc13`). Unlike `qualify_arch`, which qualifies with every default name in the chain, each file opts in independently and chooses the exact string; see [Selective qualification with append_arch](#selective-qualification-with-append_arch) below. | | `own_hash_neutral` | Set to `true` next to an `append_arch` (typically a build type such as `opt`/`dbg`) to leave that qualifier out of the architecture of `own_hash` packages, so one toolchain build serves every build type. See [Shared toolchains](#shared-toolchains-own_hash). | -| `docker_registry` | (in `defaults-release`) Registry the default `--docker` builder image is taken from; `$BITS_DOCKER_REGISTRY` wins. See [§22](#22-docker-support). | +| `docker_registry`, `docker_tag` | (in `defaults-release`, under `system:`; a top-level `docker_registry:` is still read) Registry and tag of the default `--docker` builder image; `$BITS_DOCKER_REGISTRY` / `$BITS_DOCKER_TAG` win, the [site configuration](#site-configuration) comes after. See [§22](#22-docker-support). | | `checksum_mode` | Base checksum verification policy for every build using this profile. Accepted values: `off` (default), `warn`, `enforce`, `print`. Equivalent to passing the corresponding `--*-checksums` flag on every invocation. CLI flags override this setting; see [Checksum policy in defaults profiles](#checksum-policy-in-defaults-profiles) below. | | `write_checksums` | Set to `true` to automatically write/update `checksums/.checksum` files after every build. Equivalent to passing `--write-checksums` on every invocation. The CLI flag overrides this setting. | | `variables` | Values for `(?NAME)` matchers and `%(NAME)s` substitution; see [Defaults `variables:` and predefined platform variables](#defaults-variables-and-predefined-platform-variables). | | `expand_recipe` | `true` expands every recipe body strictly, as a recipe's own `variables:` or `expand_recipe: true` does: an unknown `%(name)s` in it is an error. | | `auto_patch` | `false` turns off automatic patch application for every package; see [Controlling patch application](#controlling-patch-application). | | `force_revision`, `revision_policy` | Revision labels for installed packages; see [Forcing or dropping the revision suffix](#forcing-or-dropping-the-revision-suffix-force_revision). | -| `system` | Build-host and publishing settings, e.g. `remote_store`, `sandbox_network`, `mem_per_job_default`, `build_oversubscribe`, `build_unleash_final`, `source_mode` and the CVMFS path templates (see [CVMFS layout](#cvmfs-layout)). | +| `system` | Build-host and publishing settings, e.g. `remote_store`, `fetch_cache` (the name of the community's cache of downloaded sources, see the recipe field `fetch_cache`), `sandbox_network`, `mem_per_job_default`, `build_oversubscribe`, `build_unleash_final`, `source_mode` and the CVMFS path templates (see [CVMFS layout](#cvmfs-layout)). | --- @@ -2934,7 +3282,8 @@ Packages that declare `architecture: share` (see [§19](#19-architecture-indepen The module frontend (`bits enter`/`q`/`load`) auto-detects only the **raw** architecture, so when a build was qualified you must point it at the combined string. After a successful qualified build the success banner prints the exact -command, e.g. `bits -a slc7_x86-64-dev-gcc13 enter MyPackage/latest-…`, and +command, e.g. `bits -a slc7_x86-64-dev-gcc13 enter MyPackage/1.2.3-1` (the version +just built; `MyPackage/latest-` enters the family's newest), and suggests `export BITS_ARCHITECTURE=slc7_x86-64-dev-gcc13` to make it the default for the session. As a convenience, when `-a` is not given and the detected raw architecture has no install tree under the work dir, the frontend uses the sole @@ -3177,6 +3526,42 @@ The content-addressed store path (`TARS//store/

//`) is unaffecte --- +### Site configuration + +The defaults and assumptions an installation may change (stores, container images, the console, the providers repository, the SBOM namespace and a few paths) are in YAML files named `NN-.yaml`; CERN's values, formerly built into bits, are in its `10-CERN.yaml`. The files are read from these directories in order: + +1. `bits_helpers/site.d/` in bits: `00-bits.yaml` (site-neutral) and `10-CERN.yaml` (CERN); +2. `/etc/bits/site.d/` (a host, a runner); +3. `~/.config/bits/site.d/` (`$XDG_CONFIG_HOME`); +4. `$BITS_SITE_DIR`. + +A file in a later directory replaces the one of the same name, so an empty `10-CERN.yaml` in `/etc/bits/site.d` switches off CERN's. All files are then merged in name order, the last winning: mappings key by key, other values replaced whole, `null` removing a value. Suggested ranges: `00`–`09` bits, `10`–`49` organisation, `50`–`89` host or runner, `90`–`99` personal. The command line, the environment and the community's `system:` defaults come before all of them. `bits doctor` lists the files in effect and where the main values come from; a file that does not parse stops bits. + +| Key | Use | +|-----|-----| +| `store.read` | Read store of `bits build`/`doctor` without `--remote-store` (after `system: remote_store`). | +| `store.read_by_arch` | Architecture → read store, before `store.read` (matched by distribution and CPU). | +| `store.operations` | Store of `certify`, `compliance`, `gc`, `store-stats`, `publish` and `bits store` (after `$BITS_S3_STORE`). | +| `store.s3_endpoint` | Endpoint of `b3://` stores (after `--s3-endpoint` and the environment); unset: boto3's default. | +| `containers.registry`, `containers.tag`, `containers.legacy_image` | The default `--docker` image; see [§22](#22-docker-support). | +| `providers.url` | The always-on providers repository (after `$BITS_PROVIDERS`). | +| `sbom.namespace` | SPDX document namespace base. | +| `console.backend_url` | The console backend: what `bits trace` asks (after `--backend`, `$BITS_BACKEND_URL`), and the console `bits certify` asks for approvals (after `--console`, `$BITS_CONSOLE_URL`, `system: console_url`) and `bits compliance --enforce` for decertifications (after `--console`, `$BITS_CONSOLE_URL`). | +| `console.time_zone` | The bits instance's time zone, for every community it serves: the time of its nightly schedules, and the times the console and the app show (`UTC`; at CERN `Europe/Zurich`). | +| `paths.work_dir`, `paths.gitlab_token_file`, `paths.s3_keys_file` | Defaults after `$BITS_WORK_DIR`, the token and S3 key file variables. | + +Outside CERN, mask CERN's file and give your own, e.g. an empty `/etc/bits/site.d/10-CERN.yaml` and a `/etc/bits/site.d/10-mylab.yaml` with your stores and console. The order in which a value is looked for, from the command line down to these files, is in [USERGUIDE — Where defaults come from](USERGUIDE.md#where-defaults-come-from). + +```yaml +# /etc/bits/site.d/50-runner.yaml: this runner reads and writes its own bucket +store: + read: b3://my-bucket + operations: b3://my-bucket + s3_endpoint: https://minio.example.org +``` + +--- + ## 19. Architecture-Independent (Shared) Packages Some packages — calibration databases, reference data files, pure-Python libraries, architecture-neutral scripts — produce identical output regardless of the build platform. Rebuilding them on every architecture wastes time and storage. The `architecture: share` recipe field tells bits to install such packages into a single, platform-neutral directory tree that all architectures can read. @@ -3297,6 +3682,7 @@ bits sets these variables in each package's build script before the recipe body | `$WORK_DIR` | Absolute path of the work directory (`sw/` by default). | | `$PKGDIR` | Directory containing the package's recipe file (its `patches/` live here). | | `$BITS_CONFIG_DIR` | Absolute path of the recipe directory (`-c`/`--config-dir`). | +| `$BITS_FETCH_CACHE` | With `fetch_cache: true`: `FETCH_CACHE////` in the work directory, restored from the store before the build; empty otherwise. | #### Package identity @@ -3369,23 +3755,24 @@ For each built dependency `DEP`, bits also sets `${DEP_ROOT}` to its absolute in | Variable | Default | Purpose | |----------|---------|---------| | `BITS_BRANDING` | _(empty)_ | Cosmetic program-name branding; set by the `aliBuild` wrapper. | -| `BITS_COMMUNITY` | _(empty)_ | Community whose registry entry (`.bits.sh`) the bootstrap clones when `-c` names a missing recipe directory. Empty by default; the `aliBuild` wrapper sets `ALICE`. (`bits init --community` does not persist it; set the variable.) `BITS_ORGANISATION`, its former name, still works. | +| `BITS_COMMUNITY` | _(empty)_ | Community whose registry entry (`.bits.sh`) the bootstrap clones when `-c` names a missing recipe directory. Empty by default; the `aliBuild` wrapper sets `ALICE`. (`bits init --community` does not persist it; set the variable, or give `bits --community NAME` before the sub-command.) `BITS_ORGANISATION`, its former name, still works. | | `BITS_PKG_PREFIX` | _(empty)_ | Display prefix for `bits q`. Empty prints native `PKG/VERSION`; when set (e.g. `VO_ALICE` via `aliBuild`) output becomes `PREFIX@PKG::VERSION`. | | `BITS_REPO_DIR` | `.` (`alidist` under the `aliBuild` wrapper) | Recipe directory (`-c`/`--config-dir` default): normally the community repository you run bits in. | | `BITS_WORK_DIR` | `sw` | Output and work directory (`ALICE_WORK_DIR` is accepted as a fallback). | | `BITS_PATH` | _(empty)_ | Comma-separated list of additional recipe search directories. Absolute paths are used directly; a relative name resolves to `/.bits`. `--search-path` seeds it; an explicit `BITS_PATH` wins. | | `BITS_CHDIR` | _(unset)_ | Directory to change to before building (same as `-C`). | | `BITS_TAR_COMPRESSOR` | `gzip -n` | Compressor for package tarballs. Must be deterministic (e.g. `pigz -n -p4` on a farm with a uniform pigz, never plain `pigz`). | -| `BITS_DOCKER_REGISTRY` | `gitlab-registry.cern.ch/bits/containers` | Registry of the default `--docker` builder image (wins over `docker_registry:` in `defaults-release`). | -| `BITS_DOCKER_TAG` | `latest` | Tag of the default builder image. | -| `BITS_LEGACY_REGISTRY` | _(unset)_ | `1` selects the legacy `alisw/-builder` images (the `aliBuild` wrapper sets it). | +| `BITS_DOCKER_REGISTRY` | site `containers.registry` | Registry of the default `--docker` builder image (wins over `system: docker_registry` in `defaults-release`). | +| `BITS_DOCKER_TAG` | site `containers.tag` (`latest`) | Tag of the default builder image. | +| `BITS_LEGACY_REGISTRY` | _(unset)_ | `1` selects the site's `containers.legacy_image` (the `aliBuild` wrapper sets it). | +| `BITS_SITE_DIR` | _(unset)_ | One more directory of [site configuration](#site-configuration) files, read last. | | `BITS_LEGACY_INITDOTSH` | _(unset)_ | `1` selects the legacy build-time `init.sh` (same as `--legacy-initdotsh`); the `aliBuild` wrapper sets it. | | `BITS_PROVIDERS` | `https://github.com/bitsorg/bits-providers` (empty under the `aliBuild` wrapper) | URL of the repository-provider set; an `@` suffix pins a snapshot. Environment only (no build flag). | | `BITS_CVMFS_PREFIX` | _(unset)_ | A community's CVMFS prefix (e.g. `/cvmfs/bits.cern.ch/key4hep`): `bits q`, `enter`, `load`, `printenv`, `unload` and `setenv` also use its modules for the architecture, after the local ones (see [bits query](#bits-query--list--avail)). The bits entry point on CVMFS sets it for a community. Unset, a recipe repository's `cvmfs.yaml` gives the trees; empty turns them off. | | `BITS_CATALOG_LISTING` | _(unset)_ | `1`: list trees on CVMFS (module and install trees) from their serving catalog (the `bitsModules` helper) before walking the directory (see [bits query](#bits-query--list--avail)). | | `BITS_REUSE_FROM` | _(unset)_ | Default of `--reuse-from` for `bits build` (e.g. `cvmfs`); the bits entry point on CVMFS sets it for a community. Any `--reuse-from`, from the command line or a `bits use` profile, wins, and an empty one turns reuse off. When the recipes declare no CVMFS layout, or the modules tree it names does not exist, the default is skipped with a warning instead of stopping the build; an explicit `--reuse-policy` wins over its `::relaxed`/`::strict` suffix. | | `REMOTE_STORE`, `WRITE_STORE` | _(unset)_ | Read and write store URLs when no flag is given; `BITS_REMOTE_STORE`/`BITS_WRITE_STORE` override them (see [§21](#21-remote-binary-store-backends)). | -| `BITS_S3_STORE` | `https://s3.cern.ch/lcgapp-bits-testing` | Default store for `bits publish`, `certify`, `sign`, `bits store` and `compliance`. | +| `BITS_S3_STORE` | the site's `store.operations` | Default store for `bits publish`, `certify`, `sign`, `bits store` and `compliance`. | | `BITS_AWS_KEYS_FILE` | `~/.bits/s3keys` | Private file with S3 credentials, used when they are not in the environment. | | `BITS_STRICT_STORE_INTEGRITY` | _(unset)_ | `1` makes a recalled tarball with no integrity-ledger entry fatal (with `--store-integrity`). | @@ -3426,7 +3813,7 @@ A **remote binary store** is an external storage location where bits uploads com | `--insecure` | Skip TLS certificate verification for `https://` stores. | | `--trust-manifest SRC`, `--no-require-signed-reuse` | Signed reuse is on by default: a remote tarball is reused only if a verified signed manifest lists it. `--trust-manifest` names the manifest (otherwise derived from an `http(s)`/`s3`/`b3` store); `--no-require-signed-reuse` disables the check (insecure, warns). See [Signing and verifying the archive tier](#signing-and-verifying-the-archive-tier). | -A `--write-store` given on its own is also used as the read store, unless the architecture has a default read store (`https://s3.cern.ch/swift/v1/alibuild-repo` on slc7/slc8/slc9 and ubuntu2004/2204/2404 x86-64 and on slc9_aarch64, unless `--always-prefer-system` is given): that one stays the read store, so use `--remote-store URL::rw` to read from and write to one store. Unlike aliBuild, configuring a store does **not** force `--no-system` (reuse is content-hash addressed); pass `--no-system` for a self-contained build. Store URLs may also come from the environment (`REMOTE_STORE`/`WRITE_STORE`, overridden by `BITS_`-prefixed names; see [S3 store: common CI/CD config](#s3-store-common-cicd-config-with-per-runner-overrides)) or from `remote_store:` under the defaults `system:` block. CERN S3 path-style URLs (`https://s3.cern.ch/`) are accepted and rewritten to the listable form. +A `--write-store` given on its own is also used as the read store (the site configuration's `store.read` is then not used), unless the architecture has its own read store in the [site configuration](#site-configuration) (`store.read_by_arch`: at CERN `https://s3.cern.ch/swift/v1/alibuild-repo` on slc7/slc8/slc9 and ubuntu2004/2204/2404 x86-64 and on slc9_aarch64), unless `--always-prefer-system` is given: that one stays the read store, so use `--remote-store URL::rw` to read from and write to one store. Unlike aliBuild, configuring a store does **not** force `--no-system` (reuse is content-hash addressed); pass `--no-system` for a self-contained build. Store URLs may also come from the environment (`REMOTE_STORE`/`WRITE_STORE`, overridden by `BITS_`-prefixed names; see [S3 store: common CI/CD config](#s3-store-common-cicd-config-with-per-runner-overrides)) or from `remote_store:` under the defaults `system:` block. CERN S3 path-style URLs (`https://s3.cern.ch/`) are accepted and rewritten to the listable form. ### Supported backends @@ -3507,7 +3894,7 @@ The `b3://` store holds only the content-addressed tarballs, plus a small revisi Building with `--write-store` uploads each package **as it is built**. To push a store you already built (nothing re-uploads on a cached rebuild), use `bits publish` — it reads the build manifest and uploads each package's content -tarball. +tarball, checking 8 packages at a time (`BITS_PUBLISH_JOBS`). ```bash # credentials from the environment or ~/.bits/s3keys (see above) @@ -3537,7 +3924,8 @@ without contacting S3 — handy to check the package set and target before pushi endpoint and path-style addressing are derived), or `b3://` / `s3://`. It is the canonical store flag across `publish`, `certify`, `sign`, `prune --retain`, `bits store` (`gc`/`stats`/`upload`) and `compliance`; the old `--store` spelling still works but is deprecated. -The default is `$BITS_S3_STORE` if set, else `https://s3.cern.ch/lcgapp-bits-testing`. +The default is `$BITS_S3_STORE` if set, else the site's `store.operations` (at CERN +`https://s3.cern.ch/lcgapp-bits-testing`, i.e. `b3://lcgapp-bits-testing`). A bulk publish also uploads a trimmed copy of the manifest (one per architecture) under `MANIFESTS/`, but only if every package uploaded; `bits certify` and `bits sign` turn it into the signed manifest that consumers trust. Packages whose @@ -3819,10 +4207,67 @@ bits build --reuse-from cvmfs::relaxed \ --build-local xrootd xrootd ``` -Reused components are logged as **Reuse: … (not built)**; only the requested top -package is **Compiling**. Use `--build-local PKG[,PKG…]` to force specific +Reused components are logged as **Reusing CVMFS overlay module … (not built)**; +only the requested top package is **Compiling**. Use `--build-local PKG[,PKG…]` to force specific packages to build locally anyway (e.g. one you need patched). +Use `--symlink-packages-from ROOT` to stage each selected dependency at +`////`. Payload +entries such as `bin`, `lib`, and `include` link to the deployed package. +bits copies `etc/profile.d`, which holds `init.sh`; copied regular files are +made writable, and symlinks inside copied directories remain symlinks. +Anything else a package needs as its own copy is the recipe repository's +business: a `SYMLINK_STAGE` hook (`hook: {SYMLINK_STAGE: name[,name]}` in the +recipe or the defaults, the script in `hooks/` as for the other hooks) +runs with `bash -e` in the staged package, at its final place, before its +reference paths are replaced. It gets `PKG_ROOT` (the stage), `SOURCE_ROOT` +(the deployed package), `REFERENCE_ROOT`, `WORK_DIR`, `ARCHITECTURE`, +`PKGNAME` and `PKGVERSION`. Entries in the stage are links into the +deployment: a hook replaces a link with a copy before it changes it. For CMS, +cms.bits' hook copies `etc/scram.d`, `tools` (with `tools/selected`), and for +SCRAM projects `.SCRAM` and `config`, then runs +`config/SCRAM/projectAreaRename.pl`, as `REFERENCE_PACKAGE_POST` does. The +hook is not part of the package hash: it shapes no build. A stage is made once, +so a changed hook applies to new stages (remove a stage to make it again); one +left unfinished by an interrupted hook is made again. `init.sh` retains bits' local architecture setup. Builds +source this local copy, so environment-derived dependency paths are inside the +work directory and can be rewritten by the consumer's normal relocation step. +The deployment is never modified. Existing local installations are not +overwritten; repeated staging of the same source and initialization is accepted. + +```bash +bits build -c cms.bits -a el9_amd64_gcc14 -w temp \ + --symlink-packages-from /path/to/sw Python +``` + +This requires access to the source installation while using the links. A user +moving a consumer elsewhere must provide its compatible dependencies in the +destination layout. The option does not rewrite config files in linked packages +(`.pc`, `.la`, `*-config`). Instead, after POST_INSTALL it rewrites reference paths +in the newly built consumer's regular text files, like CMS `relocateReference`. +Exact dependency roots retain their local family and effective architecture; +reference work-directory prefixes recorded by the reused packages are also +mapped to the local work directory. This runs before `.bits-relocate` is collected +and before packing, so later relocation can rewrite the resulting local paths. +Relocation uses CMS's `find`/`grep`/`perl` pattern: it substitutes each +literal prefix globally, without restricting matches to path boundaries or +special config extensions. It skips symlinks and binary files. Rewriting a leaked `BUILD/...` path does not +provide the missing build directory; such references still need a recipe fix. +`--reuse-from` continues to select packages through deployed modulefiles and +source their initialization files directly; it does not create local package links. + +**Which to use.** `--reuse-from` builds on a release published with modulefiles +(an LCG or Key4hep stack on CVMFS): its packages are used in place, and what is +built on them refers to `/cvmfs/...` and is meant for the same repository. +`--symlink-packages-from` takes a package tree without modulefiles (a CMS `sw` +area, another work directory): each package it finds gets a local stage, so the +packages built on it refer to the work directory and are packed and relocated +like any other. A package neither option finds comes from the store or is built. +With signed reuse on (the default), strict symlinking takes only packages whose +hash the signed manifest certifies, as for store tarballs. The build manifest +records a symlinked package as `already_installed`, with `symlinked_from` and +`symlink_policy`. + **Policy.** `--reuse-policy strict` (default) reuses a component only on an exact content-hash match, so the build stays reproducible and publishable. `--reuse-policy relaxed` reuses any version present in the one-release overlay @@ -3836,6 +4281,16 @@ policy inline; an explicit `--reuse-policy` must agree with it. > binaries carry the toolchain they were built with, so run them in an image that > provides a compatible runtime. +**Results depend on the reused paths.** Reused components are used in place, so a +package built on top of them references their `/cvmfs/...` paths (its `init.sh` +sources theirs, and its build may record them in RPATHs and config files) and +needs that repository mounted at run time. Relocation moves only the package's +own prefix. This is what publishing to the same CVMFS repository needs. For a +self-contained, relocatable work directory, or to publish to a different +repository or prefix, build without `--reuse-from`: the dependencies then come +from the remote store (the same artifacts, where the store has them) or are +built, and are installed and relocated with the rest of the tree. + > **Note.** Earlier versions reused deployed packages through a `cvmfs://` > `--remote-store` and a `--reuse-base ` graft; that path has been > removed in favour of `--reuse-from`. A `cvmfs://` `--remote-store` now errors @@ -3858,7 +4313,7 @@ bits build --docker --docker-image alisw/slc9-builder:latest ROOT bits build --docker --docker-extra-args "--memory=8g --cpus=4" ROOT ``` -Without `--docker-image`, the image is derived from the architecture as `/-[-cuda]:`, for example `gitlab-registry.cern.ch/bits/containers/x86_64-slc9:latest` for `slc9_x86-64`. Only the OS, the CPU and a `cuda` part of the architecture select the image; compiler and build-type parts do not. The registry is `$BITS_DOCKER_REGISTRY`, else `docker_registry:` in `defaults-release` (top level or under `system:`), else `gitlab-registry.cern.ch/bits/containers`. The tag is `$BITS_DOCKER_TAG`, else `latest`. With `BITS_LEGACY_REGISTRY=1` (set by default by the `aliBuild` wrapper), or when the architecture cannot be split into OS and CPU, the legacy `registry.cern.ch/alisw/-builder` image is used instead. Images are pulled when missing. +Without `--docker-image`, the image is derived from the architecture as `/-[-cuda]:`, for example `gitlab-registry.cern.ch/bits/containers/x86_64-slc9:latest` for `slc9_x86-64`. Only the OS, the CPU and a `cuda` part of the architecture select the image; compiler and build-type parts do not. The registry is `$BITS_DOCKER_REGISTRY`, else `system: docker_registry` in `defaults-release` (a top-level `docker_registry:` is still read), else the [site configuration](#site-configuration)'s `containers.registry` (at CERN `gitlab-registry.cern.ch/bits/containers`). The tag is `$BITS_DOCKER_TAG`, else `system: docker_tag`, else `containers.tag` (`latest`). With `BITS_LEGACY_REGISTRY=1` (set by default by the `aliBuild` wrapper), or without a registry or when the architecture cannot be split into OS and CPU, the site's `containers.legacy_image` is used instead (at CERN `registry.cern.ch/alisw/-builder`); without one, `--docker-image` must be given. Images are pulled when missing. Bits mounts into the container the work directory (read-write, except `SOURCES/`, which is read-only so a recipe cannot modify shared sources in place; `BITS_READONLY_SOURCES=0` turns this off), the recipe directory (read-only), the bits installation, and any `-v` volumes. `~/.ssh` is **not** mounted. When components deployed on CVMFS are reused (`--reuse-from`), `/cvmfs` is also mounted read-only. @@ -3907,7 +4362,7 @@ Bits can run each recipe build script inside an isolated sandbox to limit the da #### How it works -Sandboxing is **off by default** (`--sandbox=off`); opt in with `--sandbox=auto` (or a specific mode), or persist it with `bits use build --sandbox auto`. With `auto`: +Sandboxing is **off by default** (`--sandbox=off`); opt in with `--sandbox=auto` (or a specific mode), or persist it with `bits use build --sandbox auto`. A macOS build without `--docker` also runs in `sandbox-exec` when `BITS_SANDBOX=1` is set, or when the active defaults say `sandbox: on` under `system:` (where a sandbox cannot start, with a warning, it does not); `BITS_SANDBOX=0` turns that off, and `--sandbox` overrides both. The network is then blocked as with any sandbox, unless `sandbox_network: off` (below). With `auto`: | Platform | With `--sandbox=auto` | Mechanism | |----------|-----------------|-----------| @@ -3927,7 +4382,7 @@ Pass `--sandbox MODE` to `bits build`: |------|-----------| | `auto` | Pick the best available option: `sandbox-exec` on macOS, nested podman when `--docker` is active and the builder image contains `podman`, and `off` on a local Linux build (no `--docker`). On local Linux, podman is neither used nor probed — request it explicitly with `--sandbox=podman` if you want it. | | `podman` | Always use podman; fails with an error if it is not available. With `--docker`, `podman` must exist in the builder image. Without `--docker`, `podman info` must succeed on the host and `--sandbox-image` must name the image (otherwise sandboxing is disabled with a warning). | -| `sandbox-exec` | macOS only. Fails with an error on Linux. | +| `sandbox-exec` | macOS only. Fails with an error on Linux, and where a sandbox cannot start (inside another seatbelt sandbox, which cannot nest; `auto` is then `off`, with a warning). | | `off` | (default) No sandboxing. Recipe runs directly on the host. | ```bash @@ -3948,6 +4403,10 @@ Giving `--sandbox-image` alone selects `--sandbox=podman`. When `--docker` is us bits build --docker --sandbox=auto --docker-image alisw/slc9-builder:latest ROOT ``` +#### What a build may write (macOS) + +Under `sandbox-exec` a build may write only its own files in the work dir: its `BUILD/`, `INSTALLROOT/`, `TMP/`, its store tarball and tarball links under `TARS/`, its installation (`/[/]/-`, with its `latest` links), and its source directory under `SOURCES/` (a development package's checkout too); besides those, the temp directories, `$CCACHE_DIR`, `~/Library/Caches`, and the module files a `post-relocate.sh` writes to `$MODULES_STAGING` or the work dir's `MODULES/`. Other caches in the home directory (`~/.cargo`, `~/.m2`, `~/.npm`, `~/.cache`, …) are refused: `--sandbox-allow-write` them, or point the tool at a cache under the temp directory. Another package's installation, tarballs or sources are out of its reach. `--sandbox-allow-write PATH` (repeatable) allows more. The profile is kept as `sandbox.sb` beside the package's `build.sh`, and replays its build with `sandbox-exec -f sandbox.sb /bin/bash -e -x build.sh`; a failed build says so, and `grep 'Operation not permitted'` in its log shows what was refused. + #### Per-recipe network control By default the sandbox blocks all outgoing network access from the recipe script. Some recipes need to reach the internet during their build (for example, to run `pip install` or `gem install`). Use the `sandbox_network` recipe field to opt in: @@ -4264,6 +4723,11 @@ bits build ROOT ls -la $WORK_DIR/MANIFESTS/bits-manifest-latest.json ``` +At the end, the build prints a reuse summary from the manifest: how many +packages were already installed, taken from the store and built, and the built +ones by name with the reason (`rebuilt_because`). With build-host metrics on, +the counts are pushed there too. + ### What is recorded The manifest records every input and output that could affect reproducibility: @@ -4307,6 +4771,9 @@ The manifest records every input and output that could affect reproducibility: | `source_checksums` | List of `{url, checksum}` entries from the recipe's `sources:` list; `checksum` is `null` when none was declared | | `built_by` | `user@host` that compiled this hash; `null` unless `outcome` is `"built_from_source"` (recalled artifacts carry their builder in another build's manifest) | | `completed_at` | ISO-8601 UTC timestamp of package completion | +| `rebuilt_because` | Why a package built here was not reused (e.g. `store tarball not certified`); only when known | +| `references`, `pinned_by`, `sbom` | The recipe's CPEs and purls for scanners, the profile or package that pinned its version, and `tool` for a host tool; only when set | +| `symlinked_from`, `symlink_policy` | The deployed package an `already_installed` package was linked from (`--symlink-packages-from`), and the policy; only then | | `pkg_family` | Install family sub-directory (empty if none) | | `effective_architecture` | `share` for noarch packages, the neutral arch for `own_hash` packages, else the build arch | | `patches`, `variables` | Recipe patches with their checksums, and the resolved recipe variables (schema v3) | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b6f35b9f..e9375131 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -196,6 +196,33 @@ O(10⁴) users, plus any project deploying software via CVMFS. `--remote-store` (`--store` deprecated), `--parallel` (`--builders` alias); `--makeflow` / `--pipeline` and usage analytics removed. +### Delivered since 0.6 (2026-10-04) + +- **bits-orchestrator** (console-backend): builds reconciled from GitLab and the + store, failures deduplicated into issues, a hash-chained audit log with an + outside copy, a build queue across build hosts, and certification by the + worker itself, which now owns the certified set and writes the signed, + numbered manifests (bits-manifests is its mirror). +- **Supply-chain analysis**: SBOMs carry CPEs and purls scanners can match and + the versions of system packages; every published build is scanned, findings + are triaged by community admins, and upgrade plans, licence and dependency + analyses come from the scans. `bits upgrade` applies a plan to the recipes. +- **Build plan before every build**: build-only dependencies of reused packages + are neither fetched nor built, the store is listed in parallel and certified + listings are remembered; a reuse summary says why each package was built. +- **Stores**: several read and write stores, each bucket with its own + credentials; offline builds; `fetch_cache` for a recipe's own downloads. +- **Trust**: manifests fetched afresh (ETag), entries per architecture and hash, + anti-rollback by manifest sequence. +- **Environment commands without Python or a work directory**, from CVMFS + (`bits --community NAME setenv …`), replacing `bitsenv`; complete release + views sourced through their `setup.sh`. +- **`--symlink-packages-from`**: packages of a deployed tree without + modulefiles linked into the work directory (CMS), with a recipe-repository + hook for what needs a private copy. +- **macOS build sandbox**, site configuration in `site.d/NN-.yaml`, and + `bits trace` for who built, published and approved a file on CVMFS. + --- ### Near term — within 6 months diff --git a/docs/USERGUIDE.md b/docs/USERGUIDE.md index 97be68ae..dbb2efde 100644 --- a/docs/USERGUIDE.md +++ b/docs/USERGUIDE.md @@ -178,6 +178,12 @@ For another community the steps are the same, e.g. Record per-directory options (work directory, stores) once with `bits use build …` inside the repository (see [Configuration](#4-configuration)). +### Getting help + +`bits help` lists the commands by group, and `bits help COMMAND` (or +`bits COMMAND --help`) one command's options; `bits help enter` covers all the +module commands (`q`, `enter`, `setenv`, `load`, …). + ### ALICE: the aliBuild workflow The `aliBuild` wrapper keeps ALICE's classic workflow, working from an `alidist` checkout: @@ -217,7 +223,7 @@ Global settings come from environment variables: | Variable | Related flag | Description | |----------|--------------|-------------| -| `$BITS_COMMUNITY` | — | Community name (uppercase), e.g. `LHCB` (`$BITS_ORGANISATION`, its former name, still works). Used only when `-c`/`--config-dir` names a directory that does not exist: bits then clones that community's recipe repository from the registry and uses it. The `aliBuild` wrapper sets `ALICE`. | +| `$BITS_COMMUNITY` | — | Community name (uppercase), e.g. `LHCB` (`$BITS_ORGANISATION`, its former name, still works). Used only when `-c`/`--config-dir` names a directory that does not exist: bits then clones that community's recipe repository from the registry and uses it. The `aliBuild` wrapper sets `ALICE`; `bits --community NAME ` sets it too. | | `$BITS_WORK_DIR` | `-w` / `--work-dir` | Output directory for built packages (default: `sw`). | | `$BITS_REPO_DIR` | `-c` / `--config-dir` | Root directory for recipe repositories. | | `$BITS_PROVIDERS` | — | URL of the bits-providers registry, optionally `@tag` (default `https://github.com/bitsorg/bits-providers`; off under the `aliBuild` wrapper). | @@ -226,7 +232,36 @@ Global settings come from environment variables: `$BITS_COMMUNITY` is set **uppercase** (`ALICE`, `LHCB`, …). Bits lowercases it internally when resolving the community recipe repository from bits-providers (e.g. `LHCB` → `lhcb.bits.sh` → `https://github.com/bitsorg/lhcb.bits`). Normally you do not need it: check out the community repository and run bits inside it. -Settings follow the precedence `CLI flag > bits use profile > environment variable > built-in default`; the exception is `$BITS_PATH`, which, when set, overrides `--search-path`. For the full list of environment variables, see [REFERENCE.md §20](REFERENCE.md#20-environment-variables) and [REFERENCE.md — bits init](REFERENCE.md#bits-init). +### Where defaults come from + +A setting is taken from the first of these that gives it: + +1. the command line; +2. the `bits use` profile (inserted before your own options, so those still win); +3. the environment (`$BITS_WORK_DIR`, `$BITS_S3_STORE`, `$BITS_CONSOLE_URL`, …); +4. the community's defaults: the `system:` block of the recipe repository's `defaults-*.sh` (its `remote_store`, `certify_group`, `console_url`, …), which is publish policy and part of no package hash; +5. the **site configuration**: `NN-.yaml` files describing an installation (its stores, container images, console, providers repository, SBOM namespace and a few paths); +6. bits' own site-neutral `00-bits.yaml`. + +The exception is `$BITS_PATH`, which, when set, overrides `--search-path`. + +The CERN stores, images and console URL, formerly built into bits, are in the site file `10-CERN.yaml`, shipped with bits next to `00-bits.yaml`. Files are read from bits' own `site.d/`, then `/etc/bits/site.d/` (a host or runner), `~/.config/bits/site.d/` and `$BITS_SITE_DIR`; a file replaces an earlier one of the same name, then all of them are merged by file name (not directory), the last winning. So: + +- **At CERN** there is nothing to do. +- **Elsewhere**, an empty `10-CERN.yaml` in `/etc/bits/site.d/` switches CERN's values off, and a file of your own sets yours: + + ```yaml + # /etc/bits/site.d/10-mylab.yaml + store: + read: b3://mylab-bits + operations: b3://mylab-bits + s3_endpoint: https://s3.mylab.example.org + console: + backend_url: https://bits.mylab.example.org + ``` +- **A runner or a person** can override single keys in a later file (`50-…` for a host, `90-…` for yourself). + +`bits doctor` lists the files in effect and the store, container and providers values they give. The keys are in [REFERENCE.md — Site configuration](REFERENCE.md#site-configuration); for every environment variable, see [REFERENCE.md §20](REFERENCE.md#20-environment-variables) and [REFERENCE.md — bits init](REFERENCE.md#bits-init). --- @@ -258,6 +293,7 @@ Bits resolves the full transitive dependency graph of each requested package, co | `--build-nice` | Stagger concurrent builders across OS priority levels so CPU contention degrades gracefully — one build runs at full speed, the others are backed off, and the freed top slot is taken over as builds finish. Native builds use `nice`; `--docker` builds use `docker run --cpu-shares`. Opt-in; only affects `--parallel > 1`. Memory is still capped separately. | | `--build-nice-step N` | Priority spread between concurrent build slots for `--build-nice` (slot *k* → nice `min(k×N, 19)`). `N=1` is a gentle ladder; larger separates slots more. Default: 5. | | `--prefetch-workers N` | Background threads that fetch remote tarballs and sources ahead of the build loop. Default: auto (`min(builders, 4)`); `0` disables. | +| `--fetch-build-requires` | Also fetch or build the build-only dependencies of reused packages (by default they are not needed and skipped). | | `-u`, `--fetch-repos` | Update all source mirrors before building (the default), so a branch builds its current commit. | | `--no-fetch-repos` | Build what the mirrors already have (offline). | | `-w DIR`, `--work-dir DIR` | Work/output directory. Default: `sw`. | @@ -288,7 +324,7 @@ Without `--parallel` packages are built one after another, each with the full `- bits build --dry-run ROOT ``` -Computes every package's hash exactly as the build would and prints, in build order, where each one would come from — **installed**, **local tarball**, **from remote store**, **from reuse overlay** — or that it would be **built**, followed by a summary. Nothing is downloaded or installed; http(s):// and b3:// stores are only listed, and other stores (s3://, rsync) are not checked. `bits status --check-store` uses the same store listing. On macOS a dry run also writes `sw//Brewfile`. +Computes every package's hash exactly as the build would and prints, in build order, where each one would come from — **installed**, **local tarball**, **from store**, **from overlay** — or that it would be **built** (**build (unsigned)**: in the store, but no verified signed manifest vouches for it; **build (development)**: a local checkout), followed by a summary. The recipe repository most packages come from is named once, above the table; a package from another shows its own. The store is checked for all packages at once, 16 requests at a time, with a progress line. A real build prints the same plan before it starts. A package that only reused packages need to build (their `build_requires`) is **not needed**: it is neither fetched nor built (`--fetch-build-requires` fetches it anyway). Nothing is downloaded or installed; http(s)://, s3:// and b3:// stores are only listed, and rsync stores are not checked. `bits status --check-store` uses the same store listing. On macOS a dry run also writes `sw//Brewfile`. ### Sharing binaries through a store @@ -470,7 +506,7 @@ Pass the same `--set` / `--day` values as the build so `{release}` and `{day}` r bits certify # latest build: upload what is missing, approve with a passkey, open the MR ``` -`bits certify` is what a user runs: it uploads anything still missing from the store, asks bits-console for a passkey approval (scan the QR code, compare the code, approve on your phone) and then opens the certification merge request in the manifests repository. `bits sign` — the command formerly called `certify` — is run by the manifests-repo CI to merge the approved build manifests into the signed common manifest. Group, manifests repo and console URL default from the defaults' `system:` block. See [REFERENCE.md — Publishing and certifying a build](REFERENCE.md#publishing-and-certifying-a-build--bits-publish-bits-certify) and [Signing — bits sign](REFERENCE.md#signing--bits-sign). +`bits certify` is what a user runs: it uploads anything still missing from the store, asks bits-console for a passkey approval (scan the QR code, compare the code, approve on your phone) and then opens the certification merge request in the manifests repository. Where bits-orchestrator owns the store's certified set (`MANIFESTS/orchestrator.json` in the store), it stops after the approval: the orchestrator certifies the build as soon as it is in the store, and `bits sign` refuses that store. `bits sign` — the command formerly called `certify` — is run by the manifests-repo CI to merge the approved build manifests into the signed common manifest. Group, manifests repo and console URL default from the defaults' `system:` block. See [REFERENCE.md — Publishing and certifying a build](REFERENCE.md#publishing-and-certifying-a-build--bits-publish-bits-certify) and [Signing — bits sign](REFERENCE.md#signing--bits-sign). ### Export an SBOM diff --git a/docs/WORKFLOWS.md b/docs/WORKFLOWS.md index ee02dbf7..556987be 100644 --- a/docs/WORKFLOWS.md +++ b/docs/WORKFLOWS.md @@ -29,6 +29,8 @@ This document describes the complete bits development-to-deployment workflow — [Phase 4 — Publish] Build → Production ───→ /cvmfs/.../releases/ Build → Personal ───→ /cvmfs/.../user/me/ + Nightly slot ───→ /cvmfs/.../nightlies// + git push ──────→ CI build of the slot (store only) [Phase 5 — Verify] bits verify --from-manifest --cvmfs-root /cvmfs/alice.cern.ch @@ -177,11 +179,99 @@ The pipeline enforces this independently of the UI. Even a manually crafted API The **Builds** tab in bits-console shows a live pipeline list with log streaming. The **CVMFS Status** tab updates once ingestion completes and the stratum-0 transaction is published. Production builds become available experiment-wide — to all developers' `bits enter` sessions, to batch grid jobs at WLCG sites, and to downstream CI pipelines. Stratum-1 replicas propagate the change automatically. +### Step 4 — Schedule nightly builds + +A nightly build is a **slot**: a build defined once, run by the orchestrator on +given days at a given time, and approved in advance. Slots are on the console's +**Schedules** tab; everyone signed in sees them, the community's admins make, +change and approve them (a certifying slot: admins of every group it certifies +for). + +| Field | What it sets | +|-------|--------------| +| Package, Version | What is built (a blank version: the recipe's). | +| Defaults chain | How it is built and **where its builds go**. With stacks.bits' `dev3`, builds go to `…/nightlies/dev3/{day}/…`; the orchestrator gives each build its night's weekday as `{day}`, so a build that starts after midnight still lands in its night's directory. Blank: `release`. | +| Variants | Defaults chains put before it, one a line: `gcc15::opt` and `clang::opt` with `dev3` build `gcc15::opt::dev3` and `clang::opt::dev3`. | +| Platforms | Each platform × each variant is one build of the night, a *cell* (64 at most). | +| Variables, Extra recipe repositories, Extra build arguments, Jobs | As in the Build dialog: `--set` (e.g. `release=LCG_110`), more recipe repositories, more `bits build` options (not `--defaults`), `-j`. | +| Days, Time | When it runs, in the instance's time zone (shown beside the time). | +| Publish to CVMFS, Certify, Groups it certifies for | Where the result goes after the build. | +| Build on each commit to | The branches it also builds on, see Step 5. | + +To set one up: + +1. On **Schedules**, click **+ New slot**, fill it in and **Save**. Or set up a + build in the Build dialog (package, platforms, defaults) and click + **💾 Save as pipeline**: a full platforms × defaults matrix (2 to 16 chains) + becomes one slot with the chains as its variants, any other one slot per + chain. It is saved paused, every day at 02:00, without variables or followed + branches: **Edit** it first. +2. If it certifies, click **Approve**, then **Use your passkey**. The approval + covers 30 nights of every cell, but 100 builds at most (fewer nights for a + slot of more than 3 cells), or 30 days, whichever ends first; **▶ Run now** + uses it up too. A night it cannot cover whole, and a slot without one, builds + but certifies nothing. A change of what it builds or where needs a new + approval; a new time, days or followed branches do not. +3. **Resume** a slot saved paused. + +At its time the orchestrator queues a build per cell, and they start as build +hosts become free. A night missed by less than 6 hours still runs; a cell whose +last build still runs skips the night; a build that finds no free host for 12 +hours is not run. **▶ Run now** builds every cell not still running, as +today's night; **Pause**, **Edit** and **✕** (delete) do what they say. + +Each night's results are in the nightly matrix, on the console's **Nightlies** +page (14 nights; a build opens its log) and the phone app's **Nightlies** tab +(7): a row per slot, a cell per platform, variant and night (queued, running, +passed, failed and where, or not run and why). A slot's page in the app has +**Run again** for the failed cells of its last night. + +### Step 5 — Build on each push (CI) + +A slot can also build whenever a branch it follows gets a new commit. In its +**Build on each commit to**, list up to 8 `https://host/path.git@branch` entries, +separated by `;`: the recipe repository at its release branch, and the sources of +packages built from git, e.g. `https://github.com/root-project/root.git@master`. +The list only decides *when* the slot builds: the build takes its recipes and +versions as a nightly does, so following a source branch helps only when the +slot's recipes build that branch (as dev3 builds ROOT `master`). +*Follow its recipe repositories* replaces the list with the community's recipe +repository at the branch the slot's `release` variable names (else `main`), and +those of its extra recipe repositories that name a branch. + +- The orchestrator reads each branch's head every 2 minutes: no webhook and no + token in the repository. Only public repositories can be followed. +- A branch just added is not built; its next commit is. A paused slot follows + nothing, and after **Resume** the heads it finds count as just added. +- Pushes come in bursts, so it waits until the branches have not moved for + 5 minutes (30 minutes at most after the first new commit), then builds every + cell of the slot once for the burst. +- A slot runs one CI build at a time. Commits pushed meanwhile are built + together when it ends, and nothing is cancelled. Nightlies and CI builds do + not wait for each other. +- A CI build goes to the binary store only: it is never certified nor published + to CVMFS. It shows whether the branches build. +- Each build fetches its recipes and sources when it starts, so it may hold + commits newer than the heads recorded when it fired. A commit that changes no package's hash still makes a short + pipeline, which finds everything built. + +The Schedules tab shows each followed branch's head, marked *new: builds soon* +until it is built. CI builds have a matrix of their own, apart from the +nightlies: the console's **CI builds** page shows each such slot's latest 10, a +column per build with the heads when it fired and a row per cell, and the phone +app's **CI** tab the same, its latest 7. + +**For operators:** slots need the orchestrator's build queue +(`BITS_ORCH_QUEUE=1`). The instance's time zone is the site setting +`console.time_zone`; `BITS_ORCH_WATCH_S` (120 s, at least 30) sets how often +heads are read, and the worker must reach the repositories' hosts over HTTPS. +See *Nightly schedules* and *Commit CI* in `console-backend/README.md`. + ### Pipeline internals (for runner admins) The following is relevant to teams setting up or maintaining build runners, not to developers triggering builds through the UI. -Clicking **Build → Production** queues a GitLab CI pipeline that runs `bits build --docker` on a registered runner, with the workDir bind-mounted at the community's `cvmfs_prefix` path inside the container so binaries compile with their final deployment paths already embedded. The resulting tarballs are then ingested into CVMFS via one of two paths depending on the community's `publish_pipeline` setting: +Clicking **Build → Production** queues the build; with the orchestrator's build queue on, it waits there (not in GitLab) for a free build host, and then starts a GitLab CI pipeline pinned to that host. The pipeline runs `bits build --docker` on a registered runner, with the workDir bind-mounted at the community's `cvmfs_prefix` path inside the container so binaries compile with their final deployment paths already embedded. The resulting tarballs are then ingested into CVMFS via one of two paths depending on the community's `publish_pipeline` setting: - **cvmfs-prepub path** (`publish_pipeline: .gitlab/cvmfs-prepub-publish.yml`): the build host POSTs a tar directly to the `cvmfs-prepub` REST API, which handles CAS ingestion and the CVMFS gateway transaction. No dedicated ingest runners are required. - **Legacy spool path** (`publish_pipeline: .gitlab/cvmfs-publish.yml`): tarballs are spooled to a staging area, a CVMFS transaction is opened on the stratum-0, and `cvmfs_server publish` is run on a dedicated publisher runner. This path is retained for communities that have not yet migrated to cvmfs-prepub. @@ -226,6 +316,8 @@ bits build --docker open build modal git push + recipe MR ──────────→ merge triggers ──→ [Build → Production] → /releases/… bits build --write-checksums [Build → Personal] → /user/me/… Monitor: Builds + CVMFS Status tabs + Schedules: a slot ──────────────→ nightlies//… +git push (a followed branch) ──→ the slot's CI build ─→ store only bits verify --from-manifest … ← confirm ✓ ``` @@ -236,6 +328,7 @@ bits verify --from-manifest … ← c | Topic | Location | |-------|----------| +| Nightly slots and CI builds (the orchestrator) | `console-backend/README.md`: *Nightly schedules*, *Commit CI* | | Full command reference (`bits build`, `bits status`, `bits verify`, …) | [REFERENCE.md §16](REFERENCE.md#16-command-line-reference) | | Docker builds and `--cvmfs-prefix` | [REFERENCE.md §22](REFERENCE.md#22-docker-support) | | Remote binary store backends (S3, HTTP, rsync) | [REFERENCE.md §21](REFERENCE.md#21-remote-binary-store-backends) | diff --git a/pyproject.toml b/pyproject.toml index 7f477a9a..63d6d078 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ script-files = ["bits", "bitsBuild", "bitsDoctor", "bitsDeps", "pb", "aliBuild"] write_to = "bits_helpers/_version.py" [tool.setuptools.package-data] -bits_helpers = ['build_template.sh', 'relocate-me.sh', 'relativize-configs.sh', 'view-list.sh'] +bits_helpers = ['build_template.sh', 'relocate-me.sh', 'relocate-references.sh', 'relativize-configs.sh', 'view-list.sh', 'site.d/*.yaml'] [tool.setuptools.packages.find] where = ["."] @@ -76,6 +76,8 @@ readme = {file = ["README.md"], content-type = "text/markdown"} # config drives a CHECK-ONLY lint gate. line-length = 127 target-version = "py38" +# Third-party code kept unmodified (its own globals come from the browser build). +extend-exclude = ["console-backend/console_backend/vendor"] [tool.ruff.lint] # Start with the real-bug subset only — syntax errors (E9), bad comparisons/ diff --git a/requirements.txt b/requirements.txt index 671a97ea..a79aaa84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ jinja2 boto3 botocore qrcode +cryptography diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..bd023506 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +# The tests see bits' own site configuration only, not this machine's. +# tox (unittest discover) does the same through BITS_SITE_ISOLATED in tox.ini. +import os + +os.environ["BITS_SITE_ISOLATED"] = "1" diff --git a/tests/test_args.py b/tests/test_args.py index 9e1825d2..5b04d4ab 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -27,6 +27,8 @@ PARSER_ERRORS = { "build --force-unknown-architecture": BUILD_MISSING_PKG_ERROR, "build --force-unknown-architecture zlib --foo": 'unrecognized arguments: --foo', + "build --force-unknown-architecture zlib --symlinks": 'unrecognized arguments: --symlinks', + "status zlib --symlinks": 'unrecognized arguments: --symlinks', "init --docker-image": 'unrecognized arguments: --docker-image', "builda --force-unknown-architecture zlib" : "argument action: invalid choice: 'builda'.*", "build --force-unknown-architecture zlib --no-system --always-prefer-system" : 'argument --prefer-system/--always-prefer-system: not allowed with argument --no-system', @@ -45,6 +47,10 @@ class FakeExit(Exception): pass CORRECT_BEHAVIOR = [ + ((), "build --force-unknown-architecture gcc --symlink-packages-from /deploy --symlink-policy relaxed", [("symlinkPolicy", "relaxed"), ("symlinkPackagesFrom", "/deploy")]), + ((), "build --force-unknown-architecture gcc --reuse-from /deployment/Modules/modulefiles", [("reuseFrom", "/deployment/Modules/modulefiles"), ("symlinkPackagesFrom", None)]), + ((), "build --force-unknown-architecture gcc --reuse-from /deployment/Modules/modulefiles --symlink-packages-from /local/Packages", [("symlinkPackagesFrom", "/local/Packages")]), + ((), "build --force-unknown-architecture gcc", [("symlinkPackagesFrom", None)]), ((), "build --force-unknown-architecture zlib" , [("action", "build"), ("workDir", "sw"), ("referenceSources", "sw/MIRROR")]), ((), "init" , [("action", "init"), ("workDir", "sw"), ("referenceSources", "sw/MIRROR")]), ((), "version" , [("action", "version")]), @@ -55,6 +61,7 @@ class FakeExit(Exception): ((), "build --force-unknown-architecture -u zlib" , [("fetchRepos", True)]), ((), "build --force-unknown-architecture --no-fetch-repos zlib" , [("fetchRepos", False)]), ((), "status zlib" , [("fetchRepos", True)]), + ((), "status zlib --symlink-packages-from /deploy --symlink-policy relaxed --build-local gcc", [("symlinkPackagesFrom", "/deploy"), ("symlinkPolicy", "relaxed"), ("buildLocal", "gcc")]), ((), "status --no-fetch-repos zlib" , [("fetchRepos", False)]), ((), "build --force-unknown-architecture -j 10 zlib --disable gcc --disable foo" , [("disable", ["gcc", "foo"])]), ((), "build --force-unknown-architecture -j 10 zlib --disable gcc --disable foo,bar" , [("disable", ["gcc", "foo", "bar"])]), @@ -66,7 +73,11 @@ class FakeExit(Exception): ((), "build --force-unknown-architecture zlib --remote-store rsync://test.local/::rw", [("noSystem", None), ("remoteStore", "rsync://test.local/"), ("writeStore", "rsync://test.local/")]), ((), "build --force-unknown-architecture zlib --no-remote-store --remote-store rsync://test.local/", [("noSystem", None), ("remoteStore", "")]), ((), "build zlib --architecture slc7_x86-64" , [("noSystem", None), ("preferSystem", False), ("remoteStore", "https://s3.cern.ch/swift/v1/alibuild-repo")]), - ((), "build zlib --architecture ubuntu1804_x86-64" , [("noSystem", None), ("preferSystem", False), ("remoteStore", "")]), + ((), "build zlib --architecture ubuntu1804_x86-64" , [("noSystem", None), ("preferSystem", False), ("remoteStore", "https://s3.cern.ch/swift/v1/lcgapp-bits-testing")]), + # A write store alone is also the read store (build.py): no site store.read then, + # but an architecture's own read store stays. + ((), "build zlib --architecture ubuntu1804_x86-64 --write-store b3://mine" , [("remoteStore", ""), ("writeStore", "b3://mine")]), + ((), "build zlib --architecture slc7_x86-64 --write-store b3://mine" , [("remoteStore", "https://s3.cern.ch/swift/v1/alibuild-repo"), ("writeStore", "b3://mine")]), ((), "status zlib --write-store b3://bits-write" , [("action", "status"), ("writeStore", "b3://bits-write")]), ((), "build zlib -a slc7_x86-64" , [("docker", False), ("dockerImage", None), ("docker_extra_args", ["--network=host", _MOCK_CPUSET_ARG])]), ((), "build zlib -a slc7_x86-64 --docker-image registry.cern.ch/alisw/some-builder" , [("docker", True), ("dockerImage", "registry.cern.ch/alisw/some-builder")]), @@ -96,8 +107,19 @@ class FakeExit(Exception): } class ArgsTestCase(unittest.TestCase): + @mock.patch.dict(os.environ, {"BITS_BRANDING": "aliBuild"}) + @mock.patch("bits_helpers.args._defaults_docker", return_value=None) + @mock.patch("bits_helpers.arch.getoutput", new=lambda cmd: "x86_64") + @mock.patch("bits_helpers.args._host_online_cpus", return_value=_MOCK_CPUSET) + @mock.patch("bits_helpers.args._docker_memory_args", return_value=[]) + @mock.patch("bits_helpers.args._is_rootless_podman", return_value=False) + def test_alibuild_has_no_site_wide_read_store(self, *_): + for arch, store in (("ubuntu1804_x86-64", ""), ("slc7_x86-64", "https://s3.cern.ch/swift/v1/alibuild-repo")): + with patch.object(sys, "argv", ["alibuild", "build", "zlib", "--architecture", arch]): + self.assertEqual(doParseArgs()[0].remoteStore, store) + @mock.patch.dict(os.environ, {"BITS_DOCKER_REGISTRY": "", "BITS_LEGACY_REGISTRY": "", "BITS_DOCKER_TAG": ""}) - @mock.patch("bits_helpers.args._defaults_docker_registry", return_value=None) + @mock.patch("bits_helpers.args._defaults_docker", return_value=None) @mock.patch("bits_helpers.arch.getoutput", new=lambda cmd: "x86_64") # for uname -m @mock.patch("bits_helpers.args._host_online_cpus", return_value=_MOCK_CPUSET) # Neutralise the host-dependent --memory/--memory-swap docker injection so @@ -117,6 +139,26 @@ def test_actionParsing(self, mock_commands, _mock_podman, _mock_mem, _mock_cpus, for k, v in effects: self.assertEqual(args[k], v) + @mock.patch.dict(os.environ, {"BITS_DOCKER_REGISTRY": "", "BITS_LEGACY_REGISTRY": "", "BITS_DOCKER_TAG": ""}) + @mock.patch("bits_helpers.args._defaults_docker", return_value=None) + @mock.patch("bits_helpers.arch.getoutput", new=lambda cmd: "x86_64") # for uname -m + @mock.patch("bits_helpers.args._host_online_cpus", return_value=_MOCK_CPUSET) + @mock.patch("bits_helpers.args._docker_memory_args", return_value=[]) + @mock.patch("bits_helpers.args._is_rootless_podman", return_value=False) + @mock.patch('bits_helpers.args.commands') + def test_status_docker_as_build(self, mock_commands, *_): + """bits status takes the container options as bits build does.""" + mock_commands.getstatusoutput.side_effect = lambda x : GETSTATUSOUTPUT_MOCKS[x] + with patch.object(sys, "argv", ["bits", "status", "--docker-image", "img:1", "-a", "slc9_x86-64", + "zlib"]): + args, _ = doParseArgs() + self.assertTrue(args.docker) + self.assertEqual(args.dockerImage, "img:1") + self.assertIn("--network=host", args.docker_extra_args) + with patch.object(sys, "argv", ["bits", "status", "--docker", "-a", "slc9_x86-64", "zlib"]): + args, _ = doParseArgs() + self.assertEqual(args.dockerImage, "gitlab-registry.cern.ch/bits/containers/x86_64-slc9:latest") + @mock.patch("bits_helpers.arch.getoutput", new=lambda cmd: "x86_64") # for uname -m @mock.patch('bits_helpers.args.argparse.ArgumentParser.error') def test_failingParsing(self, mock_print): @@ -446,6 +488,12 @@ def test_set_is_parsed_into_flavours(self): args, _ = doParseArgs() self.assertEqual(args.flavours, {"release": "LCG_110", "foo": "true"}) + def test_status_takes_set_too(self): + with patch.object(sys, "argv", ["bits"] + shlex.split( + "status ROOT -a slc7_x86-64 --set release=LCG_110")): + args, _ = doParseArgs() + self.assertEqual(args.flavours, {"release": "LCG_110"}) + class SubcommandDryRunTestCase(unittest.TestCase): """-n/--dry-run listed in init/clean/publish help keeps the hoisted value.""" @@ -467,5 +515,28 @@ def test_forms(self): self.assertEqual(bits_string(""), {"repo": "alisw/alidist", "ver": ""}) + +class DryRunOrderTestCase(unittest.TestCase): + """-n before a command with its own --dry-run is that command's.""" + + def _parse(self, cmd): + with mock.patch("bits_helpers.arch.getoutput", return_value="x86_64"), \ + mock.patch("bits_helpers.args._host_online_cpus", return_value="0-7"), \ + mock.patch("bits_helpers.args.commands") as mock_cmd, \ + patch.object(sys, "argv", ["bits"] + shlex.split(cmd)): + mock_cmd.getstatusoutput.side_effect = lambda x: GETSTATUSOUTPUT_MOCKS[x] + args, _ = doParseArgs() + return vars(args) + + def test_prune(self): + self.assertTrue(self._parse("-n prune")["dryRun"]) + self.assertTrue(self._parse("prune -n")["dryRun"]) + self.assertFalse(self._parse("prune")["dryRun"]) + + def test_compliance(self): + self.assertTrue(self._parse("--dry-run compliance")["dryRun"]) + with self.assertRaises(SystemExit): # it has no -n: refused, not ignored + self._parse("-n compliance") + if __name__ == '__main__': unittest.main() diff --git a/tests/test_async_build.py b/tests/test_async_build.py index 1cf66fe5..3343ef7b 100644 --- a/tests/test_async_build.py +++ b/tests/test_async_build.py @@ -9,6 +9,7 @@ * ``checkout_sources()`` with ``parallel_sources > 1`` (concurrent source downloads) """ +from pathlib import Path import os import re import shutil @@ -16,6 +17,7 @@ import tempfile import threading import unittest +from argparse import Namespace from collections import OrderedDict from unittest.mock import MagicMock, patch @@ -100,6 +102,26 @@ def test_single_quotes_escaped(self): self.assertTrue(cmd.startswith("bash -e -c '")) +class S3RemoteSyncListTest(unittest.TestCase): + """S3RemoteSync lists a store path with s3cmd ls (bits build --dry-run).""" + + def test_listing(self): + from unittest import mock + from bits_helpers.sync import S3RemoteSync + from bits_helpers.plan import store_can_list + sync = S3RemoteSync("s3://bkt", "", "slc9_x86-64", "/sw") + self.assertTrue(store_can_list(sync)) + out = ("2026-10-19 10:00 1234 s3://bkt/TARS/slc9_x86-64/store/ab/abc/zlib-1.3-1.slc9_x86-64.tar.gz\n" + " DIR s3://bkt/TARS/slc9_x86-64/store/ab/abc/sub/\n") + run = mock.Mock(return_value=mock.Mock(returncode=0, stdout=out, stderr="")) + with mock.patch("subprocess.run", run): + self.assertEqual(sync.list_store_tarballs("slc9_x86-64", "abc"), + ["zlib-1.3-1.slc9_x86-64.tar.gz"]) + self.assertEqual(run.call_args[0][0][-1], "s3://bkt/TARS/slc9_x86-64/store/ab/abc/") + with mock.patch("subprocess.run", mock.Mock(side_effect=OSError("no s3cmd"))): + self.assertEqual(sync.list_store_tarballs("slc9_x86-64", "abc"), []) + + class S3RemoteSyncUploadCmdTest(unittest.TestCase): """S3RemoteSync (s3cmd backend) upload_shell_command tests.""" @@ -559,7 +581,7 @@ def test_summary_lists_direct_and_cascaded(self): # sharing one work area do not clobber each other self.assertEqual(os.path.dirname(path), os.path.join(self.dir, "LOGS", "el9_x86-64")) - text = open(path).read() + text = Path(path).read_text() self.assertIn("FAILED: motif@2.3.8", text) self.assertIn("err: boom", text) # excerpt included self.assertIn("1 package(s) failed", text) @@ -568,7 +590,29 @@ def test_summary_lists_direct_and_cascaded(self): self.assertNotIn("motif", text.split("Skipped")[1]) # not double-counted # the combined full error log is also written self.assertTrue(full and os.path.exists(full)) - self.assertIn("build:motif", open(full).read()) + self.assertIn("build:motif", Path(full).read_text()) + + def test_banner_lists_each_failed_package_log(self): + from bits_helpers.build import report_failure_summary + sched = self._Sched( + fails=[{"package": "zlib@1.3.1", "log": "/scratch/BUILD/zlib-latest/log"}, + {"package": "motif@2.3.8", "log": "/sw/BUILD/motif-latest-dev/log"}], + errors={"build:foo": "The following dependencies could not complete:\nbuild:motif"}) + with self.assertLogs("bits", level="INFO") as logs: + report_failure_summary(self.dir, sched, ARCH) + text = "\n".join(logs.output) + self.assertIn("motif@2.3.8: /sw/BUILD/motif-latest-dev/log", text) + self.assertIn("zlib@1.3.1: /scratch/BUILD/zlib-latest/log", text) + self.assertLess(text.index("motif@"), text.index("zlib@")) + self.assertNotIn("", text) + self.assertNotIn("foo", text) + self.assertIn(os.path.join(self.dir, "LOGS", ARCH, "build-summary.log"), text) + + def test_success_has_no_failure_banner(self): + from bits_helpers.build import report_failure_summary + with patch("bits_helpers.build.info") as info: + report_failure_summary(self.dir, self._Sched([], {}), ARCH) + info.assert_not_called() def test_no_failures_writes_nothing(self): from bits_helpers.build import write_failure_summary @@ -576,3 +620,41 @@ def test_no_failures_writes_nothing(self): (None, None)) self.assertFalse(os.path.exists( os.path.join(self.dir, "LOGS", "el9_x86-64", "build-summary.log"))) + + def test_failure_log_paths_are_absolute(self): + from bits_helpers.build import runBuildCommand + for builders in (1, 2): + for devel in (False, True): + for override in ("", os.path.relpath(os.path.join(self.dir, "scratch"))): + with self.subTest(builders=builders, devel=devel, override=override): + args = Namespace(builders=builders, workDir=os.path.relpath(self.dir), + develPrefix="dev", debug=False, resourceMonitoring=False, + architecture=ARCH) + spec = dict(GOOD_SPEC, is_devel_pkg=devel, requires=[]) + build_dir = self.dir if devel or not override else os.path.abspath(override) + log_path = os.path.join(build_dir, "BUILD", + "zlib-latest-dev" if devel else "zlib-latest", "log") + os.makedirs(os.path.dirname(log_path), exist_ok=True) + with open(log_path, "w") as log: + log.write("error: fixture build failed\n") + sched = self._Sched([], {}) + sched.buildFailuresLock = threading.Lock() + sched.log = MagicMock() + with patch.dict(os.environ, {"BITS_BUILD_WORK_DIR": override or args.workDir, + "BITS_DIST_HASH": "0123456789"}), \ + patch("bits_helpers.build.execute", return_value=7), \ + patch("bits_helpers.build.detectArch", return_value=ARCH), \ + patch("bits_helpers.monitor.note_build"): + if builders == 1: + with self.assertLogs("bits", level="ERROR") as logs: + with self.assertRaises(SystemExit) as exc: + runBuildCommand(sched, "zlib", {"zlib": spec}, args, + "exit 7", None, self.dir, self.dir, None) + self.assertEqual(exc.exception.code, 1) + message = "\n".join(logs.output) + else: + message = runBuildCommand(sched, "zlib", {"zlib": spec}, args, + "exit 7", None, self.dir, self.dir, None) + self.assertEqual(sched.buildFailures[0]["log"], log_path) + self.assertIn(" " + log_path + "\n", message) + self.assertIn("error: fixture build failed", message) diff --git a/tests/test_bits_use.py b/tests/test_bits_use.py index 754808c1..f911ddee 100644 --- a/tests/test_bits_use.py +++ b/tests/test_bits_use.py @@ -3,6 +3,7 @@ """Tests for bits_helpers/bits_use — the .bitsuse saved-arg profile.""" +from pathlib import Path import os import shutil import sys @@ -204,7 +205,7 @@ def test_home_fallback_when_cwd_not_writeable(self, _access): p = U.write_section("build", ["--docker"]) self.assertTrue(U._is_home_path(p)) self.assertTrue(os.path.exists(p)) - self.assertIn("# dir:", open(p).read()) # dir header recorded + self.assertIn("# dir:", Path(p).read_text()) # dir header recorded self.assertEqual(os.path.abspath(U._read_path()), os.path.abspath(p)) self.assertEqual(U.rewrite_argv(["build", "x"]), ["build", "--docker", "x"]) diff --git a/tests/test_build.py b/tests/test_build.py index a81e1cc8..02dde7f2 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -16,7 +16,9 @@ from bits_helpers.utilities import resolve_tag, resolve_store_path from bits_helpers.recipe import parseRecipe -from bits_helpers.build import doBuild, build_one_package, _BuildLoopCtx +from bits_helpers.build import (doBuild, build_one_package, _BuildLoopCtx, enter_target, + latest_alias_note) +from bits_helpers import plan from bits_helpers.hashing import storeHashes from bits_helpers.initdotsh import generate_initdotsh @@ -595,6 +597,11 @@ def test_initdotsh(self) -> None: self.assertIn("export APPEND_ROOT_1=", complete_initdotsh) self.assertIn("export PREPEND_ROOT_1=", complete_initdotsh) + # A not-needed dependency (issue #141: build-only, of a reused package) + # is not installed, so it is not sourced. + specs["zlib"]["not_needed"] = True + self.assertNotIn("/zlib/", generate_initdotsh("ROOT", specs, "slc7_x86-64", post_build=True)) + def test_initdotsh_from_modules(self) -> None: """--initdotsh-from-modules adds the modulefile-equivalent dev env (_INCLUDE_DIR, PYTHONPATH site-packages) to the package's own @@ -732,6 +739,20 @@ def test_initdotsh_reuse_orders_prereq_before_reused(self) -> None: "local prerequisite Tools must be sourced before reused CMake") +class AfterRecipeTest(unittest.TestCase): + def test_the_error_after_a_successful_recipe_is_shown(self): + from bits_helpers.build import _after_recipe + with tempfile.NamedTemporaryFile("w", suffix=".log") as log: + log.write("+ make\nbuilt\n") + log.flush() + out = ["make: error: an earlier, harmless one", "+ make", "built", "+ rsync -aH a b", + "build.sh: line 534: rsync: command not found", "+ grep error: x", + "bits: done building"] + self.assertEqual(_after_recipe(out, log.name), + ["build.sh: line 534: rsync: command not found"]) + self.assertEqual(_after_recipe(["+ make", "built", "+ exit 2"], log.name), []) + self.assertEqual(_after_recipe(out, log.name + ".missing"), []) + class HashPolicyBuildFallbackTest(unittest.TestCase): """Exercise the remote-first hash-policy decision in build_one_package.""" @@ -741,7 +762,7 @@ class HashPolicyBuildFallbackTest(unittest.TestCase): class StopAtBuildScript(Exception): pass - def _run_package(self, work_dir, *, remote_tarball=False): + def _run_package(self, work_dir, *, remote_tarball=False, prepare=None): arch = TEST_ARCHITECTURE remote_hash = self.remote_hash spec = { @@ -790,6 +811,8 @@ def fetch_tarball(self, _spec): packages=[], prefetch_executor=None, cmake_prefix_env=False, specs_for_checksum_phase=[], monitoredDirs={}, ) + if prepare: + ctx = prepare(ctx) real_open = _REAL_OPEN @@ -853,5 +876,74 @@ def test_remote_match_is_preferred_to_existing_local_install(self): self.remote_hash, TEST_ARCHITECTURE))) + +class PlannedReuseBuiltAfterAllTest(unittest.TestCase): + """Issue #141: a package planned as reused whose remote tarball the signed + manifest does not vouch for gets the build-only dependencies the plan + skipped back first, and its remote tarball stays gated on the next pass.""" + + remote_hash = HashPolicyBuildFallbackTest.remote_hash + local_hash = HashPolicyBuildFallbackTest.local_hash + StopAtBuildScript = HashPolicyBuildFallbackTest.StopAtBuildScript + _run_package = HashPolicyBuildFallbackTest._run_package + + def _run(self, work_dir, skipped, order, prefetched=None, planned=plan.REMOTE): + def prepare(ctx): + app = ctx.specs["hash-policy-test"] + if prefetched is not None: # what the pass before brought + app["prefetched_tarballs"] = prefetched + ctx.specs["cmake"] = {"package": "cmake", "is_devel_pkg": False, "requires": [], + "hash": "c" * 40, "revision": "1", "not_needed": True} + app["requires"] = app["build_requires"] = ["cmake"] + ctx.cfg.require_signed_reuse = True + ctx.planned = {"hash-policy-test": planned, "cmake": plan.REMOTE} + ctx.skipped, ctx.build_order = skipped, order + ctx.full_order = ["cmake", "hash-policy-test"] + self.ctx = ctx + return ctx + with patch("bits_helpers.build.trusted_reuse_index", return_value={}): + return self._run_package(work_dir, remote_tarball=True, prepare=prepare) + + def test_skipped_dependencies_come_back_first(self): + with tempfile.TemporaryDirectory() as work_dir: + order = [] + result, spec = self._run(work_dir, {"cmake"}, order) + self.assertIsNone(result) + self.assertEqual(order, ["cmake", "hash-policy-test"]) + self.assertEqual(self.ctx.skipped, set()) + self.assertEqual(set(self.ctx.specs["cmake"]), {"package", "is_devel_pkg", "requires"}) + self.assertEqual(len(spec["prefetched_tarballs"]), 1) + # The next pass: the remote tarball left on disk is still not reused. + result, spec = self._run(work_dir, set(), [], spec["prefetched_tarballs"]) + self.assertIsInstance(result, self.StopAtBuildScript) + self.assertEqual(spec["cachedTarball"], "") + + def test_without_skipped_dependencies_it_is_built_at_once(self): + for skipped, planned in ((set(), plan.REMOTE), ({"cmake"}, plan.BUILD)): + with self.subTest(planned=planned), tempfile.TemporaryDirectory() as work_dir: + order = [] + result, spec = self._run(work_dir, skipped, order, planned=planned) + self.assertIsInstance(result, self.StopAtBuildScript) + self.assertEqual((order, spec["cachedTarball"]), ([], "")) + + +class EnterTargetTest(unittest.TestCase): + def test_built_version(self): + spec = {"package": "Athena", "version": "25.0.71", "revision": "1"} + self.assertEqual(enter_target("Athena", spec, "release_atlas_gcc15_dbg"), + "Athena/25.0.71-1") + + def test_no_spec_falls_back_to_latest(self): + self.assertEqual(enter_target("Athena", None, "release_atlas_gcc15_dbg"), + "Athena/latest-release_atlas_gcc15_dbg") + + def test_the_latest_alias_is_mentioned_with_the_version(self): + spec = {"package": "Athena", "version": "25.0.71", "revision": "1"} + self.assertIn("Athena/latest-release_atlas_gcc15_dbg enters the newest build", + latest_alias_note("Athena", spec, "release_atlas_gcc15_dbg")) + self.assertEqual(latest_alias_note("Athena", None, "release_atlas_gcc15_dbg"), "") + self.assertEqual(latest_alias_note("Athena", spec, ""), "") # no family: no alias + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_build_config.py b/tests/test_build_config.py index 63d1c815..1c4e92a9 100644 --- a/tests/test_build_config.py +++ b/tests/test_build_config.py @@ -26,6 +26,8 @@ def test_resolved_values_pass_through(self): memPerJobDefault=2048, parallelDownloads=8, parallelSources=4, prefetchWorkers=3, reuseOverlay="/sw/MODULES/ov", reuseCvmfsBase="/cvmfs/x/Packages", + symlinkPackagesFrom="/scratch/deploy/Packages", + symlinkPolicy="relaxed", reuseBeacon="https://beacon", storeIntegrity=True, monitor="on", monitorUrl="https://m", monitorInstance="i7", monitorInterval=30.0, monitorDiskInterval=90.0, @@ -43,6 +45,8 @@ def test_resolved_values_pass_through(self): self.assertEqual(cfg.prefetch_workers, 3) self.assertEqual(cfg.reuse_overlay, "/sw/MODULES/ov") self.assertEqual(cfg.reuse_cvmfs_base, "/cvmfs/x/Packages") + self.assertEqual(cfg.symlink_packages_from, "/scratch/deploy/Packages") + self.assertEqual(cfg.symlink_policy, "relaxed") self.assertEqual(cfg.reuse_beacon, "https://beacon") self.assertIs(cfg.store_integrity, True) self.assertEqual(cfg.monitor, "on") @@ -77,6 +81,8 @@ def test_absent_knobs_take_the_site_hard_defaults(self): self.assertEqual(cfg.prefetch_workers, -1) self.assertIsNone(cfg.reuse_overlay) self.assertIsNone(cfg.reuse_cvmfs_base) + self.assertIsNone(cfg.symlink_packages_from) + self.assertEqual(cfg.symlink_policy, "strict") self.assertIsNone(cfg.reuse_beacon) self.assertIs(cfg.store_integrity, False) self.assertIsNone(cfg.monitor) diff --git a/tests/test_build_progress.py b/tests/test_build_progress.py new file mode 100644 index 00000000..9b774f14 --- /dev/null +++ b/tests/test_build_progress.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Console progress must work in terminals, redirected logs and parallel builds.""" + +from argparse import Namespace +from concurrent.futures import ThreadPoolExecutor +from io import StringIO +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from bits_helpers import build +from bits_helpers.build_progress import BuildProgress, package_info, package_label, set_package_progress +from bits_helpers.cmd import execute +from bits_helpers.download import download +from bits_helpers.workarea import checkout_sources + + +class BuildProgressTest(unittest.TestCase): + def test_plan_numbers_include_reuse_and_exclude_defaults_and_unselected(self): + specs = {p: {"package": p, "version": "1"} for p in ("defaults-release", "A", "B", "Main")} + set_package_progress(specs, ["defaults-release", "A", "B"]) + self.assertEqual(package_label(specs["A"]), "[1/2] A@1") + self.assertEqual(package_label(specs["B"], "dev"), "[2/2] B@dev") + self.assertNotIn("_build_progress", specs["defaults-release"]) + self.assertNotIn("_build_progress", specs["Main"]) + + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=False) + def test_redirected_command_reports_stages_without_compiler_noise(self, _tty): + messages = [] + progress = BuildProgress("[3/16] Python@3.13.11", "Building", emit=messages.append) + command = ["/bin/bash", "-ex", "-c", + 'echo bits:stage:prepare; echo bits:stage:recipe; ' + 'Configure() { :; }; Make() { echo "[3/5] compiler output"; }; ' + 'MakeInstall() { :; }; Configure; Make; MakeInstall; ' + 'echo bits:stage:package; echo bits:stage:relocate'] + self.assertEqual(execute(command, printer=progress), 0) + progress.end() + self.assertEqual(messages, [ + "==> [3/16] Python@3.13.11: Building", + "==> [3/16] Python@3.13.11: Preparing build environment", + "==> [3/16] Python@3.13.11: Running build recipe", + "==> [3/16] Python@3.13.11: Configuring", + "==> [3/16] Python@3.13.11: Compiling", + "==> [3/16] Python@3.13.11: Installing", + "==> [3/16] Python@3.13.11: done", + ]) + + @patch("bits_helpers.log.logger.isEnabledFor", return_value=False) + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=True) + def test_terminal_stage_changes_reset_compiler_percentage(self, _tty, _debug): + output = StringIO() + with patch("bits_helpers.log.sys.stderr", output): + progress = BuildProgress("[1/2] A@1", "Compiling") + progress("%s", "bits:stage:recipe") + progress.display.lasttime -= 1 + progress("%s", "[1/2] Building CXX object") + self.assertEqual(progress.display.percent, 50) + progress("%s", "bits:stage:package") + self.assertEqual(progress.display.percent, -1) + progress.end() + self.assertIn("Running build recipe", output.getvalue()) + self.assertIn("50%", output.getvalue()) + self.assertIn("Installing", output.getvalue()) + # the line ends cleared: no spaces left where the spinner was + self.assertIn(": \033[32mdone\033[m\033[K\n", output.getvalue()) + + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=True) + def test_parallel_output_is_labeled_and_does_not_use_spinners(self, _tty): + messages = [] + + def report(i): + progress = BuildProgress("[{}/8] P{}@1".format(i, i), "Compiling", + emit=messages.append, interactive=False) + progress("%s", "bits:stage:recipe") + progress.end() + + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(report, range(1, 9))) + self.assertEqual(len(messages), 24) + for i in range(1, 9): + self.assertEqual(len([m for m in messages if m.startswith("==> [{}/8]".format(i))]), 3) + self.assertFalse(any("\033" in m for m in messages)) + + @patch("bits_helpers.build_progress.logger.isEnabledFor", return_value=True) + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=True) + @patch("bits_helpers.build_progress.debug") + def test_debug_retains_build_output_and_ignores_traced_markers(self, debug, _tty, _enabled): + messages = [] + progress = BuildProgress("A@1", "Compiling", emit=messages.append) + progress("%s", "+ echo bits:stage:package") + progress("%s", "compiler details") + progress("%s", "bits:stage:unknown") + self.assertEqual(len(messages), 1) + self.assertEqual(debug.call_count, 3) + progress("%s", "bits:stage:package") + self.assertEqual(messages[-1], "==> A@1: Installing") + + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=False) + def test_failure_is_visible_even_without_subprocess_output(self, _tty): + messages = [] + progress = BuildProgress("[2/3] A@1", "Compiling", emit=messages.append) + self.assertEqual(execute(["/bin/bash", "-c", "exit 7"], printer=progress), 7) + progress.end(error=True) + self.assertEqual(messages, ["==> [2/3] A@1: Compiling", "==> [2/3] A@1: failed"]) + + @patch("bits_helpers.build_progress.sys.stdout.isatty", return_value=False) + def test_run_build_reports_cached_archive_and_store_sync(self, _tty): + spec = {"package": "A", "version": "1", "revision": "1", "hash": "a" * 40, + "is_devel_pkg": False, "requires": [], "_build_progress": (2, 3)} + args = Namespace(builders=1, debug=False, resourceMonitoring=False, + workDir="/sw", architecture="slc7_x86-64") + helper = Namespace(remoteStore="b3://cache", writeStore="b3://cache") + messages = [] + with patch.object(build, "info", messages.append), \ + patch.object(build, "execute", side_effect=lambda cmd, printer: execute(cmd, printer=printer)), \ + patch.object(build, "doFinalSync") as sync, \ + patch.object(build, "detectArch", return_value="slc7_x86-64"), \ + patch.dict("os.environ", {"BITS_DIST_HASH": "0" * 40}), \ + patch("bits_helpers.monitor.note_build"), \ + patch("bits_helpers.progress.tick"): + build.runBuildCommand(None, "A", {"A": spec}, args, + ["/bin/bash", "-c", "echo bits:stage:unpack"], + "/cache/A.tar.gz", "/specs/A", "/sw", helper) + sync.assert_called_once() + self.assertEqual(messages, [ + "==> [2/3] A@1: Unpacking [from store]", + "==> [2/3] A@1: Unpacking cached archive", + "==> [2/3] A@1: Syncing package to store", + "==> [2/3] A@1: done", + ]) + + def test_common_build_commands_report_their_stage(self): + commands = { + "configure": [ + "++ ./configure --prefix='/path with spaces'", + "+ /src/Configure linux-x86_64", "+ perl ./Configure", + "+ cmake -S /src -B build", "+ env CC=gcc cmake ..", + "+ meson setup build", + ], + "compile": [ + "+ make -j8", "+ gmake -C build", "+ ninja -C build", + "+ cmake --build build --parallel 8", "+ meson compile -C build", + "+ python3 -m build", "+ python3 setup.py build_ext", + ], + "install": [ + "+ make -j 8 install", "+ make -j install", "+ gmake install-strip", + "+ ninja -C build install", "+ cmake --install build", + "+ cmake --build build --target install", "+ meson install -C build", + "+ python3.13 -m pip install .", "+ python setup.py install", + "+ install -m 755 program /dest/bin", + ], + } + for stage, traces in commands.items(): + for trace in traces: + with self.subTest(trace=trace): + progress = BuildProgress("A@1", "Building", emit=lambda text: None, interactive=False) + progress("bits:stage:recipe") + progress(trace) + self.assertEqual(progress.stage, {"configure": "Configuring", + "compile": "Compiling", "install": "Installing"}[stage]) + + def test_trace_detection_ignores_output_queries_and_commands_outside_recipe(self): + messages = [] + progress = BuildProgress("A@1", "Building", emit=messages.append, interactive=False) + progress("+ cmake ..") + self.assertEqual(progress.stage, "Building") + progress("bits:stage:recipe") + for line in ("make install", "+ echo make install", "+ echo bits:stage:install", + "+ cmake --version", "+ cmake -E copy a b", "+ cmake -P helper.cmake", + "+ make --version", "+ make clean", "+ ninja -t targets", + "+ ninja -n install", "+ make -C install", "+ echo 'unterminated"): + progress(line) + # -C install names a directory; it is still a compile command. + self.assertEqual(messages, ["==> A@1: Building", "==> A@1: Running build recipe", "==> A@1: Compiling"]) + progress("bits:stage:post-install") + progress("+ cmake ..") + self.assertEqual(progress.stage, "Installing") + + def test_explicit_recipe_markers_support_custom_build_tools(self): + messages = [] + progress = BuildProgress("A@1", "Building", emit=messages.append, interactive=False) + self.assertEqual(execute(["/bin/bash", "-ex", "-c", + "echo bits:stage:configure; echo bits:stage:compile; " + "echo bits:stage:install"], printer=progress), 0) + self.assertEqual(messages, ["==> A@1: Building", "==> A@1: Configuring", "==> A@1: Compiling", + "==> A@1: Installing"]) + + +class PackageLineTest(unittest.TestCase): + def test_package_lines_align_with_the_stage_lines(self): + with patch("bits_helpers.build_progress.info") as log: + package_info("%s: Fetching sources from %s", "[3/16] demo@1", "a.tar.gz") + log.assert_called_once_with("==> %s: Fetching sources from %s", "[3/16] demo@1", "a.tar.gz") + + +class SourceProgressTest(unittest.TestCase): + def test_archive_origin_reports_upstream_store_fallback_and_local_cache(self): + url = "https://example.com/demo-1.tar.gz" + messages = [] + helper = Namespace(remoteStore="b3://cache", fetch_source=lambda *args: False, + upload_source=lambda *args: None) + + def fetch(source, directory, work_dir): + Path(directory, "demo-1.tar.gz").write_bytes(b"source archive") + + with tempfile.TemporaryDirectory() as tmp, \ + patch.dict("bits_helpers.download.downloadHandlers", {"https": fetch}): + download(url, None, tmp, sync_helper=helper, progress=messages.append) + download(url, None, tmp, sync_helper=helper, progress=messages.append) + self.assertEqual(messages[:2], ["b3://cache (source: {})".format(url), url]) + self.assertTrue(messages[2].startswith("local cache " + tmp)) + self.assertTrue(messages[2].endswith("(source: {})".format(url))) + + def test_store_hit_does_not_report_upstream_download(self): + url = "https://example.com/demo-1.tar.gz" + messages = [] + + def fetch(url_hash, filename, directory): + Path(directory, filename).write_bytes(b"cached source archive") + + helper = Namespace(remoteStore="https://cache.example.com", fetch_source=fetch) + with tempfile.TemporaryDirectory() as tmp: + download(url, None, tmp, sync_helper=helper, progress=messages.append) + self.assertEqual(messages, ["https://cache.example.com (source: {})".format(url)]) + + def test_git_checkout_reports_package_url_and_ref(self): + from bits_helpers.git import Git + spec = {"package": "demo", "version": "1", "source": "https://example.com/demo.git", + "scm": Git(), "tag": "v1", "commit_hash": "v1", "is_devel_pkg": False, + "_build_progress": (3, 16)} + with tempfile.TemporaryDirectory() as tmp, \ + patch("bits_helpers.workarea.logged_scm"), \ + patch("bits_helpers.workarea.package_info") as log: + checkout_sources(spec, tmp, tmp, False) + log.assert_called_once_with("%s: Fetching sources from %s", "[3/16] demo@1", + "https://example.com/demo.git (ref: v1)") + + def test_resolved_archive_url_has_package_label(self): + spec = {"package": "demo", "version": "1", "sources": [ + "(osx.*)https://example.com/mac.tar.gz", + "(slc.*)https://example.com/%(name)s-%(version)s.tar.gz,sha256:abc"], + "scm": None, "tag": "v1", "commit_hash": "v1", "_build_progress": (3, 16)} + + def fetch(url, directory, work_dir, **kwargs): + kwargs["progress"](url) + + with tempfile.TemporaryDirectory() as tmp, \ + patch("bits_helpers.workarea.download", side_effect=fetch), \ + patch("bits_helpers.workarea.package_info") as log: + checkout_sources(spec, tmp, tmp, False, architecture="slc7_x86-64") + log.assert_called_once_with("%s: Fetching sources from %s", "[3/16] demo@1", + "https://example.com/demo-1.tar.gz") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_build_stats.py b/tests/test_build_stats.py index 8c4d1123..07ed0296 100644 --- a/tests/test_build_stats.py +++ b/tests/test_build_stats.py @@ -6,6 +6,7 @@ that feeds the --builders scheduler's ResourceManager. """ +from pathlib import Path import json import os import tempfile @@ -60,7 +61,7 @@ def test_aggregate_takes_peaks(self): ]) path = bs.aggregate_and_write(self.dir, {"root": sd}) self.assertEqual(path, bs.default_stats_path(self.dir)) - stats = json.load(open(path)) + stats = json.loads(Path(path).read_text()) pkg = stats["packages"]["build"]["root"] self.assertEqual(pkg["rss"], 3_000_000_000) self.assertEqual(pkg["cpu"], 2800) @@ -81,7 +82,7 @@ def test_missing_trace_skipped(self): sd_ok = _write_trace(self.dir, "ok", [{"rss": 10, "cpu": 50, "time": 5}]) path = bs.aggregate_and_write( self.dir, {"ok": sd_ok, "gone": os.path.join(self.dir, "SPECS", "gone")}) - stats = json.load(open(path)) + stats = json.loads(Path(path).read_text()) self.assertIn("ok", stats["packages"]["build"]) self.assertNotIn("gone", stats["packages"]["build"]) @@ -90,13 +91,13 @@ def test_autoload_restamps_resources(self): bs.aggregate_and_write(self.dir, {"root": sd}) # corrupt the machine totals as if copied from another node path = bs.default_stats_path(self.dir) - data = json.load(open(path)) + data = json.loads(Path(path).read_text()) data["resources"] = {"cpu": -1, "rss": -1} - json.dump(data, open(path, "w")) + Path(path).write_text(json.dumps(data)) # autoload must re-stamp with sane current-machine values ap = bs.autoload_stats_path(self.dir) self.assertEqual(ap, path) - restamped = json.load(open(ap)) + restamped = json.loads(Path(ap).read_text()) self.assertGreater(restamped["resources"]["cpu"], 0) def test_autoload_missing_returns_none(self): @@ -127,7 +128,7 @@ def test_output_consumable_by_resource_manager(self): sd = _write_trace(self.dir, "root", [{"rss": 2_000_000, "cpu": 200, "time": 60}]) bs.aggregate_and_write(self.dir, {"root": sd}) path = bs.autoload_stats_path(self.dir) - rm = ResourceManager(json.load(open(path)), _FakeScheduler()) + rm = ResourceManager(json.loads(Path(path).read_text()), _FakeScheduler()) # a package present in the stats and a brand-new one (uses defaults) admitted = rm.allocResourcesForExternals( ["build:root", "build:brandnew"], count=4) @@ -194,7 +195,7 @@ def test_tuning_embedded_in_stats_file(self, _cc): rep = bs.tuning_report({"root": sd}, wall_seconds=100, builders=2, jobs=8, oversubscribe=1.0) bs.aggregate_and_write(self.dir, {"root": sd}, tuning=rep) - stats = json.load(open(bs.default_stats_path(self.dir))) + stats = json.loads(Path(bs.default_stats_path(self.dir)).read_text()) self.assertEqual(stats["tuning"], rep) self.assertIn("recommendation", stats["tuning"]) diff --git a/tests/test_certify.py b/tests/test_certify.py index 90db2b5e..4f53b03c 100644 --- a/tests/test_certify.py +++ b/tests/test_certify.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later """Tests for the certification core (bits_helpers/certify.py).""" +from pathlib import Path import json import os import shutil @@ -65,6 +66,17 @@ def test_provenance_fields_carried_into_common_manifest(self): self.assertEqual(e["bits_version"], "1.2.3") self.assertEqual(e["bits_dist_hash"], "deadbeefcafe") + def test_who_built_it_comes_from_the_build_that_compiled_it(self): + # A later build that only reused the package lists it without built_by, + # and with its own completed_at; the build that compiled it wins either way. + reused = _manifest("b2", [_pkg("A", "h1", "sha256:aa", built_by=None, + completed_at="2026-10-22T00:00:00Z")]) + built = _manifest("b1", [_pkg("A", "h1", "sha256:aa", built_by="u@host", + completed_at="2026-10-20T00:00:00Z")]) + for order in ([reused, built], [built, reused]): + e = certify.merge_common_manifest(order)["packages"][0] + self.assertEqual((e["built_by"], e["completed_at"]), ("u@host", "2026-10-20T00:00:00Z")) + def test_conflicting_sha_same_arch_fails_closed(self): # Same hash + same architecture => same store path => must be same bytes. m1 = _manifest("b1", [_pkg("A", "h1", "sha256:aa", arch="slc7_x86-64")]) @@ -383,7 +395,7 @@ def test_certified_by_recorded_in_signed_manifest(self): out = os.path.join(self.tmp, "out", "common.json") certify.certify([m], self.key_pem, out, probe=lambda a, h, t=None: "sha256:aa", approval_check=lambda common: ["alice", "bob"]) - doc = json.load(open(out)) + doc = json.loads(Path(out).read_text()) self.assertEqual(doc["certified_by"], ["alice", "bob"]) self.assertIn("certified_at", doc) kid, _ = trust.trusted_index(out) # signature still valid @@ -414,7 +426,7 @@ def error(self, m): with patch.object(forge, "gitlab_identify", return_value="alice"): certify.doSign(args, _P()) per_arch = certify._arch_stem(out, "slc7_x86-64") - self.assertEqual(json.load(open(per_arch))["certified_by"], ["alice"]) + self.assertEqual(json.loads(Path(per_arch).read_text())["certified_by"], ["alice"]) def test_identity_token_rejects_unauthorised_user(self): from bits_helpers import forge @@ -453,7 +465,7 @@ def error(self, m): certify.doSign(args, _P()) per_arch = certify._arch_stem(out, "slc7_x86-64") - self.assertEqual(json.load(open(per_arch))["certified_by"], ["alice"]) + self.assertEqual(json.loads(Path(per_arch).read_text())["certified_by"], ["alice"]) def test_certifier_username_rejected_when_not_admin(self): admins = os.path.join(self.tmp, "ADMINS") @@ -509,7 +521,7 @@ def test_tampering_breaks_signature(self): out = os.path.join(self.tmp, "out", "common.json") out_path, _ = certify.certify([m], self.key_pem, out, probe=lambda a, h, t=None: store.get((a, h))) - data = json.load(open(out_path)) + data = json.loads(Path(out_path).read_text()) data["packages"][0]["tarball_sha256"] = "sha256:evil" with open(out_path, "w") as fh: json.dump(data, fh, indent=1, sort_keys=True) @@ -535,6 +547,27 @@ def test_group_filter_scopes_reuse_index(self): _, lcg_idx = trust.trusted_index(out, accept_groups=["lcg"]) self.assertEqual(set(lcg_idx), {"hbase", "hlcg"}) + def test_the_bytes_verified_are_the_bytes_trusted(self): + # Another build replacing the local copy between the signature check and + # the parse must not get its (unverified) bytes trusted. + pkgs = [_pkg("Base", "hbase", "sha256:00")] + store = {("slc7_x86-64", "hbase"): "sha256:00"} + out = os.path.join(self.tmp, "out", "common.json") + certify.certify([_manifest("b1", pkgs)], self.key_pem, out, + probe=lambda a, h, t=None: store.get((a, h))) + tampered = json.loads(Path(out).read_text()) + tampered["packages"][0]["hash"] = "hevil" + real = trust.verify_bytes + + def verify_then_swap(*a, **k): + kid = real(*a, **k) + with open(out, "w") as fh: + json.dump(tampered, fh) + return kid + with patch.object(trust, "verify_bytes", side_effect=verify_then_swap): + _, idx = trust.trusted_index(out) + self.assertEqual(set(idx), {"hbase"}) + def test_build_trusted_reuse_index_honours_trust_groups(self): # End-to-end through the consumer gate: --trust-groups lcg parsed and # applied to the verified signed manifest. @@ -551,7 +584,39 @@ def test_build_trusted_reuse_index_honours_trust_groups(self): args = SimpleNamespace(trustManifest=out, trustGroups="lcg", requireSignedReuse=True) idx = build.trusted_reuse_index(args, self.tmp) - self.assertEqual(set(idx), {"hbase", "hlcg"}) # ship filtered out + self.assertEqual(set(idx), {("slc7_x86-64", "hbase"), + ("slc7_x86-64", "hlcg")}) # ship filtered out + + def test_the_same_hash_under_two_architectures_keeps_both(self): + # A hash does not include the architecture: the same one certified for + # two architectures is two tarballs, each with its own sha256, and + # reuse checks each against its own (not whichever manifest came last). + from bits_helpers import build + store = {("el9-opt", "h"): "sha256:11", ("el9", "h"): "sha256:22"} + srcs = [] + for arch in ("el9-opt", "el9"): # the bare one read last + out = os.path.join(self.tmp, "out", "common-%s.json" % arch) + certify.certify([_manifest("b-" + arch, [_pkg("A", "h", store[(arch, "h")], + arch=arch)])], + self.key_pem, out, probe=lambda a, h, t=None: store.get((a, h))) + srcs.append(out) + args = SimpleNamespace(trustManifest=",".join(srcs), trustGroups=None, + requireSignedReuse=True) + self.assertEqual(build.trusted_reuse_index(args, self.tmp), store) + + def test_one_object_certified_with_two_sha256_is_not_trusted(self): + from unittest import mock + from bits_helpers import build + parts = {"m1": ("k", {("a", "h"): "sha256:11", ("a", "g"): "sha256:33"}), + "m2": ("k", {("a", "h"): "sha256:22"}), + "m3": ("k", {("a", "h"): "sha256:11"})} + with mock.patch("bits_helpers.build._load_trusted_index", + side_effect=lambda src, *a: parts[src]), \ + mock.patch("bits_helpers.build.warning") as warn: + args = SimpleNamespace(trustManifest="m1,m2,m3", trustGroups=None) + self.assertEqual(build.trusted_reuse_index(args, self.tmp), + {("a", "g"): "sha256:33"}) + self.assertIn("two sha256", warn.call_args[0][0]) def test_build_trusted_reuse_index_missing_manifest_degrades(self): # Default-on safety: an unreachable/absent signed manifest (e.g. an @@ -571,14 +636,14 @@ def test_guessed_absent_manifest_is_not_warned(self): from bits_helpers import build url = "https://s3.cern.ch/swift/v1/b/MANIFESTS/common-manifest-shared.json" arch_url = url.replace("-shared", "-x86_64-el9") - found = lambda src: (("kid", {"h": "sha"}) if src == arch_url else (None, {})) + found = lambda src: (("kid", {("a", "h"): "sha"}) if src == arch_url else (None, {})) with mock.patch("bits_helpers.build._load_trusted_index", side_effect=lambda src, *a: found(src)), \ mock.patch("bits_helpers.build._url_not_found", return_value=True), \ mock.patch("bits_helpers.build.warning") as warn: args = SimpleNamespace(trustManifest=arch_url + "," + url, trustGroups=None, _guessedTrustManifests={arch_url, url}) - self.assertEqual(build.trusted_reuse_index(args, self.tmp), {"h": "sha"}) + self.assertEqual(build.trusted_reuse_index(args, self.tmp), {("a", "h"): "sha"}) warn.assert_not_called() # All guessed manifests absent: one clear warning. args = SimpleNamespace(trustManifest=url, trustGroups=None, @@ -705,7 +770,7 @@ def test_consumer_merges_arch_and_shared(self): trustGroups=None, requireSignedReuse=True) idx = build.trusted_reuse_index(args, self.tmp) # own arch + shared, but NOT the other arch (osx h2). - self.assertEqual(idx, {"h1": "sha256:aa", "h3": "sha256:cc"}) + self.assertEqual(idx, {("slc7_x86-64", "h1"): "sha256:aa", ("shared", "h3"): "sha256:cc"}) def test_freshness_fields_and_expiry_enforced(self): import datetime @@ -713,7 +778,7 @@ def test_freshness_fields_and_expiry_enforced(self): certify.certify([_manifest("b1", [_pkg("A", "h1", "sha256:aa")])], self.key_pem, out, probe=lambda a, h, t=None: "sha256:aa", valid_days=30, source_commit="cafe1234") - doc = json.load(open(out)) + doc = json.loads(Path(out).read_text()) self.assertEqual(doc["source_commit"], "cafe1234") self.assertIn("expires", doc) # Fresh now -> trusted. @@ -730,7 +795,7 @@ def test_no_expiry_by_default_never_expires(self): out = os.path.join(self.tmp, "out", "common.json") certify.certify([_manifest("b1", [_pkg("A", "h1", "sha256:aa")])], self.key_pem, out, probe=lambda a, h, t=None: "sha256:aa") - self.assertNotIn("expires", json.load(open(out))) + self.assertNotIn("expires", json.loads(Path(out).read_text())) import datetime far = datetime.datetime(2999, 1, 1, tzinfo=datetime.timezone.utc) kid, index = trust.trusted_index(out, now=far) # still trusted @@ -741,7 +806,7 @@ def test_certify_group_stamps_untagged_entries(self): certify.certify([_manifest("b1", [_pkg("A", "h1", "sha256:aa")])], self.key_pem, out, probe=lambda a, h, t=None: "sha256:aa", default_group="lcg") - entry = json.load(open(out))["packages"][0] + entry = json.loads(Path(out).read_text())["packages"][0] self.assertEqual(entry["group"], "lcg") # Consumer in a different group won't trust it (not base). _, ship_idx = trust.trusted_index(out, accept_groups=["ship"]) diff --git a/tests/test_checksum_store.py b/tests/test_checksum_store.py index ae582b12..f871ee3e 100644 --- a/tests/test_checksum_store.py +++ b/tests/test_checksum_store.py @@ -3,6 +3,7 @@ """Tests for bits_helpers.checksum_store.""" +from pathlib import Path import os import sys import tempfile @@ -456,7 +457,7 @@ def test_commits_parse_and_roundtrip(self): path = write_checksum_file(self.recipes, "ROOT", store) parsed = parse_checksum_file(path) self.assertEqual(parsed["commits"], {"v6-40-00": GOOD_SHA1, "1.10": GOOD_SHA256}) - self.assertIn('"1.10"', open(path).read()) # quoted: not the float 1.1 + self.assertIn('"1.10"', Path(path).read_text()) # quoted: not the float 1.1 def test_invalid_commit_rejected(self): self._file(self.recipes, "commits:\n v1: nothex\n") @@ -547,7 +548,7 @@ def _update(self, text, new): from bits_helpers.checksum_store import update_checksum_file _write(os.path.join(self.repo, "checksums", "p.checksum"), text) path, _ = update_checksum_file(self.repo, "p", new) - return open(path).read(), parse_checksum_file(path) + return Path(path).read_text(), parse_checksum_file(path) def test_comments_survive_and_entries_land_in_their_section(self): text, parsed = self._update("""\ diff --git a/tests/test_checksums_cmd.py b/tests/test_checksums_cmd.py index aadfad1c..0e848bd5 100644 --- a/tests/test_checksums_cmd.py +++ b/tests/test_checksums_cmd.py @@ -409,6 +409,19 @@ def test_prefetch_resolves_arch_gates_and_only_warms_the_cache(self): "slc9_x86-64-gcc14", "slc9_x86-64") self.assertEqual(calls, [(URL + "linux.tgz", None, self.tmp)]) + def test_prefetch_follows_the_plan(self): + from bits_helpers import build, plan + states = {"a": plan.INSTALLED, "b": plan.LOCAL_TARBALL, "c": plan.REMOTE, + "d": plan.BUILD, "e": plan.DEVEL, "g": plan.SYMLINK} + self.assertEqual(build.prefetch_wanted(list("abcdefg"), states), + [("c", False), ("d", True), ("e", True), ("f", True)]) + self.assertEqual(build.prefetch_wanted(["a"], {}), [("a", True)]) # no plan + spec = {"package": "p", "version": "1", "is_devel_pkg": True, "hash": "ab12", + "sources": [URL + "p.tgz"]} + with patch("bits_helpers.download.download") as download: + build._prefetch_package(spec, None, self.tmp, "slc9_x86-64", sources=False) + download.assert_not_called() + def test_prefetch_survives_a_fatal_url(self): from bits_helpers import build spec = {"package": "p", "version": "1", "is_devel_pkg": True, "hash": "ab12", @@ -427,6 +440,7 @@ def test_readdefaults_records_override_dirs(self): with patch.dict(os.environ, {"BITS_PATH": b}): meta, _ = readDefaults(a, ["release", "dev"], lambda *_: None, "slc9_x86-64") self.assertEqual(meta["_override_dirs"], {"root": a, "root:osx": b, "boost": b}) + self.assertEqual(meta["_version_profiles"], {"boost": "dev"}) # who pins a version def test_build_write_respects_a_legacy_pin(self): from unittest.mock import MagicMock diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 4235b953..c7ffc74b 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -434,5 +434,31 @@ def test_refreshes_mtime(self): "acquire_build_lock should refresh the sentinel mtime") + +class PublishRecordRefreshTest(unittest.TestCase): + """mark_published_from reads the publish record fetched now, not a copy a + previous run left in MANIFESTS/retention.""" + + def test_the_record_is_fetched_afresh(self): + import json + from bits_helpers.cleanup import mark_published_from, published_marker_path + work = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, work, True) + old = os.path.join(work, "MANIFESTS", "retention") + os.makedirs(old) + with open(os.path.join(old, "cvmfs-status.json"), "w") as fh: + json.dump({"packages": []}, fh) + rec = {"packages": [{"package": "zlib", "version": "1.3", "platform": "x86_64-el9", + "pkg_id": "zlib-1.3-2-x86_64-el9"}]} + + def fake(source, destDir, work_dir, dest_filename=None): + with open(os.path.join(destDir, dest_filename), "w") as fh: + json.dump(rec, fh) + return True + with patch("bits_helpers.download.downloadUrllib2", side_effect=fake): + mark_published_from(work, "https://cvmfs.example/cvmfs-status.json") + self.assertTrue(os.path.exists(published_marker_path(work, "x86_64-el9", "zlib", "1.3-2"))) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 6f698554..797326ba 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -37,7 +37,7 @@ def test_DockerRunner(self, mock_getstatusoutput, mock_getoutput): with DockerRunner("") as getstatusoutput_docker: mock_getoutput.assert_not_called() getstatusoutput_docker("echo foo") - mock_getstatusoutput.assert_called_with("/bin/bash -c 'echo foo'", cwd=None) + mock_getstatusoutput.assert_called_with("/bin/bash -c 'echo foo'", cwd=None, env=mock.ANY) mock_getstatusoutput.reset_mock() mock_getstatusoutput.assert_not_called() @@ -69,7 +69,7 @@ def test_DockerRunner_with_env_vars(self, mock_getstatusoutput, mock_getoutput): with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: mock_getoutput.assert_not_called() getstatusoutput_docker("echo test") - mock_getstatusoutput.assert_called_with("env TEST_VAR=test_value ANOTHER_VAR=another_value /bin/bash -c 'echo test'", cwd=None) + mock_getstatusoutput.assert_called_with("env TEST_VAR=test_value ANOTHER_VAR=another_value /bin/bash -c 'echo test'", cwd=None, env=mock.ANY) @mock.patch("bits_helpers.cmd.getoutput") @mock.patch("bits_helpers.cmd.getstatusoutput") @@ -80,7 +80,7 @@ def test_DockerRunner_multiline_env_var(self, mock_getstatusoutput, mock_getoutp with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: mock_getoutput.assert_not_called() getstatusoutput_docker("echo test") - mock_getstatusoutput.assert_called_with("env MULTILINE_VAR='line1\nline2\nline3' /bin/bash -c 'echo test'", cwd=None) + mock_getstatusoutput.assert_called_with("env MULTILINE_VAR='line1\nline2\nline3' /bin/bash -c 'echo test'", cwd=None, env=mock.ANY) @mock.patch("bits_helpers.cmd.getoutput") @mock.patch("bits_helpers.cmd.getstatusoutput") @@ -91,7 +91,7 @@ def test_DockerRunner_env_var_with_semicolon(self, mock_getstatusoutput, mock_ge with DockerRunner("", extra_env=extra_env) as getstatusoutput_docker: mock_getoutput.assert_not_called() getstatusoutput_docker("echo test") - mock_getstatusoutput.assert_called_with("env SEMICOLON_VAR='value1;value2;value3' /bin/bash -c 'echo test'", cwd=None) + mock_getstatusoutput.assert_called_with("env SEMICOLON_VAR='value1;value2;value3' /bin/bash -c 'echo test'", cwd=None, env=mock.ANY) if __name__ == '__main__': diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 3dfcb65b..14e86dcf 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -109,6 +109,14 @@ def get_object(self, Bucket, Key): self.assertEqual(st["certified"], ["Secret 1-1 el9"]) +class _NoSuchKey(Exception): + """What boto3 raises for an absent object, as far as the code looks.""" + + def __init__(self, key): + super().__init__(key) + self.response = {"Error": {"Code": "NoSuchKey"}} + + class EnforceTestCase(unittest.TestCase): """--enforce: purge offending objects, rewrite BOMs, delete emptied ones.""" @@ -153,7 +161,9 @@ def paginate(Bucket, Prefix=""): def get_object(self, Bucket, Key): import io - body = outer.put.get(Key, objects[Key]) + if Key not in outer.put and Key not in objects: + raise _NoSuchKey(Key) + body = outer.put.get(Key, objects.get(Key)) return {"Body": io.BytesIO(body.encode())} def delete_object(self, Bucket, Key): diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index c3d63e7b..45856296 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -3,6 +3,7 @@ """Tests for bits_helpers/cvmfs_import.""" +from pathlib import Path import json import os import tempfile @@ -515,14 +516,14 @@ def test_layout_and_alias_remap(self): # modulefile present at name/version mf = os.path.join(arch_root, "root", "6.38.00") self.assertTrue(os.path.isfile(mf)) - text = open(mf).read() + text = Path(mf).read_text() self.assertIn('module-whatis "build_id: %s"' % bid, text) # dep edge remapped to bits name in the prereq self.assertIn("prereq python/3.13.11", text) # modulefile is the single env artifact — no init.sh sidecar self.assertFalse(os.path.exists(os.path.join(arch_root, "root", ".6.38.00.init.sh"))) self.assertIn("setenv ROOT_ROOT /cvmfs/x/ROOT/6.38.00", text) - meta = json.load(open(os.path.join(arch_root, "root", ".6.38.00.meta.json"))) + meta = json.loads(Path(os.path.join(arch_root, "root", ".6.38.00.meta.json")).read_text()) self.assertEqual(meta["build_id"], bid) self.assertEqual(meta["abi_tag"], "x86-64-gcc13") # one nested catalog per build_id diff --git a/tests/test_cvmfs_install_dirs.py b/tests/test_cvmfs_install_dirs.py new file mode 100644 index 00000000..f2d535f0 --- /dev/null +++ b/tests/test_cvmfs_install_dirs.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""bits q / enter / setenv on CVMFS trees laid out by install dir, as ALICE's +(/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles): a cvmfs.yaml whose +cvmfs_modules_template has {install_dir} in place of {arch}. The install dir +is the one published with the arch's (or the host's) CPU and OS; retired +modulefiles in Modules/archive are searched only with --archive. +""" + +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tests")) + +from test_cvmfs_modules import _host_os # noqa: E402 +from test_release_view import _modulecmd # noqa: E402 + +# As on /cvmfs/alice.cern.ch, with old names that carry no OS. +DIRS = ("el8-x86_64", "el9-x86_64", "el10-x86_64", "el9-aarch64", "ubuntu2604-riscv64", + "x86_64-2.6-gnu-4.8.3", "noarch") + +MODULE = """#%%Module1.0 +set version %s +module load BASE/1.0%s +setenv %s_ROOT $::env(BASEDIR)/%s/$version +""" + + +def _write(path, text, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + os.chmod(path, mode) + + +class InstallDirTreesTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.inst = os.path.join(self.tmp, "inst") + os.makedirs(self.inst) + for f in ("bits", "bitsBuild"): + shutil.copy2(os.path.join(ROOT, f), self.inst) + self.calls = os.path.join(self.tmp, "calls") + self.stubs = os.path.join(self.tmp, "stubs") + _write(os.path.join(self.stubs, "python3"), + '#!/bin/sh\necho "python3 $*" >> "%s"\nexit 1\n' % self.calls, 0o755) + _write(os.path.join(self.stubs, "modulecmd"), + '#!/bin/sh\necho "modulecmd MODULEPATH=$MODULEPATH BASEDIR=$BASEDIR $*" >> "%s"\n' + % self.calls, 0o755) + self.cvmfs = os.path.join(self.tmp, "cvmfs", "alice") + for d in DIRS: + self._module(d, "P/%s-1" % d.replace("-", "_")) + # Retired, and needing a module that is not. + self._module("el9-x86_64", "Q/1-1") + self._module("el9-x86_64", "P/v0-1", archive=True, load="Q/1-1") + self.job = os.path.join(self.tmp, "job") + _write(os.path.join(self.job, "cvmfs.yaml"), + 'prefix: "%s"\ncvmfs_modules_template: "{prefix}/{install_dir}/Modules/modulefiles/{pkg}"\n' + % self.cvmfs) + self.env = {"HOME": self.tmp, "PATH": self.stubs + ":/usr/bin:/bin", "MODULES_SHELL": "bash"} + + def _module(self, d, name, archive=False, load=None): + mods = os.path.join(self.cvmfs, d, "Modules") + _write(os.path.join(mods, "modulefiles", "BASE", "1.0"), + "#%%Module1.0\nsetenv BASEDIR %s\n" % os.path.join(self.cvmfs, d, "Packages")) + _write(os.path.join(mods, "archive" if archive else "modulefiles", name), + MODULE % (name.split("/")[1], "\nmodule load %s" % load if load else "", + name.split("/")[0], name.split("/")[0])) + + def _bits(self, *args): + return subprocess.run([os.path.join(self.inst, "bits")] + list(args), cwd=self.job, + env=self.env, capture_output=True, text=True) + + def _calls(self): + try: + with open(self.calls) as f: + return f.read().splitlines() + except OSError: + return [] + + def _q(self, *args): + r = self._bits(*args, "q") + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout.split() + + def test_the_install_dir_of_the_arch(self): + el9 = ["P/el9_x86_64-1", "Q/1-1"] + self.assertEqual(sorted(self._q("-a", "slc9_x86-64")), el9) + self.assertEqual(sorted(self._q("-a", "slc9-gpu_x86-64")), el9) + self.assertEqual(sorted(self._q("-a", "x86_64-el9-gcc14-opt")), el9) + self.assertEqual(self._q("-a", "slc8_x86-64"), ["P/el8_x86_64-1"]) + self.assertEqual(self._q("-a", "slc10_x86-64"), ["P/el10_x86_64-1"]) + self.assertEqual(self._q("-a", "slc9_aarch64"), ["P/el9_aarch64-1"]) + self.assertEqual(self._q("-a", "ubuntu2604_riscv64"), ["P/ubuntu2604_riscv64-1"]) + # -a may name the install dir itself, also one without an OS. + self.assertEqual(self._q("-a", "el8-x86_64"), ["P/el8_x86_64-1"]) + self.assertEqual(self._q("-a", "x86_64-2.6-gnu-4.8.3"), ["P/x86_64_2.6_gnu_4.8.3-1"]) + self.assertEqual(self._calls(), []) + + def test_an_installed_arch_gives_its_install_dir(self): + _write(os.path.join(self.job, "sw", "slc9_x86-64", "A", "1-1", "etc", "modulefiles", "A"), + "#%Module1.0\n") + self.assertEqual(sorted(self._q()), ["A/1-1", "P/el9_x86_64-1", "Q/1-1"]) + self.assertEqual(self._calls(), []) + + def test_several_for_the_cpu_and_os(self): + self._module("el9-x86_64-cuda", "P/cuda-1") + r = self._bits("-a", "slc9_x86-64", "q") + self.assertEqual((r.returncode, r.stdout), (0, "")) + self.assertIn("Choose one: bits -a q", r.stderr) + self.assertIn("/{install_dir}/Modules/modulefiles", r.stderr) + self.assertEqual(self._q("-a", "el9-x86_64-cuda"), ["P/cuda-1"]) + + def test_a_template_without_pkg_gives_no_trees(self): + _write(os.path.join(self.job, "cvmfs.yaml"), + 'prefix: "%s"\ncvmfs_modules_template: "{prefix}/{install_dir}/Modules/modulefiles"\n' + % self.cvmfs) + r = self._bits("-a", "slc9_x86-64", "q") + self.assertNotEqual(r.returncode, 0) + self.assertIn("no work dir", r.stdout + r.stderr) + + def test_no_install_dir_for_the_arch(self): + for arch in ("slc1_x86-64", "slc7_x86-64", "ubuntu2404_x86-64", "x86_64-gnu"): + r = self._bits("-a", arch, "q") + self.assertEqual((r.returncode, r.stdout), (0, ""), arch) + self.assertIn("NOTE: no CVMFS module tree", r.stderr, arch) + + def test_the_hosts_install_dir(self): + os_ = _host_os() + if not os_: + self.skipTest("bits picks the host's on el and Ubuntu hosts only") + self._module("%s-%s" % (os_, platform.machine()), "P/host-1") + self.assertEqual(self._q(), ["P/host-1"]) + self.assertEqual(self._calls(), []) + + def test_the_archive_only_with_archive(self): + for opt in ("--archive", "-archive"): + self.assertEqual(sorted(self._q("-a", "slc9_x86-64", opt)), ["P/el9_x86_64-1", "P/v0-1", "Q/1-1"]) + r = self._bits("-a", "slc9_x86-64", "printenv", "P/v0-1") + self.assertEqual(r.returncode, 1) + self.assertIn("P/v0-1 was not found", r.stderr) + + def test_the_archives_of_arch_trees(self): + # BITS_CVMFS_PREFIX trees: the archive of the second only. + prefix = os.path.join(self.tmp, "cvmfs", "bits") + for arch, name, archive in (("A-opt", "X/1-1", False), ("A", "T/1-1", False), + ("A", "T/0-1", True)): + _write(os.path.join(prefix, arch, "Modules", "archive" if archive else "modulefiles", name), + "#%Module1.0\n") + self.env["BITS_CVMFS_PREFIX"] = prefix + self.assertEqual(sorted(self._q("-a", "A-opt")), ["T/1-1", "X/1-1"]) + self.assertEqual(sorted(self._q("-a", "A-opt", "--archive")), ["T/0-1", "T/1-1", "X/1-1"]) + self._bits("-a", "A-opt", "--archive", "printenv", "T/0-1") + t = os.path.join(prefix, "A", "Modules") + self.assertIn("MODULEPATH=%s/modulefiles:%s/archive " % (t, t), self._calls()[0]) + + def test_an_archived_module_uses_its_trees_packages(self): + tree = os.path.join(self.cvmfs, "el9-x86_64", "Modules") + want = "MODULEPATH=%s/modulefiles:%s/archive BASEDIR=%s" % ( + tree, tree, os.path.join(self.cvmfs, "el9-x86_64", "Packages")) + self._bits("-a", "slc9_x86-64", "--archive", "printenv", "P/v0-1") + self.assertTrue(self._calls()[0].startswith("modulecmd " + want), self._calls()) + # unload finds it where it was loaded from, without --archive. + os.unlink(self.calls) + self.env.update(LOADEDMODULES="BASE/1.0:P/v0-1", + _LMFILES_="%s/modulefiles/BASE/1.0:%s/archive/P/v0-1" % (tree, tree)) + self._bits("-a", "slc9_x86-64", "unload", "P/v0-1") + self.assertTrue(self._calls()[0].startswith("modulecmd " + want), self._calls()) + + @unittest.skipUnless(_modulecmd(), "needs modulecmd (Environment Modules)") + def test_setenv_with_modulecmd(self): + os.unlink(os.path.join(self.stubs, "modulecmd")) + os.symlink(_modulecmd(), os.path.join(self.stubs, "modulecmd")) + pkgs = os.path.join(self.cvmfs, "el9-x86_64", "Packages") + for args, want in ((("P/el9_x86_64-1",), ["P/el9_x86_64-1", ""]), + (("--archive", "P/v0-1"), ["P/v0-1", "Q/1-1"])): + r = self._bits("-a", "slc9_x86-64", "setenv", *args, "-c", "sh", "-c", + 'echo "$P_ROOT|${Q_ROOT-}"') + self.assertEqual(r.returncode, 0, (args, r.stderr)) + self.assertEqual(r.stdout.strip().split("|"), + [os.path.join(pkgs, w) if w else "" for w in want], args) + self.assertEqual(self._calls(), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cvmfs_modules.py b/tests/test_cvmfs_modules.py index a76554a3..23f4d37e 100644 --- a/tests/test_cvmfs_modules.py +++ b/tests/test_cvmfs_modules.py @@ -3,8 +3,8 @@ """bits q/printenv/setenv with a community's CVMFS module trees ($BITS_CVMFS_PREFIX). -The trees of the arch that match up to -opt/-dbg are listed and loaded after the -local ones; each module is loaded against its own tree, with that tree's BASEDIR. +The trees of the arch and of the arch without its build type (the toolchain) are +listed and loaded after the local ones, never the other build type; each module is loaded against its own tree, with that tree's BASEDIR. Needs a real modulecmd (Environment Modules): on PATH, or $BITS_TEST_MODULECMD. """ @@ -82,7 +82,8 @@ def setUp(self): trees = {ARCH: {"A/1-1": (), "B/2-1": ("E/5-1",), "E/5-1": ()}, "x86_64-el9-gcc14": {"gcc/14-1": ()}, "x86_64-el9-gcc14-dbg": {"C/3-1": ()}, - "x86_64-el9-gcc15-opt": {"D/1-1": ()}} + "x86_64-el9-gcc15-opt": {"D/1-1": ()}, + "x86_64-el9-gcc13": {"S/1-1": ()}} # published without a build type (ship) for arch, mods in trees.items(): mroot = os.path.join(self.cvmfs, arch, "Modules", "modulefiles") _write(os.path.join(mroot, "BASE", "1.0"), _CVMFS_BASE) @@ -119,7 +120,7 @@ def _printenv(self, *mods, **env): def test_q_lists_local_then_matching_trees_once(self): r = self._bits("q") self.assertEqual(r.returncode, 0, r.stderr) - self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "C/3-1", "E/5-1", "gcc/14-1"]) + self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"]) def test_options_after_the_command(self): # bits q -a ARCH: -a after the command, and an argument after it. @@ -134,11 +135,11 @@ def test_catalog_listing_only_on_request(self): with open(os.path.join(self.inst, "bitsModules"), "w") as f: f.write("#!/bin/sh\ntouch %s\nexit 3\n" % mark) os.chmod(os.path.join(self.inst, "bitsModules"), 0o755) - self.assertEqual(self._bits("q").stdout.split(), ["A/1-1", "B/2-1", "C/3-1", "E/5-1", "gcc/14-1"]) + self.assertEqual(self._bits("q").stdout.split(), ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"]) self.assertFalse(os.path.exists(mark)) self.assertEqual(self._bits("q", BITS_CATALOG_LISTING="1", PATH=self.inst + ":" + self.env["PATH"]).stdout.split(), - ["A/1-1", "B/2-1", "C/3-1", "E/5-1", "gcc/14-1"]) + ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"]) self.assertTrue(os.path.exists(mark)) def test_q_without_prefix_is_local_only(self): @@ -146,15 +147,30 @@ def test_q_without_prefix_is_local_only(self): self.assertEqual(r.stdout.split(), ["A/1-1"]) def test_each_module_from_its_own_tree(self): - roots = self._printenv("A/1-1", "B/2-1", "gcc/14-1", "C/3-1") + roots = self._printenv("A/1-1", "B/2-1", "gcc/14-1") self.assertEqual(roots["A"], os.path.join(self.sw, ARCH, "A", "1-1")) # local wins opt = os.path.join(self.cvmfs, ARCH, "Packages") self.assertEqual(roots["B"], os.path.join(opt, "B", "2-1")) self.assertEqual(roots["E"], os.path.join(opt, "E", "5-1")) # B's dep, same tree self.assertEqual(roots["GCC"], os.path.join(self.cvmfs, "x86_64-el9-gcc14", "Packages", "gcc", "14-1")) - self.assertEqual(roots["C"], - os.path.join(self.cvmfs, "x86_64-el9-gcc14-dbg", "Packages", "C", "3-1")) + + def test_build_types(self): + # -opt never loads -dbg (bits q notes it); -dbg gets its own tree and the + # toolchain; no build type is -opt; an arch published without one is found. + r = self._bits("q") + self.assertIn("also published: x86_64-el9-gcc14-dbg", r.stderr) + r = subprocess.run([os.path.join(self.inst, "bits"), "-a", "x86_64-el9-gcc14-dbg", "q"], + cwd=self.work, env=self.env, capture_output=True, text=True) + self.assertEqual(r.stdout.split(), ["C/3-1", "gcc/14-1"], r.stderr) + self.assertIn("also published: x86_64-el9-gcc14-opt", r.stderr) + r = subprocess.run([os.path.join(self.inst, "bits"), "-a", "x86_64-el9-gcc14", "q"], + cwd=self.work, env=self.env, capture_output=True, text=True) + self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"], r.stderr) + for arch in ("x86_64-el9-gcc13-opt", "x86_64-el9-gcc13-dbg", "x86_64-el9-gcc13"): + r = subprocess.run([os.path.join(self.inst, "bits"), "-a", arch, "q"], + cwd=self.work, env=self.env, capture_output=True, text=True) + self.assertEqual(r.stdout.split(), ["S/1-1"], (arch, r.stderr)) def test_cvmfs_only(self): roots = self._printenv("B") @@ -218,7 +234,7 @@ def test_layout_file(self): self._layout() r = self._bits("q") self.assertEqual(r.returncode, 0, r.stderr) - self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "C/3-1", "E/5-1", "gcc/14-1"]) + self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"]) self.assertEqual(self._printenv("B")["B"], os.path.join(self.cvmfs, ARCH, "Packages", "B", "2-1")) def test_layout_file_beside_the_work_dir(self): @@ -246,7 +262,7 @@ def test_layout_file_without_a_uses_the_local_arch(self): self._layout() r = subprocess.run([os.path.join(self.inst, "bits"), "q"], cwd=self.work, env=self.env, capture_output=True, text=True) - self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "C/3-1", "E/5-1", "gcc/14-1"], r.stderr) + self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "E/5-1", "gcc/14-1"], r.stderr) def test_layout_file_picks_this_hosts_arch(self): # No -a and no tree for the local arch: the trees published for this @@ -264,12 +280,13 @@ def test_layout_file_picks_this_hosts_arch(self): q = lambda: subprocess.run([os.path.join(self.inst, "bits"), "q"], cwd=self.work, env=self.env, capture_output=True, text=True) r = q() - self.assertEqual(r.stdout.split(), ["B/2-1", "gcc/14-1"], r.stderr) + # A, built here for the local arch (no -a: the one installed), then the host's. + self.assertEqual(r.stdout.split(), ["A/1-1", "B/2-1", "gcc/14-1"], r.stderr) # Two compilers for this host: none is picked, the note lists them. _write(os.path.join(root, "%s-%s-gcc15-opt" % (cpu, osname), "Modules", "modulefiles", "BASE", "1.0"), _CVMFS_BASE) r = q() - self.assertEqual(r.stdout.split(), []) + self.assertEqual(r.stdout.split(), ["A/1-1"]) self.assertIn("%s-%s-gcc15-opt" % (cpu, osname), r.stderr) self.assertIn("bits -a q", r.stderr) diff --git a/tests/test_cvmfs_publish.py b/tests/test_cvmfs_publish.py index 3d95b37b..6b13f33c 100644 --- a/tests/test_cvmfs_publish.py +++ b/tests/test_cvmfs_publish.py @@ -6,6 +6,7 @@ carry hard-won fixes (template expansion, the reused-artefact re-root, the INSTALLROOT symlink relativiser, sanitize). The stage/submit/pipeline parts need a build host and are proven separately via the remote-runner.""" +from pathlib import Path import os import shutil import tempfile @@ -266,7 +267,7 @@ def test_empty_or_corrupt_sentinel_falls_back_not_raises(self): fh.write(b"x" * sz) for p, body in (("big", ""), ("small", "not-a-number\n")): # blank + garbage d = os.path.join(work, ".packages", "el9", p); os.makedirs(d, exist_ok=True) - open(os.path.join(d, "1.0-1"), "w").write(body) + Path(os.path.join(d, "1.0-1")).write_text(body) self.addCleanup(os.environ.pop, "BITS_WORK_DIR", None) os.environ["BITS_WORK_DIR"] = work specs = [{"package": p, "version": "1.0", "revision": "1"} for p in ("small", "big")] @@ -740,13 +741,14 @@ def test_view_tar_has_relative_links(self): self.assertEqual(os.path.normpath(os.path.join( "g/releases/LCG_110/ROOT/6.36", link.linkname)), "g/el9-gcc15-opt/ROOT/6.36-2") - def _main(self, fake, extra=(), templates=None, base_state={"exists": True}): + def _main(self, fake, extra=(), templates=None, base_state={"exists": True}, manifest=None): import io, contextlib from unittest import mock import bits_helpers.cvmfs_publish as cp m = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) - json.dump({"cvmfs_templates": templates or self.TM, "architecture": "el9-gcc15-opt", - "packages": [self.SPEC, dict(self.SPEC, package="six", hash="h2")]}, m) + json.dump(dict({"cvmfs_templates": templates or self.TM, "architecture": "el9-gcc15-opt", + "packages": [self.SPEC, dict(self.SPEC, package="six", hash="h2")]}, + **(manifest or {})), m) m.close(); self.addCleanup(os.unlink, m.name) sent = [] out, err = io.StringIO(), io.StringIO() @@ -926,6 +928,30 @@ def test_no_view_without_both_templates_or_a_release(self): self.assertIn("no release: ", err) self.assertIn(why, err) + def test_no_release_for_a_toolchain_build(self): + """The toolchain (own_hash) lives outside the build arch: a build of it + alone makes no release links and no merged view, and does not fail.""" + fake = lambda s, c: [("J", "x(pkg)")] + tm = dict(self.TM, views="{prefix}/views/{release}/{arch}") + gcc = dict(self.SPEC, package="GCC-Toolchain", effective_architecture="el9-gcc15") + rc, sent, _, err = self._main(fake, ["--release-view"], templates=tm, manifest={ + "packages": [gcc], "requested_packages": ["GCC-Toolchain"]}) + self.assertEqual((rc, sent), (0, [])) + self.assertIn("no release: its top-level packages (GCC-Toolchain) live outside " + "el9-gcc15-opt", err) + # A noarch top level is a release of the build arch. + six = dict(self.SPEC, package="six", effective_architecture="share") + rc, sent, _, err = self._main(fake, ["--release-view"], manifest={ + "packages": [six], "requested_packages": ["six"]}, base_state={"exists": False}) + self.assertNotIn("no release", err) + # With a top-level package of the build arch the release is made. + rc, sent, _, err = self._main(fake, ["--release-view"], manifest={ + "packages": [gcc, self.SPEC], "requested_packages": ["ROOT", "GCC-Toolchain"]}, + base_state={"exists": False}) + self.assertEqual(rc, 0) + self.assertNotIn("no release", err) + self.assertTrue(any("release-view@" in lbl for _p, lbl, _h in sent)) + def test_view_without_release_links(self): """The merged view links into the packages, so a group with a views template but no releases template still gets one.""" @@ -1150,7 +1176,7 @@ def test_merged_view_folds_directories_one_package_fills(self): "share/root"): self.assertFalse(os.path.islink(os.path.join(staging, d)), d) cp.write_view_setup(staging, "/cvmfs/r/" + vp) # still finds what it looks for - setup = open(os.path.join(staging, "setup.sh")).read() + setup = Path(os.path.join(staging, "setup.sh")).read_text() self.assertIn("lib/python3.12/site-packages", setup) self.assertIn("lib/pkgconfig", setup) @@ -1165,8 +1191,8 @@ def test_merged_view_keeps_share_man_for_manpath(self): cp.merged_view(ctx, [self.SPEC], staging, vp) self.assertFalse(os.path.islink(os.path.join(staging, "share", "man"))) cp.write_view_setup(staging, "/cvmfs/r/" + vp) - self.assertIn('/share/man:$MANPATH"', open(os.path.join(staging, "setup.sh")).read()) - csh = open(os.path.join(staging, "setup.csh")).read() + self.assertIn('/share/man:$MANPATH"', Path(os.path.join(staging, "setup.sh")).read_text()) + csh = Path(os.path.join(staging, "setup.csh")).read_text() self.assertIn('if ($?MANPATH) then\n setenv MANPATH', csh) if shutil.which("tcsh"): # unset MANPATH must not break sourcing it import subprocess @@ -1267,7 +1293,7 @@ def test_view_setup_locates_itself(self): out = subprocess.run(["dash", "-c", '. "%s/setup.sh" 2>/dev/null; echo "$BITS_VIEW"' % d], capture_output=True, text=True).stdout.strip() self.assertIn(out, ("/cvmfs/r/g/views/X/a", "")) - self.assertIn('set _v = "/cvmfs/r/g/views/X/a"', open(os.path.join(d, "setup.csh")).read()) + self.assertIn('set _v = "/cvmfs/r/g/views/X/a"', Path(os.path.join(d, "setup.csh")).read_text()) if shutil.which("tcsh"): out = subprocess.run(["tcsh", "-f", "-c", 'source "%s/setup.csh"; echo $BITS_VIEW' % d], capture_output=True, text=True).stdout.strip() @@ -1288,7 +1314,7 @@ def test_view_setup_adds_man_pages_only_to_a_set_manpath(self): 'echo "${MANPATH-}"' % (before, d)], capture_output=True, text=True).stdout.strip() self.assertEqual(out, want) - self.assertIn("if ($?MANPATH)", open(os.path.join(d, "setup.csh")).read()) + self.assertIn("if ($?MANPATH)", Path(os.path.join(d, "setup.csh")).read_text()) def test_merged_view_published_once(self): import tarfile @@ -1513,6 +1539,193 @@ def test_base_module_published_when_missing(self): self.assertEqual(sent, [("g/el9-gcc15-opt/Modules/BASE", "BASE@g/el9-gcc15-opt/Modules(modules)", "ingest")]) + ROOT_MF = """#%Module1.0 +set version @@PKGREVISION@h1@@ +if ![ is-loaded "zlib/1.3-1" ] { module load zlib/1.3-1 } +set PKG_ROOT $::env(BASEDIR)/ROOT/6.36-$version +if {[file isdirectory $PKG_ROOT/bin]} { prepend-path PATH $PKG_ROOT/bin } +prepend-path CMAKE_PREFIX_PATH $PKG_ROOT +setenv ROOTSYS $PKG_ROOT +prepend-path ROOT_PLUGIN_PATH $PKG_ROOT/etc/plugins +setenv BITS_VIEW 1 +""" + ZLIB_MF = """#%Module1.0 +set PKG_ROOT $::env(BASEDIR)/zlib/1.3-1 +if {[file isdirectory $PKG_ROOT/lib]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib } +setenv ZLIB_ROOT $PKG_ROOT +""" + + def _release(self, ctx): + """zlib and ROOT, whose modulefile loads zlib; ROOT's tarball still has + the revision placeholder, as bits packs it.""" + zlib = dict(self.SPEC, package="zlib", version="1.3", revision="1", hash="z") + root = dict(self.SPEC) + self._tarball(ctx, zlib, ["lib/libz.so", "etc/modulefiles/zlib"]) + self._tarball(ctx, root, ["bin/root", "etc/plugins/p", "etc/modulefiles/ROOT"]) + for spec, text in ((zlib, self.ZLIB_MF), (root, self.ROOT_MF)): + self._retar(ctx, spec, "etc/modulefiles/" + spec["package"], text) + return [zlib, root] + + def _retar(self, ctx, spec, rel, text): + """Put *text* into *rel* of the package's tarball.""" + import subprocess + tar = cp_tar_path(spec, ctx) + src = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, src, True) + subprocess.run(["tar", "-xzf", tar, "-C", src], check=True) + top = os.path.join(src, spec["effective_architecture"], spec["package"], + spec["version"] + "-" + spec["revision"]) + with open(os.path.join(top, rel), "w") as fh: + fh.write(text) + subprocess.run(["tar", "-czf", tar, "-C", src, spec["effective_architecture"]], + check=True) + + def test_view_setup_sets_what_the_modules_set(self): + import subprocess + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm) + specs = self._release(ctx) + vp = "g/views/LCG_110/el9-gcc15-opt" + env, problems = cp.view_environment(ctx, specs, vp, ["ROOT"]) + self.assertEqual(problems, []) + # The published tree: the view and the packages beside it. + repo = os.path.realpath(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, repo, True) + view = os.path.join(repo, vp) + cp.merged_view(ctx, specs, view, vp) + for spec in specs: + os.makedirs(os.path.join(repo, cp.package_path(spec, ctx), "bin")) + cp.write_view_setup(view, "/cvmfs/r/" + vp, env) + out = subprocess.run(["bash", "-c", 'cd / && PATH=/usr/bin:/bin; unset LD_LIBRARY_PATH ' + 'CMAKE_PREFIX_PATH ROOT_PLUGIN_PATH; . "%s/setup.sh" && ' + 'echo "$PATH|$LD_LIBRARY_PATH|$CMAKE_PREFIX_PATH|$ROOTSYS|' + '$ZLIB_ROOT|$ROOT_PLUGIN_PATH|${BITS_VIEW}|${_b-unset}"' % view], + capture_output=True, text=True).stdout.strip() + pkgs = os.path.join(repo, "g/el9-gcc15-opt") + self.assertEqual(out.split("|"), [ + view + "/bin:/usr/bin:/bin", view + "/lib", view, + pkgs + "/ROOT/6.36-2", pkgs + "/zlib/1.3-1", + pkgs + "/ROOT/6.36-2/etc/plugins", # not in the view: the package's + view, "unset"]) + csh = Path(os.path.join(view, "setup.csh")).read_text() + self.assertIn('set _b = "/cvmfs/r/g/el9-gcc15-opt"', csh) + self.assertIn('setenv ROOTSYS "$_b/ROOT/6.36-2"', csh) + + def test_view_setup_guards_follow_links_as_the_file_system(self): + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm) + zlib, root = self._release(ctx) + self._tarball(ctx, root, ["bin/root", "lib/pkgconfig/root.pc", "etc/modulefiles/ROOT"], + links=[("lib64", "lib"), ("data", "/opt/data")]) + guard = "if {[file isdirectory $PKG_ROOT/%s]} { prepend-path %s $PKG_ROOT/%s }\n" + self._retar(ctx, root, "etc/modulefiles/ROOT", self.ROOT_MF + + guard % ("lib64/pkgconfig", "PKG_CONFIG_PATH", "lib64/pkgconfig")) + env, problems = cp.view_environment(ctx, [zlib, root], "g/views/LCG_110/el9-gcc15-opt", + ["ROOT"]) + self.assertEqual(problems, []) + self.assertIn("@V@/lib64/pkgconfig", env["paths"]["PKG_CONFIG_PATH"]) + # Below a link out of the package: modulecmd follows it, the listing cannot. + self._retar(ctx, root, "etc/modulefiles/ROOT", self.ROOT_MF + + guard % ("data/x", "DATA_PATH", "data/x")) + _env, problems = cp.view_environment(ctx, [zlib, root], "g/views/LCG_110/el9-gcc15-opt", + ["ROOT"]) + self.assertEqual(problems, ["ROOT: data is a link out of the package"]) + + def test_view_setup_problems_instead_of_failing(self): + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm) + zlib, root = self._release(ctx) + vp = "g/views/LCG_110/el9-gcc15-opt" + # A guarded directory that is a link out of the package: modulecmd + # follows it, the listing cannot. + self._tarball(ctx, root, ["bin/root", "etc/modulefiles/ROOT"], + links=[("lib64", "/opt/elsewhere/lib")]) + self._retar(ctx, root, "etc/modulefiles/ROOT", self.ROOT_MF + + "if {[file isdirectory $PKG_ROOT/lib64]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib64 }\n") + _env, problems = cp.view_environment(ctx, [zlib, root], vp, ["ROOT"]) + self.assertEqual(problems, ["ROOT: lib64 is a link out of the package"]) + # A package whose tarball is not there: a problem, not a failed view. + os.unlink(cp_tar_path(zlib, ctx)) + _env, problems = cp.view_environment(ctx, [zlib, root], vp, ["ROOT"]) + self.assertTrue(problems[0].startswith("zlib: cannot read its modulefile: "), problems) + + def test_view_records_whether_setup_sets_what_the_modules_set(self): + import tarfile + from unittest import mock + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm, publish_path="ingest") + specs = self._release(ctx) + metas, pointers, lines = [], [], [] + + def pub(c, p, t, lbl, **k): + with tarfile.open(t) as tf: + if lbl.startswith("view-pointer@"): + pointers.append((p, {m.name: m.linkname for m in tf if m.issym()})) + else: + names = tf.getnames() + metas.append((json.load(tf.extractfile("./.meta.json"))["setup"], + [n for n in names if ".setup-complete" in n])) + return "J" + with mock.patch.object(cp, "published_state", lambda c, p: {"exists": False}), \ + mock.patch.object(cp, "_publish_tar", pub): + self.assertEqual(cp._publish_merged_view(ctx, specs, lines.extend, ["ROOT"]), 0) + self._retar(ctx, specs[0], "etc/modulefiles/zlib", self.ZLIB_MF + "puts hi\n") + self.assertEqual(cp._publish_merged_view(ctx, specs, lines.extend, ["ROOT"]), 0) + self.assertEqual(metas[0], ({"modules": True, "problems": []}, + ["./.setup-complete.ROOT.6.36-2"])) + self.assertEqual(metas[1][0]["modules"], False) + self.assertIn("zlib: unknown command: puts hi", metas[1][0]["problems"]) + self.assertEqual(metas[1][1], []) + # Only the complete view gets a pointer from its top-level module, to + # that module's own marker: a view made for another one leaves it dangling. + self.assertEqual(pointers, [("g/el9-gcc15-opt/Modules", { + "ROOT/.6.36-2.view": + "../../../views/LCG_110/el9-gcc15-opt/.setup-complete.ROOT.6.36-2"})]) + self.assertEqual(lines, ["PUBLISHED J merged-view@g/views/LCG_110/el9-gcc15-opt", + "PUBLISHED J view-pointer@g/el9-gcc15-opt/Modules", + "PUBLISHED J merged-view@g/views/LCG_110/el9-gcc15-opt"]) + + def test_view_pointer_is_added_once_also_to_an_unchanged_view(self): + from unittest import mock + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm, publish_path="ingest") + specs = self._release(ctx) + sent, fps = [], {} + + def pub(c, p, t, lbl, identity_hash=None, **k): + sent.append(lbl) + fps[p] = identity_hash + return "J" + with mock.patch.object(cp, "published_state", lambda c, p: {"exists": False}), \ + mock.patch.object(cp, "_publish_tar", pub): + cp._publish_merged_view(ctx, specs, [].extend, ["ROOT"]) + vp = "g/views/LCG_110/el9-gcc15-opt" + for pointer_there in (False, True): + sent.clear() + state = lambda c, p: ({"exists": True, "hash": fps[vp]} if p == vp else + {"exists": pointer_there}) + with mock.patch.object(cp, "published_state", state), \ + mock.patch.object(cp, "_publish_tar", pub): + self.assertEqual(cp._publish_merged_view(ctx, specs, [].extend, ["ROOT"]), 0) + self.assertEqual(sent, [] if pointer_there else + ["view-pointer@g/el9-gcc15-opt/Modules"]) + + def test_no_view_pointer_for_several_top_level_modules(self): + from unittest import mock + import bits_helpers.cvmfs_publish as cp + tm = dict(self.TM, views="{prefix}/views/LCG_110/{arch}") + ctx = self._ctx(templates=tm, publish_path="ingest") + specs = self._release(ctx) + sent = [] + # setup.sh would set what both set, more than either one alone does. + with mock.patch.object(cp, "published_state", lambda c, p: {"exists": False}), \ + mock.patch.object(cp, "_publish_tar", lambda c, p, t, lbl, **k: sent.append(lbl)): + self.assertEqual(cp._publish_merged_view(ctx, specs, [].extend, ["ROOT", "zlib"]), 0) + self.assertEqual(sent, ["merged-view@g/views/LCG_110/el9-gcc15-opt"]) + def _boom(*a, **k): raise AssertionError("must not be called") @@ -1524,7 +1737,7 @@ def test_posts_tar_with_sha256_and_signs_ingest_path(self): import bits_helpers.cvmfs_publish as cp import bits_helpers.prepub as pp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"hello-tar-bytes") + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"hello-tar-bytes") want = hashlib.sha256(b"hello-tar-bytes").hexdigest() cap = {} @@ -1533,8 +1746,8 @@ class Resp: def json(self): return {"job_id": "JID-1"} class Sess: - def post(self, url, files=None, headers=None, timeout=None): - cap["files"] = files; return Resp() + def post(self, url, data=None, headers=None, timeout=None): + cap["files"] = data.fields; return Resp() def fake_auth(token, method, uri, fields=None, body_hash=None, bearer_auth=False, no_verify_tls=False): @@ -1564,7 +1777,7 @@ def test_direct_s3_sent_and_signed_when_enabled(self): import bits_helpers.cvmfs_publish as cp import bits_helpers.prepub as pp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"z") + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"z") cap = {} class Resp: @@ -1572,8 +1785,8 @@ class Resp: def json(self): return {"job_id": "J"} class Sess: - def post(self, url, files=None, headers=None, timeout=None): - cap["files"] = files; return Resp() + def post(self, url, data=None, headers=None, timeout=None): + cap["files"] = data.fields; return Resp() def fake_auth(token, method, uri, fields=None, body_hash=None, bearer_auth=False, no_verify_tls=False): @@ -1593,7 +1806,7 @@ def test_object_list_and_prewarm_sent_and_signed(self): import bits_helpers.cvmfs_publish as cp import bits_helpers.prepub as pp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"z") + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"z") cap = {} class Resp: @@ -1601,8 +1814,8 @@ class Resp: def json(self): return {"job_id": "J"} class Sess: - def post(self, url, files=None, headers=None, timeout=None): - cap["files"] = files; return Resp() + def post(self, url, data=None, headers=None, timeout=None): + cap["files"] = data.fields; return Resp() def fake_auth(token, method, uri, fields=None, body_hash=None, bearer_auth=False, no_verify_tls=False): @@ -1638,7 +1851,7 @@ def json(self): return {"job_id": "J"} class Sess: def get(self, url, timeout=None): return Health() - def post(self, url, files=None, headers=None, timeout=None): + def post(self, url, data=None, headers=None, timeout=None): posted.append(url) if post_exc: raise post_exc @@ -1650,7 +1863,7 @@ def _submit(self, url, sess, size=100): import bits_helpers.cvmfs_publish as cp import bits_helpers.prepub as pp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"z" * size) + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"z" * size) cp._HEALTH.pop(url, None) with mock.patch.object(pp, "_make_session", lambda *a, **k: sess), \ mock.patch.object(pp, "_signed_uri", lambda u: u), \ @@ -1672,12 +1885,12 @@ def test_identity_path_sent_and_signed(self): import bits_helpers.cvmfs_publish as cp import bits_helpers.prepub as pp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"z") + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"z") sess, _ = self._sess() cap = {} orig_post = sess.post - def post(url, files=None, headers=None, timeout=None): - cap["files"] = files; return orig_post(url, files, headers, timeout) + def post(url, data=None, headers=None, timeout=None): + cap["files"] = data.fields; return orig_post(url, data, headers, timeout) sess.post = post def fake_auth(token, method, uri, fields=None, **kw): cap["signed"] = fields; return {} @@ -1700,8 +1913,8 @@ def test_replace_sent_and_signed_only_with_identity_and_hash(self): sess, _ = self._sess() cap = {} orig_post = sess.post - def post(url, files=None, headers=None, timeout=None): - cap["files"] = files; return orig_post(url, files, headers, timeout) + def post(url, data=None, headers=None, timeout=None): + cap["files"] = data.fields; return orig_post(url, data, headers, timeout) sess.post = post def fake_auth(token, method, uri, fields=None, **kw): cap["signed"] = fields; return {} @@ -1711,7 +1924,7 @@ def fake_auth(token, method, uri, fields=None, **kw): for kw, want in (({"identity_path": "p", "identity_hash": "H", "replace": True}, "true"), ({"identity_path": "p", "replace": True}, None), ({"identity_path": "p", "identity_hash": "H"}, None)): - open(tarf, "wb").write(b"z") + Path(tarf).write_bytes(b"z") cp.submit_ingest("http://id", "t", "r", "p", tarf, **kw) self.assertEqual(cap["signed"].get("replace"), want, kw) self.assertEqual(cap["files"].get("replace"), want and (None, want), kw) @@ -1741,7 +1954,7 @@ def test_identity_kept_when_replacing(self): from unittest import mock import bits_helpers.cvmfs_publish as cp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - tarf = os.path.join(t, "p.tar"); open(tarf, "wb").write(b"z") + tarf = os.path.join(t, "p.tar"); Path(tarf).write_bytes(b"z") got = [] def fake_submit(*a, **k): got.append((k.get("identity_path"), k.get("identity_hash"), k.get("replace"))); return "J" @@ -1794,7 +2007,7 @@ def test_reset_upload_says_why(self): class TestPublishTar(unittest.TestCase): def _tar(self): t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) - p = os.path.join(t, "x.tar"); open(p, "wb").write(b"x"); return p + p = os.path.join(t, "x.tar"); Path(p).write_bytes(b"x"); return p def test_ingest_posts_tar_and_skips_stage(self): from unittest import mock @@ -1861,7 +2074,7 @@ def test_ingest_workers_gt1_does_not_require_stage_flags(self): import bits_helpers.cvmfs_publish as cp t = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, t, True) m = os.path.join(t, "man.json") - json.dump({"packages": [{"package": "a", "version": "1.0"}]}, open(m, "w")) + Path(m).write_text(json.dumps({"packages": [{"package": "a", "version": "1.0"}]})) with mock.patch.object(cp, "publish_one", lambda s, c: []): rc = cp.main(["--manifest", m, "--repo", "r", "--tars-root", t, "--arch", "el9", "--publish-path", "ingest", "--workers", "4"]) @@ -1892,8 +2105,8 @@ def fake_publish_tar(ctx, path, tar, label, fp=None): try: d = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, d, True) - t1 = os.path.join(d, "a.tar"); open(t1, "w").write("x") - t2 = os.path.join(d, "b.tar"); open(t2, "w").write("y") + t1 = os.path.join(d, "a.tar"); Path(t1).write_text("x") + t2 = os.path.join(d, "b.tar"); Path(t2).write_text("y") jids = P.publish_overlay_tars( {"job_id_base": "base", "tmp_dir": d}, [t1, t2]) self.assertEqual(jids, ["job-a.tar", "job-b.tar"]) diff --git a/tests/test_cvmfs_release.py b/tests/test_cvmfs_release.py index a0c68c30..e2369cc5 100644 --- a/tests/test_cvmfs_release.py +++ b/tests/test_cvmfs_release.py @@ -109,6 +109,20 @@ def test_a_noarch_top_level_takes_the_architecture_most_packages_have(self): self.assertEqual(rel.merge_boms(boms)[1], ARCH) self.assertEqual(rel.merge_boms(boms, TOOL)[1], TOOL) # --arch decides + def test_the_architecture_the_boms_record_wins(self): + # A build of the toolchain alone: its BOM is under the toolchain's arch, + # the build's own is the one recorded (publish then makes no release). + boms = [_bom(TOOL, [_pkg("GCC", "g1", arch=TOOL)], architecture=ARCH, + requested_packages=["GCC"])] + self.assertEqual(rel.merge_boms(boms)[1], ARCH) + self.assertEqual(rel.merge_boms(boms, TOOL)[1], TOOL) # --arch decides + # The publishing host's arch (older BOMs) is not the build's: not trusted. + lcg = [_bom(ARCH, [_pkg("LCG", "l1")], architecture="x86_64-el9")] + self.assertIsNone(rel.build_architecture(lcg)) + self.assertEqual(rel.merge_boms(lcg)[1], ARCH) + self.assertEqual(rel.build_architecture([_bom(ARCH, [_pkg("LCG", "l1")], + architecture=ARCH)]), ARCH) + def test_merge_refuses_boms_without_templates_or_disagreeing(self): with self.assertRaises(rel.ReleaseError): rel.merge_boms([_bom(ARCH, [_pkg("ROOT", "r1")], cvmfs_templates=None)]) @@ -120,12 +134,16 @@ def test_merge_refuses_boms_without_templates_or_disagreeing(self): def test_signed_check(self): pkgs = [_pkg("ROOT", "r1"), _pkg("GCC", "g1")] - rel.check_signed(pkgs, {"r1": "sha256:x", "g1": "x"}) + rel.check_signed(pkgs, {(ARCH, "r1"): "sha256:x", (ARCH, "g1"): "x"}) with self.assertRaises(rel.ReleaseError) as cm: - rel.check_signed(pkgs, {"r1": "sha256:x", "g1": "sha256:other"}) + rel.check_signed(pkgs, {(ARCH, "r1"): "sha256:x", (ARCH, "g1"): "sha256:other"}) self.assertIn("GCC", str(cm.exception)) with self.assertRaises(rel.ReleaseError): - rel.check_signed(pkgs, {"r1": "sha256:x"}) + rel.check_signed(pkgs, {(ARCH, "r1"): "sha256:x"}) + # The same hash certified for another architecture only does not count. + with self.assertRaises(rel.ReleaseError) as cm: + rel.check_signed(pkgs, {(ARCH, "r1"): "sha256:x", ("other-arch", "g1"): "sha256:x"}) + self.assertIn("GCC", str(cm.exception)) class FetchTest(unittest.TestCase): diff --git a/tests/test_default_build_type.py b/tests/test_default_build_type.py new file mode 100644 index 00000000..14e9622e --- /dev/null +++ b/tests/test_default_build_type.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""A chain that qualifies the architecture but names no build type builds -opt: +the same binaries land under one qualified architecture, never also the bare one.""" + +import os +import shutil +import tempfile +import unittest + +from bits_helpers.arch import compute_combined_arch, compute_own_hash_arch +from bits_helpers.defaults import readDefaults + +PROFILES = { + "release": "", + "key4hep": "", + "gcc15": "append_arch: -gcc15\n", + "opt": "append_arch: -opt\nown_hash_neutral: true\nenv:\n CMAKE_BUILD_TYPE: RELWITHDEBINFO\n", + "dbg": "append_arch: -dbg\nown_hash_neutral: true\nenv:\n CMAKE_BUILD_TYPE: DEBUG\n", + "overlay": "valid_defaults_exempt: true\n", +} + + +class DefaultBuildTypeTest(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, True) + for name, body in PROFILES.items(): + self.profile(name, body) + + def profile(self, name, body): + with open(os.path.join(self.dir, "defaults-%s.sh" % name), "w") as fh: + fh.write("package: defaults-%s\nversion: v1\n%s---\n" % (name, body)) + + def read(self, chain): + meta, _body = readDefaults(self.dir, chain, self.fail, "x86_64-el9") + return meta, compute_combined_arch(meta, chain, "x86_64-el9") + + def test_no_build_type_builds_opt(self): + chain = ["release", "key4hep", "gcc15"] + meta, arch = self.read(chain) + self.assertEqual(chain, ["release", "key4hep", "gcc15", "opt"]) # what is built + self.assertEqual(arch, "x86_64-el9-gcc15-opt") + self.assertEqual(meta["env"]["CMAKE_BUILD_TYPE"], "RELWITHDEBINFO") + self.assertIn("opt", meta["_valid_defaults_exempt"]) # implied, not gated + # The toolchain still drops the build type from its own arch. + self.assertEqual(compute_own_hash_arch(meta, chain, "x86_64-el9"), "x86_64-el9-gcc15") + + def test_same_as_naming_it(self): + named = ["release", "key4hep", "gcc15", "opt"] + implied = ["release", "key4hep", "gcc15"] + a, b = self.read(named)[0], self.read(implied)[0] + b["_valid_defaults_exempt"].remove("opt") + self.assertEqual(a, b) + self.assertEqual(named, implied) + + def test_a_named_build_type_is_kept(self): + chain = ["release", "key4hep", "gcc15", "dbg"] + _meta, arch = self.read(chain) + self.assertEqual((chain[-1], arch), ("dbg", "x86_64-el9-gcc15-dbg")) + + def test_overlays_stay_last(self): + chain = ["release", "key4hep", "gcc15", "overlay"] + self.read(chain) + self.assertEqual(chain, ["release", "key4hep", "gcc15", "opt", "overlay"]) + + def test_left_alone_without_a_qualified_arch_or_an_opt_profile(self): + chain = ["release", "key4hep"] + self.assertEqual(self.read(chain)[1], "x86_64-el9") + self.assertEqual(chain, ["release", "key4hep"]) + os.unlink(os.path.join(self.dir, "defaults-opt.sh")) + chain = ["release", "key4hep", "gcc15"] + self.assertEqual(self.read(chain)[1], "x86_64-el9-gcc15") + self.assertEqual(chain, ["release", "key4hep", "gcc15"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_doctor.py b/tests/test_doctor.py index cda9f53a..be6a64fe 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -312,6 +312,13 @@ def test_real_tmp_passes_or_warns(self): # ── _check_store ─────────────────────────────────────────────────────────────── class TestCheckStore(unittest.TestCase): + def test_an_unreachable_site_default_read_store_is_a_warning(self): + from types import SimpleNamespace + from bits_helpers import doctor + with patch.object(doctor, "_check_store", return_value=(FAIL, "cannot reach store x")): + self.assertEqual(doctor.check_read_store("https://x", SimpleNamespace(remoteStoreExplicit=False))[0], WARN) + self.assertEqual(doctor.check_read_store("https://x", SimpleNamespace(remoteStoreExplicit=True))[0], FAIL) + def test_skip_empty_url(self): status, detail = _check_store("") self.assertEqual(status, SKIP) diff --git a/tests/test_fetch_cache.py b/tests/test_fetch_cache.py new file mode 100644 index 00000000..c5b2b560 --- /dev/null +++ b/tests/test_fetch_cache.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""bits_helpers.fetch_cache: the sources a build downloads itself (issue #129).""" + +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import unittest + +from bits_helpers import fetch_cache + +ARCH = "slc9_x86-64" +TEMPLATE = os.path.join(os.path.dirname(fetch_cache.__file__), "build_template.sh") + + +def template_listing(): + """build_template.sh's lines that list the fetch cache, as a bash script.""" + with open(TEMPLATE) as f: + text = f.read() + start = text.index("# What the build left in its fetch cache") + return text[start:text.index("\nfi\n", start) + 4] + + +class Store: + """The backends' fetch_source/upload_source in memory. Like rsync, an upload + keeps the local file's name, so it must be the one asked for.""" + + def __init__(self, write=True): + self.objects, self.writeStore = {}, "bucket" if write else "" + + def fetch_source(self, url_checksum, filename, dest_dir): + if (url_checksum, filename) not in self.objects: + return False + with open(os.path.join(dest_dir, filename), "wb") as f: + f.write(self.objects[(url_checksum, filename)]) + return True + + def upload_source(self, local_path, url_checksum, filename): + assert os.path.basename(local_path) == filename, (local_path, filename) + if self.writeStore: + with open(local_path, "rb") as f: + self.objects.setdefault((url_checksum, filename), f.read()) # never overwritten + + def indexes(self): + return [json.loads(v) for (_, n), v in self.objects.items() if n == fetch_cache.INDEX] + + +def spec(version="2.16", **kw): + return dict(package="tensorflow", version=version, fetch_cache=True, **kw) + + +class FetchCacheTest(unittest.TestCase): + def setUp(self): + self.work = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.work, True) + self.store = Store() + + def cache(self, s=None): + return os.path.join(self.work, fetch_cache.relative_dir(s or spec(), ARCH)) + + def _build(self, s, files): + """Restore, download *files* into the cache, list it with build_template.sh's + lines, save.""" + state = fetch_cache.restore(s, self.store, self.work, "cms", ARCH) + for rel, data in files.items(): + os.makedirs(os.path.dirname(os.path.join(self.cache(s), rel)), exist_ok=True) + with open(os.path.join(self.cache(s), rel), "wb") as f: + f.write(data) + buildroot = os.path.join(self.work, "BUILD") + os.makedirs(buildroot, exist_ok=True) + subprocess.run(["bash", "-c", "set -eo pipefail\n" + template_listing()], check=True, + env=dict(os.environ, CACHED_TARBALL="", BUILDROOT=buildroot, + BITS_FETCH_CACHE=self.cache(s))) + fetch_cache.save(s, self.store, self.work, "cms", ARCH, state, + os.path.join(buildroot, fetch_cache.LISTING)) + return state + + def test_a_build_saves_what_it_fetched_and_a_clean_build_gets_it_back(self): + self._build(spec(), {"content_addressable/sha256/ab/file": b"llvm", "a b": b"x"}) + (index,) = self.store.indexes() + self.assertEqual([f["path"] for f in index["files"]], ["a b", "content_addressable/sha256/ab/file"]) + shutil.rmtree(os.path.join(self.work, "FETCH_CACHE")) + state = fetch_cache.restore(spec(), self.store, self.work, "cms", ARCH) + with open(os.path.join(self.cache(), "content_addressable/sha256/ab/file"), "rb") as f: + self.assertEqual(f.read(), b"llvm") + self.assertEqual((len(state.files), len(state.confirmed)), (2, 2)) + self.assertEqual(sorted(os.listdir(os.path.join(self.work, "FETCH_CACHE"))), [ARCH]) + + def test_the_list_is_written_once(self): + self._build(spec(), {"a": b"1"}) + self._build(spec(), {"b": b"2"}) # not in the list: a warning + (index,) = self.store.indexes() + self.assertEqual([f["path"] for f in index["files"]], ["a"]) + + def test_versions_and_architectures_share_files_but_have_their_own_lists(self): + self._build(spec("2.16"), {"a": b"same"}) + self._build(spec("2.17"), {"a": b"same"}) + self.assertEqual(len([k for k in self.store.objects if k[1] == fetch_cache.BLOB]), 1) + self.assertEqual(len(self.store.indexes()), 2) + state = fetch_cache.restore(spec("2.16"), self.store, self.work, "cms", "osx_arm64") + self.assertIsNone(state.files) + + def test_a_file_missing_from_the_store_is_uploaded_again(self): + self._build(spec(), {"a": b"1"}) + sha = hashlib.sha256(b"1").hexdigest() + del self.store.objects[(sha, fetch_cache.BLOB)] + shutil.rmtree(os.path.join(self.work, "FETCH_CACHE")) + self._build(spec(), {"a": b"1"}) # the build downloads it again + self.assertIn((sha, fetch_cache.BLOB), self.store.objects) + + def test_an_empty_cache_writes_no_list(self): + self._build(spec(), {}) + self.assertEqual(self.store.indexes(), []) + self._build(spec(), {"a": b"1"}) + self.assertEqual(len(self.store.indexes()), 1) + + def test_one_failed_file_does_not_stop_the_restore(self): + self._build(spec(), {"a": b"1", "b": b"2"}) + shutil.rmtree(os.path.join(self.work, "FETCH_CACHE")) + real, bad = self.store.fetch_source, hashlib.sha256(b"1").hexdigest() + + def flaky(key, filename, tmp): + if key == bad: + raise OSError("SlowDown") + return real(key, filename, tmp) + self.store.fetch_source = flaky + state = fetch_cache.restore(spec(), self.store, self.work, "cms", ARCH) + self.assertEqual((sorted(state.files), os.listdir(self.cache())), (["a", "b"], ["b"])) + + def test_a_symlink_out_of_the_cache_is_not_followed(self): + self._build(spec(), {"d/a": b"1"}) + shutil.rmtree(self.cache()) + outside = os.path.join(self.work, "outside") + os.makedirs(outside) + os.makedirs(self.cache()) + os.symlink(outside, os.path.join(self.cache(), "d")) + fetch_cache.restore(spec(), self.store, self.work, "cms", ARCH) + self.assertEqual(os.listdir(outside), []) + + def test_a_tampered_file_or_path_is_not_restored(self): + self._build(spec(), {"a": b"good"}) + self.store.objects[(hashlib.sha256(b"good").hexdigest(), fetch_cache.BLOB)] = b"evil" + key = next(k for k in self.store.objects if k[1] == fetch_cache.INDEX) + index = json.loads(self.store.objects[key]) + index["files"].append(dict(index["files"][0], path="../../escape")) + self.store.objects[key] = json.dumps(index).encode() + shutil.rmtree(os.path.join(self.work, "FETCH_CACHE")) + state = fetch_cache.restore(spec(), self.store, self.work, "cms", ARCH) + self.assertEqual((os.listdir(self.cache()), list(state.files)), ([], ["a"])) + self.assertFalse(os.path.exists(os.path.join(self.work, "escape"))) + + def test_no_write_store_or_no_redistribution_saves_nothing(self): + self.store = Store(write=False) + self._build(spec(), {"a": b"1"}) + self.assertEqual(self.store.objects, {}) + self.store = Store() + self._build(spec(redistributable="binaries"), {"b": b"2"}) + self.assertEqual(self.store.objects, {}) + + def test_an_unreadable_file_only_skips_saving(self): + if os.geteuid() == 0: + self.skipTest("root reads everything") + os.makedirs(self.cache()) + with open(os.path.join(self.cache(), "locked"), "w"): + pass + os.chmod(os.path.join(self.cache(), "locked"), 0) + self._build(spec(), {"a": b"1"}) # the listing fails, the build does not + self.assertEqual(self.store.objects, {}) + + def test_the_defaults_must_name_the_cache(self): + self.assertEqual([fetch_cache.cache_name(v) for v in (None, "cms", "../x", True)], + ["", "cms", "", ""]) + self.assertFalse(fetch_cache.enabled(spec(), "")) + self.assertFalse(fetch_cache.enabled({"package": "zlib"}, "cms")) + self.assertTrue(fetch_cache.enabled(spec(), "cms")) + + def test_a_store_error_does_not_fail_the_build(self): + class Broken(Store): + def fetch_source(self, *a): + raise OSError("503") + + def upload_source(self, *a): + raise SystemExit(1) # as dieOnError does + self.store = Broken() + self._build(spec(), {"a": b"1"}) # neither raises + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_forge.py b/tests/test_forge.py index f3c20d3d..97c7eaee 100644 --- a/tests/test_forge.py +++ b/tests/test_forge.py @@ -273,6 +273,87 @@ def test_mr_iid_for_commit_prefers_merged(self): self.assertEqual(forge.gitlab_mr_iid_for_commit("https://gl/api/v4", "t", "p", "sha"), 5) +class TestGitLabMerge(unittest.TestCase): + """gitlab_merge_mr waits out GitLab's mergeability recheck, nothing else.""" + + class _R: + def __init__(self, code, payload=None): + self.status_code, self.ok, self._p = code, code < 400, payload or {} + self.reason, self.text = "", str(payload) + + def json(self): + return self._p + + def _merge(self, puts, gets, **kw): + calls = {"put": [], "sleep": []} + now = [0.0] + + def put(url, headers=None, json=None, timeout=None): + calls["put"].append(json) + r = puts.pop(0) + if isinstance(r, Exception): + raise r + return r + + def get(url, headers=None, timeout=None): + return gets.pop(0) + + def sleep(s): + calls["sleep"].append(s) + now[0] += s + with patch("requests.put", side_effect=put), patch("requests.get", side_effect=get): + out = forge.gitlab_merge_mr("https://gl/api/v4", "t", "p", 294, sleep=sleep, + clock=lambda: now[0], **kw) + return out, calls + + def test_a_recheck_is_waited_out(self): + R = self._R + out, calls = self._merge( + [R(405), R(405), R(200, {"state": "merged", "iid": 294})], + [R(200, {"state": "opened", "detailed_merge_status": "checking"}), + R(200, {"state": "opened", "detailed_merge_status": "unchecked"})], + sha="41a42723") + self.assertEqual(out["state"], "merged") + self.assertEqual(calls["sleep"], [3, 6]) + self.assertEqual(calls["put"], [{"sha": "41a42723"}] * 3) # the validated head only + + def test_other_refusals_raise_at_once(self): + R = self._R + for status in ("conflict", "draft_status", "ci_still_running", "need_rebase"): + with self.subTest(status=status): + with self.assertRaisesRegex(RuntimeError, status): + self._merge([R(405)], [R(200, {"state": "opened", + "detailed_merge_status": status})]) + with self.assertRaisesRegex(RuntimeError, "409"): + self._merge([R(409, {"message": "SHA does not match HEAD"})], []) + + def test_already_merged_counts_only_for_the_validated_commit(self): + R = self._R + merged = {"state": "merged", "iid": 294, "sha": "41a42723"} + out, _calls = self._merge([R(405)], [R(200, merged)], sha="41a42723") + self.assertEqual(out["state"], "merged") + with self.assertRaisesRegex(RuntimeError, "not the validated"): + self._merge([R(405)], [R(200, merged)], sha="99999999") + with self.assertRaisesRegex(RuntimeError, "merged already"): + self._merge([R(405)], [R(200, merged)]) + + def test_locked_old_statuses_422_and_a_timed_out_put_are_looked_at(self): + import requests + R = self._R + out, calls = self._merge( + [R(422), R(405), requests.exceptions.Timeout()], + [R(200, {"state": "locked", "merge_status": "checking"}), + R(200, {"state": "opened", "merge_status": "cannot_be_merged_recheck"}), + R(200, {"state": "merged", "sha": "s1"})], sha="s1") + self.assertEqual((out["state"], calls["sleep"]), ("merged", [3, 6])) + + def test_gives_up_after_the_wait(self): + R = self._R + pending = R(200, {"state": "opened", "detailed_merge_status": "checking"}) + with self.assertRaisesRegex(RuntimeError, "still checking"): + self._merge([R(405)] * 20, [pending] * 20, wait=60) + + class TestGitLabSubgroups(unittest.TestCase): def test_lists_all_visible_subgroups_not_only_memberships(self): # Without all_available GitLab returns only groups the token belongs to. diff --git a/tests/test_gc.py b/tests/test_gc.py index 6f1e7636..5b52ca75 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later """Tests for reachability GC of the shared store (bits_helpers/gc.py).""" +from pathlib import Path import os import shutil import tempfile @@ -159,11 +160,108 @@ class TestLocalizeManifest(unittest.TestCase): .sig) before verification — gc cannot open() a URL like the consumer path.""" def setUp(self): + import bits_helpers.download as dl + dl._SIGNED.clear() + self.addCleanup(dl._SIGNED.clear) self.tmp = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) + def _serve(self, files, got=None): + """A download._http_get stand-in: *files* maps a URL to its body (None: + the fetch fails). Each body has the ETag ""; a matching + If-None-Match gets a 304.""" + def fake(url, headers, timeout=60): + if got is not None: + got.append(url) + body = files.get(url) + if body is None: + raise OSError("unreachable %s" % url) + tag = '"%s"' % body + if headers.get("If-None-Match") == tag: + return 304, None, tag + return 200, body.encode(), tag + return fake + + def _cached(self, name, body): + d = os.path.join(self.tmp, "MANIFESTS", "gc") + os.makedirs(d, exist_ok=True) + for n, b in ((name, body), (name + ".sig", body + "-sig")): + with open(os.path.join(d, n), "w") as fh: + fh.write(b) + return os.path.join(d, name) + + def test_a_cached_copy_is_replaced(self): + from unittest.mock import patch + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + import hashlib + old = self._cached("%s.common-manifest-x86_64-el9.json" + % hashlib.sha256(url.encode()).hexdigest()[:12], "old") + serve = self._serve({url: "new", url + ".sig": "new-sig"}) + with patch("bits_helpers.download._http_get", side_effect=serve): + local = gc._localize_manifest(url, self.tmp) + self.assertEqual(local, old) + self.assertEqual((Path(local).read_text(), Path(local + ".sig").read_text()), ("new", "new-sig")) + self.assertEqual([n for n in os.listdir(os.path.dirname(local)) if ".part" in n], []) + + def test_a_failed_fetch_never_falls_back_to_the_cached_copy(self): + from unittest.mock import patch + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + for files in ({}, {url: "new"}): # the manifest, or only its .sig + with self.subTest(files=sorted(files)): + import bits_helpers.download as dl + dl._SIGNED.clear() + import hashlib + old = self._cached("%s.common-manifest-x86_64-el9.json" + % hashlib.sha256(url.encode()).hexdigest()[:12], "old") + with patch("bits_helpers.download._http_get", + side_effect=self._serve(files)): + with self.assertRaises(OSError): + gc._localize_manifest(url, self.tmp) + self.assertNotEqual(Path(old + ".sig").read_text(), "new-sig") + + def test_two_stores_one_name(self): + from unittest.mock import patch + a = "https://s3.example/one/MANIFESTS/common-manifest-shared.json" + b = "https://s3.example/two/MANIFESTS/common-manifest-shared.json" + serve = self._serve({a: "A", a + ".sig": "a", b: "B", b + ".sig": "b"}) + with patch("bits_helpers.download._http_get", side_effect=serve): + pa, pb = gc._localize_manifest(a, self.tmp), gc._localize_manifest(b, self.tmp) + self.assertEqual((Path(pa).read_text(), Path(pb).read_text()), ("A", "B")) + + def test_fetched_once_per_run(self): + from unittest.mock import patch + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + got = [] + with patch("bits_helpers.download._http_get", + side_effect=self._serve({url: "new", url + ".sig": "s"}, got)): + a = gc._localize_manifest(url, self.tmp) + b = gc._localize_manifest(url, self.tmp) + self.assertEqual((a, got), (b, [url, url + ".sig"])) + got.clear() + with patch("bits_helpers.download._http_get", side_effect=self._serve({}, got)): + with self.assertRaises(OSError): + gc._localize_manifest(url + "x", self.tmp) + with self.assertRaises(OSError): # the failure, not a retry + gc._localize_manifest(url + "x", self.tmp) + self.assertEqual(got, [url + "x"]) + + def test_a_build_reuses_nothing_from_a_manifest_it_cannot_fetch(self): + """The stale-cache case: a build node's MANIFESTS/trust holds an old, + still valid copy; the store cannot be read. Nothing is trusted from it.""" + from unittest.mock import patch + from bits_helpers import build + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + d = os.path.join(self.tmp, "MANIFESTS", "trust") + os.makedirs(d) + for n in ("common-manifest-x86_64-el9.json", "common-manifest-x86_64-el9.json.sig"): + Path(os.path.join(d, n)).write_text("{}") + with patch("bits_helpers.download._http_get", side_effect=self._serve({})), \ + patch("bits_helpers.trust.trusted_index", + side_effect=AssertionError("the cached copy was verified")): + self.assertEqual(build._load_trusted_index(url, self.tmp, None), (None, {})) + def test_local_path_passes_through(self): p = os.path.join(self.tmp, "common-manifest-shared.json") open(p, "w").close() @@ -173,13 +271,8 @@ def test_url_is_downloaded_with_sig(self): from unittest.mock import patch url = "https://s3.example/MANIFESTS/common-manifest-osx_arm64.json" got = [] - - def fake_download(source, destDir, work_dir, dest_filename=None): - got.append(source) - with open(os.path.join(destDir, dest_filename), "w") as fh: - fh.write("{}") - - with patch("bits_helpers.download.downloadUrllib2", side_effect=fake_download): + with patch("bits_helpers.download._http_get", + side_effect=self._serve({url: "{}", url + ".sig": "{}"}, got)): local = gc._localize_manifest(url, self.tmp) # Both the manifest and its detached signature are fetched... self.assertEqual(got, [url, url + ".sig"]) @@ -188,6 +281,68 @@ def fake_download(source, destDir, work_dir, dest_filename=None): self.assertTrue(os.path.isfile(local + ".sig")) self.assertTrue(local.endswith("common-manifest-osx_arm64.json")) + def test_a_weak_etag_is_never_sent_back(self): + from unittest.mock import patch + import bits_helpers.download as dl + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + sent = [] + + def weak(u, headers, timeout=60): + sent.append(headers.get("If-None-Match")) + return 200, b"v", 'W/"v"' + for _ in range(2): + dl._SIGNED.clear() + with patch("bits_helpers.download._http_get", side_effect=weak): + gc._localize_manifest(url, self.tmp) + self.assertEqual(sent, [None] * 4) + + def test_a_truncated_body_is_a_failed_fetch(self): + from http.client import IncompleteRead + from unittest.mock import MagicMock, patch + import bits_helpers.download as dl + resp = MagicMock() + resp.__enter__.return_value = resp + resp.read.side_effect = IncompleteRead(b"par", 10) + with patch("bits_helpers.download.urlopen", return_value=resp): + with self.assertRaises(OSError): + dl._http_get("https://s3.example/x.json", {}) + + def test_an_unchanged_manifest_is_not_downloaded_again(self): + """The next run sends the ETag it kept: 304, and the local copy stays. + A copy changed on disk, or a store copy that changed, is fetched in full.""" + from unittest.mock import patch + import bits_helpers.download as dl + url = "https://s3.example/MANIFESTS/common-manifest-x86_64-el9.json" + files = {url: "v1", url + ".sig": "s1"} + sent = [] + + def run(): + dl._SIGNED.clear() + serve = self._serve(files) + + def spy(u, headers, timeout=60): + r = serve(u, headers, timeout) + sent.append((u, headers.get("If-None-Match"), r[0])) + return r + with patch("bits_helpers.download._http_get", side_effect=spy): + return gc._localize_manifest(url, self.tmp) + local = run() + sent.clear() + self.assertEqual(run(), local) + self.assertEqual([s[2] for s in sent], [304, 304]) + self.assertEqual(Path(local).read_text(), "v1") + files[url] = "v2" # re-signed in the store + files[url + ".sig"] = "s2" + sent.clear() + run() + self.assertEqual([s[2] for s in sent], [200, 200]) + self.assertEqual((Path(local).read_text(), Path(local + ".sig").read_text()), ("v2", "s2")) + with open(local, "w") as fh: # tampered with on disk + fh.write("evil") + sent.clear() + run() + self.assertEqual(sent[0][1:], (None, 200)) # no ETag sent: fetched in full + self.assertEqual(Path(local).read_text(), "v2") if __name__ == "__main__": unittest.main() diff --git a/tests/test_hashing.py b/tests/test_hashing.py index bd734769..32235077 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -176,6 +176,16 @@ def test_view_rules_are_hash_invariant(self): self.assertEqual(self._n(self.HEADER), self._n(self.HEADER.replace("requires:\n", "view: false\nrequires:\n"))) + def test_references_are_hash_invariant(self): + # references: CPEs/purls for scanners, written into the SBOM only. + withr = self.HEADER.replace( + "requires:\n", + "references:\n" + " - cpe:2.3:a:sqlite:sqlite:3.32.3:*:*:*:*:*:*:*\n" + " - pkg:pypi/foo@%(version)s\n" + "requires:\n") + self.assertEqual(self._n(self.HEADER), self._n(withr)) + def test_adding_preload_block_is_hash_invariant(self): # The preload: test list (consumed post-publish by `bits preload`) is # hash-excluded — its indented block is dropped and editing it must not diff --git a/tests/test_help.py b/tests/test_help.py new file mode 100644 index 00000000..6fa7f685 --- /dev/null +++ b/tests/test_help.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""bits -h lists every command bits runs, and each gives its own help.""" + +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# Run by the launcher itself, not by bitsBuild. +LAUNCHER = {"store", "use", "cvmfs", "preload", "overlay", "trace", "q", "query", "enter", + "setenv", "load", "printenv", "unload", "list", "avail", "modulecmd", + "shell-helper"} +MODULE_COMMANDS = LAUNCHER - {"store", "use", "cvmfs", "preload", "overlay", "trace"} +DEPRECATED = {"cleanup", "lcg-view", "cvmfs-stage", "cvmfs-publish", "store-stats"} + + +class HelpTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.mkdtemp() + stubs = os.path.join(cls.tmp, "stubs") + os.makedirs(stubs) + # A wrapper, not a link: run by a link's path, a venv's Python does not find its venv. + with open(os.path.join(stubs, "python3"), "w") as f: + f.write('#!/bin/sh\nexec %s "$@"\n' % shlex.quote(sys.executable)) + os.chmod(os.path.join(stubs, "python3"), 0o755) + cls.env = {"HOME": cls.tmp, "PATH": stubs + ":/usr/bin:/bin"} + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, True) + + def _bits(self, *args): + return subprocess.run([os.path.join(ROOT, "bits")] + list(args), cwd=self.tmp, + env=self.env, capture_output=True, text=True, timeout=120) + + def _listed(self): + out = self._bits("-h").stdout + names = set() + for line in out.splitlines(): + m = re.match(r"^ ([a-z][a-z-]*(?:, [a-z][a-z-]*)*)(?: |$)", line) + if m: + names.update(m.group(1).split(", ")) + return out, names + + def test_every_command_is_listed(self): + choices = subprocess.run([sys.executable, os.path.join(ROOT, "bitsBuild"), "--help"], + env=self.env, capture_output=True, text=True).stdout + python = set(re.search(r"\{([a-z,-]+)\}", choices).group(1).split(",")) + out, listed = self._listed() + self.assertEqual(listed, (python | LAUNCHER) - DEPRECATED, out) + + def test_each_command_has_its_own_help(self): + top, listed = self._listed() + for name in sorted(listed): + r = self._bits(name, "--help") + self.assertEqual(r.returncode, 0, (name, r.stderr)) + self.assertNotEqual(r.stdout, top, name) + if name in MODULE_COMMANDS: + self.assertTrue(r.stdout.startswith("bits: the module commands"), name) + else: + self.assertRegex(r.stdout, r"(?i)usage: bits", name) + + def test_help_command(self): + self.assertEqual(self._bits("help", "deps").stdout, self._bits("deps", "--help").stdout) + self.assertEqual(self._bits("help").stdout, self._bits("--help").stdout) + self.assertEqual(self._bits("--community", "atlas", "-h").stdout, self._bits("-h").stdout) + self.assertEqual(self._bits("help", "cvmfs", "stage").stdout, + self._bits("cvmfs", "stage", "--help").stdout) + for args in (["help", "nosuch"], ["nosuch", "--help"]): + r = self._bits(*args) + self.assertEqual((r.returncode, r.stdout), (1, ""), args) + + def test_shell_helper_prints_help(self): + # The function evals what load/unload print: their help must not be. + r = subprocess.run(["bash", "-c", 'eval "$("$BITS_ENV" shell-helper)"; bits load --help'], + cwd=self.tmp, env=dict(self.env, BITS_ENV=os.path.join(ROOT, "bits")), + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertTrue(r.stdout.startswith("bits: the module commands"), r.stdout) + + def test_mistakes_are_short(self): + for args in (["nosuch"], ["-a", "x86_64-el9"]): + r = self._bits(*args) + self.assertEqual((r.returncode, r.stdout), (1, ""), args) + self.assertIn("bits help", r.stderr) + self.assertLess(len(r.stderr.splitlines()), 4, r.stderr) + r = self._bits() + self.assertEqual(r.returncode, 1) + self.assertIn("Usage: bits", r.stderr) + + def test_options_before_a_build_command(self): + r = self._bits("-a", "x86_64-el9", "build", "zlib") + self.assertEqual(r.returncode, 1) + self.assertIn("build takes its options after it", r.stderr) + + def test_debug_before_a_command(self): + r = self._bits("--debug", "version") + self.assertEqual(r.returncode, 0, r.stderr) + + def test_fits_80_columns(self): + for args in (["-h"], ["enter", "--help"]): + wide = [l for l in self._bits(*args).stdout.splitlines() if len(l) > 80] + self.assertEqual(wide, [], args) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_key_policy.py b/tests/test_key_policy.py index 3ba7f5c7..a3765c77 100644 --- a/tests/test_key_policy.py +++ b/tests/test_key_policy.py @@ -57,12 +57,14 @@ def _manifest(self, group): with open(path, "w") as fh: json.dump({"packages": [ {"group": group, "hash": "h1", "tarball_sha256": "sha256:aa"}]}, fh) + with open(path + ".sig", "w") as fh: # verify_bytes is patched + json.dump({}, fh) return path def test_policy_denied_drops_and_warns(self): path = self._manifest("lcg") denying = {"deadbeefdeadbeef": {"lcg"}, "default": set()} # signer kid not listed - with mock.patch.object(trust, "verify_manifest", return_value="ffffffffffffffff"), \ + with mock.patch.object(trust, "verify_bytes", return_value="ffffffffffffffff"), \ mock.patch.object(trust, "load_key_policy", return_value=denying), \ mock.patch("bits_helpers.log.warning") as w: kid, entries = trust._verified_entries(path, None, None, None, None) @@ -72,7 +74,7 @@ def test_policy_denied_drops_and_warns(self): def test_authorized_key_no_warning(self): path = self._manifest("lcg") allowing = {"ffffffffffffffff": {"*"}, "default": set()} - with mock.patch.object(trust, "verify_manifest", return_value="ffffffffffffffff"), \ + with mock.patch.object(trust, "verify_bytes", return_value="ffffffffffffffff"), \ mock.patch.object(trust, "load_key_policy", return_value=allowing), \ mock.patch("bits_helpers.log.warning") as w: kid, entries = trust._verified_entries(path, None, None, None, None) diff --git a/tests/test_lcg_view.py b/tests/test_lcg_view.py index 477928dd..15c8669f 100644 --- a/tests/test_lcg_view.py +++ b/tests/test_lcg_view.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later """`bits lcg-view --build-view`: manifest + native merged view + collapsed setup.sh.""" +from pathlib import Path import json import os import tempfile @@ -16,10 +17,10 @@ def _pkg(work, arch, name, ver, files, deps=None): for rel in files: p = os.path.join(d, rel) os.makedirs(os.path.dirname(p), exist_ok=True) - open(p, "w").write(rel) - json.dump({"package": {"name": name, "version": ver, "hash": "h" + name, "revision": 1}, - "dependencies": {"direct": {"runtime": deps or []}}}, - open(os.path.join(d, ".meta.json"), "w")) + Path(p).write_text(rel) + meta = {"package": {"name": name, "version": ver, "hash": "h" + name, "revision": 1}, + "dependencies": {"direct": {"runtime": deps or []}}} + Path(os.path.join(d, ".meta.json")).write_text(json.dumps(meta)) class TestBuildView(unittest.TestCase): @@ -51,7 +52,7 @@ def test_manifest_and_view_and_setup(self): self.assertFalse(os.path.isabs(os.readlink(root_bin))) self.assertTrue(os.path.exists(root_bin)) # setup.sh collapses the path vars onto the single view dir - setup = open(os.path.join(view, "setup.sh")).read() + setup = Path(os.path.join(view, "setup.sh")).read_text() self.assertIn("export PATH='%s/bin" % os.path.abspath(view), setup) self.assertIn("export CMAKE_PREFIX_PATH='%s" % os.path.abspath(view), setup) self.assertIn("python3.11/site-packages", setup) # PYTHONPATH minor detected diff --git a/tests/test_manifest.py b/tests/test_manifest.py index f0fc9fdb..3cde7b27 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -133,6 +133,67 @@ def test_view_rules_recorded_only_when_set(self): self.assertEqual(pkgs[0]["view"], {"exclude": ["share/doc"]}) self.assertNotIn("view", pkgs[1]) + def test_references_recorded_only_when_set(self): + m = _make_manifest(self.tmp) + spec = {"package": "sqlite", "version": "3320300", "revision": "1", "hash": "h", + "tag": "v1", "references": ["cpe:2.3:a:sqlite:sqlite:3.32.3:*:*:*:*:*:*:*", + "pkg:generic/%(name)s@%(tag)s", "nonsense", + "pkg:generic/%(name)s@%(tag)s"]} + m.add_package(spec, "built_from_source") + m.add_package(dict(spec, package="zlib", references=None), "built_from_source") + pkgs = self._load(m)["packages"] + self.assertEqual(pkgs[0]["references"], ["cpe:2.3:a:sqlite:sqlite:3.32.3:*:*:*:*:*:*:*", + "pkg:generic/sqlite@v1"]) + self.assertNotIn("references", pkgs[1]) + + def test_a_pip_recipe_is_named_by_its_pypi_purl(self): + m = _make_manifest(self.tmp) + body = "package: dm_tree\n---\n. $(bits-include PythonPipRecipe)\nPYPI_NAME=\"dm-tree\"\n" + spec = {"package": "dm_tree", "version": "0.1.9", "revision": "1", "hash": "h", "recipe": body} + m.add_package(spec, "built_from_source") + m.add_package(dict(spec, package="Jinja2", hash="j", + recipe="---\n. $(bits-include PythonPipRecipe)\n"), "built_from_source") + m.add_package(dict(spec, package="own", hash="o", + references=["pkg:pypi/other@1"]), "built_from_source") + m.add_package(dict(spec, package="cmake", hash="c", + recipe="---\n. $(bits-include CMakeRecipe)\n"), "built_from_source") + pkgs = {p["package"]: p for p in self._load(m)["packages"]} + self.assertEqual(pkgs["dm_tree"]["references"], ["pkg:pypi/dm-tree@0.1.9"]) + self.assertEqual(pkgs["Jinja2"]["references"], ["pkg:pypi/jinja2@0.1.9"]) + self.assertEqual(pkgs["own"]["references"], ["pkg:pypi/other@1"]) # the recipe's wins + self.assertNotIn("references", pkgs["cmake"]) + + def test_cpes_are_completed_escaped_or_dropped(self): + m = _make_manifest(self.tmp) + spec = {"package": "torch", "version": "2.1.0+cpu", "revision": "1", "hash": "h", + "references": ["cpe:2.3:a:linuxfoundation:pytorch:%(version)s", + "cpe:2.3:a:x:y:1:*:*:*:*:*:*:*:*:extra", "cpe:2.3:a", + "pkg:generic/torch@%(version)s"]} + m.add_package(spec, "built_from_source") + self.assertEqual(self._load(m)["packages"][0]["references"], + ["cpe:2.3:a:linuxfoundation:pytorch:2.1.0\\+cpu:*:*:*:*:*:*:*", + "pkg:generic/torch@2.1.0%2Bcpu"]) + + def test_a_cpe_version_loses_a_leading_v(self): + m = _make_manifest(self.tmp) + spec = {"package": "ROOT", "version": "v6.40.02", "revision": "1", "hash": "h", + "references": ["cpe:2.3:a:cern:root:%(version)s", "pkg:generic/root@%(version)s"]} + m.add_package(spec, "built_from_source") + self.assertEqual(self._load(m)["packages"][0]["references"], + ["cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*", "pkg:generic/root@v6.40.02"]) + + def test_only_a_live_pip_include_names_a_pypi_package(self): + m = _make_manifest(self.tmp) + spec = {"package": "xgboost", "version": "2.0", "revision": "1", "hash": "h", + "recipe": "---\n# . $(bits-include PythonPipRecipe)\n. $(bits-include CMakeRecipe)\n"} + m.add_package(spec, "built_from_source") + m.add_package(dict(spec, package="zope_interface", hash="z", + recipe="---\n. $(bits-include PythonPipRecipe)\n" + "PYPI_NAME=zope.interface # the pip name\n"), "built_from_source") + pkgs = {p["package"]: p for p in self._load(m)["packages"]} + self.assertNotIn("references", pkgs["xgboost"]) + self.assertEqual(pkgs["zope_interface"]["references"], ["pkg:pypi/zope.interface@2.0"]) + def test_path_in_work_dir(self): m = _make_manifest(self.tmp) self.assertTrue(m.path.startswith(self.tmp)) @@ -353,12 +414,19 @@ def test_system_dependencies_and_untracked_recorded(self): # The build drops system-provided packages from the specs' requires; # the manifest gets them back from set_system_packages. m = _make_manifest(self.tmp) - m.set_system_packages({"zlib", "OpenSSL"}, {"ROOT": (["zlib"], ["OpenSSL"])}) + m.set_system_packages({"zlib", "OpenSSL"}, {"ROOT": (["zlib"], ["OpenSSL"])}, + {"zlib": {"version": "1.2.11", "references": []}, + "OpenSSL": {"version": "", "references": []}, + "perf": {"version": "", "references": [], "sbom": "tool"}}) spec = dict(self.spec, package="ROOT", runtime_requires=["Python"], - untracked_requires=["xrootd"], build_requires=["CMake"]) + untracked_requires=["xrootd"], build_requires=["CMake"], sbom="Tool") m.add_package(spec, "already_installed") data = self._load(m) self.assertEqual(data["system_packages"], ["OpenSSL", "zlib"]) + self.assertEqual(data["system_package_info"], # a host tool kept, version or not + {"zlib": {"version": "1.2.11", "references": []}, + "perf": {"version": "", "references": [], "sbom": "tool"}}) + self.assertEqual(data["packages"][0]["sbom"], "tool") pkg = data["packages"][0] self.assertEqual(pkg["requires"], ["Python", "xrootd", "zlib"]) self.assertEqual(pkg["build_requires"], ["CMake", "OpenSSL"]) diff --git a/tests/test_modulefile_env.py b/tests/test_modulefile_env.py new file mode 100644 index 00000000..988cfb01 --- /dev/null +++ b/tests/test_modulefile_env.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Reading the environment of bits modulefiles without Tcl (modulefile_env).""" + +import unittest + +from bits_helpers import modulefile_env as mfe + +# As bits-recipe-tools' ModuleRecipe writes it, plus a recipe's own lines. +ROOT = """#%Module1.0 +proc ModulesHelp { } { + global version + puts stderr "Modulefile for ROOT 6.36-2" +} +set version 6.36-2 +module-whatis "Modulefile for ROOT 6.36-2" +# Dependencies +if ![ is-loaded 'BASE/1.0' ] { + module load BASE/1.0 +} +if ![ is-loaded "zlib/1.3-1" ] { module load zlib/1.3-1 } + +set PKG_ROOT $::env(BASEDIR)/ROOT/$version +if {[file isdirectory $PKG_ROOT/bin]} { prepend-path PATH $PKG_ROOT/bin } +if {[file isdirectory $PKG_ROOT/lib]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib } +if {[file isdirectory $PKG_ROOT/lib64]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib64 } +setenv ROOT_INCLUDE_DIR $PKG_ROOT/include +prepend-path CMAKE_PREFIX_PATH $PKG_ROOT +setenv ROOT_ROOT $PKG_ROOT +if {[file isdirectory $PKG_ROOT/include]} { prepend-path ROOT_INCLUDE_PATH $PKG_ROOT/include } +if {[info exists ::env(MANPATH)] && $::env(MANPATH) ne "" && [file isdirectory $PKG_ROOT/share/man]} { prepend-path MANPATH $PKG_ROOT/share/man } +setenv BITS_VIEW 1 + +setenv ROOTSYS $PKG_ROOT +setenv ZLIB_FROM_ROOT $env(ZLIB_ROOT)/lib +prepend-path ROOT_PLUGIN_PATH $PKG_ROOT/etc/plugins +""" + +ZLIB = """#%Module1.0 +set version 1.3-1 +if ![ is-loaded 'BASE/1.0' ] { + module load BASE/1.0 +} +set PKG_ROOT $::env(BASEDIR)/zlib/$version +if {[file isdirectory $PKG_ROOT/lib]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib } +setenv ZLIB_ROOT $PKG_ROOT +""" + +MEMBERS = {"ROOT": "6.36-2", "zlib": "1.3-1"} +DIRS = {("ROOT", "bin"), ("ROOT", "lib"), ("ROOT", "include"), ("zlib", "lib")} + + +def ops_of(texts, top=("ROOT",), members=MEMBERS, dirs=DIRS): + return mfe.release_ops(list(top), members, texts.get, + lambda pkg, rel: (pkg, rel) in dirs) + + +class TestReleaseOps(unittest.TestCase): + + def test_a_dependency_is_loaded_where_its_module_is_loaded(self): + ops, problems = ops_of({"ROOT": ROOT, "zlib": ZLIB}) + self.assertEqual(problems, []) + r, z = "@B@/ROOT/6.36-2", "@B@/zlib/1.3-1" + self.assertEqual(ops, [ + ("zlib", "prepend-path", "LD_LIBRARY_PATH", z + "/lib"), + ("zlib", "setenv", "ZLIB_ROOT", z), + ("ROOT", "prepend-path", "PATH", r + "/bin"), + ("ROOT", "prepend-path", "LD_LIBRARY_PATH", r + "/lib"), # no lib64 + ("ROOT", "setenv", "ROOT_INCLUDE_DIR", r + "/include"), + ("ROOT", "prepend-path", "CMAKE_PREFIX_PATH", r), + ("ROOT", "setenv", "ROOT_ROOT", r), + ("ROOT", "prepend-path", "ROOT_INCLUDE_PATH", r + "/include"), + ("ROOT", "setenv", "BITS_VIEW", "1"), + ("ROOT", "setenv", "ROOTSYS", r), + ("ROOT", "setenv", "ZLIB_FROM_ROOT", z + "/lib"), # $env() of the dependency + ("ROOT", "prepend-path", "ROOT_PLUGIN_PATH", r + "/etc/plugins")]) + + def test_what_is_not_understood_is_a_problem(self): + # A module not read leaves what depends on it unset too. + _ops, problems = ops_of({"ROOT": ROOT, "zlib": ZLIB + "set x [exec uname]\n"}) + self.assertEqual(problems, ["zlib: unknown command: set x [exec uname]", + "ROOT: $env(ZLIB_ROOT) is not set by the release"]) + _ops, problems = ops_of({"ROOT": ROOT, "zlib": ZLIB + "setenv X [exec uname]\n"}) + self.assertEqual(problems[0], "zlib: unknown command: setenv X [exec uname]") + _ops, problems = ops_of({"ROOT": ROOT, "zlib": ZLIB + 'setenv X "a\\$b"\n'}) + self.assertEqual(problems, ['zlib: command or escape in a\\$b']) + _ops, problems = ops_of({"ROOT": ROOT.replace("$env(ZLIB_ROOT)", "$env(HOME)"), + "zlib": ZLIB}) + self.assertEqual(problems, ["ROOT: $env(HOME) is not set by the release"]) + + def test_a_module_outside_the_release_is_a_problem(self): + _ops, problems = ops_of({"ROOT": ROOT, "zlib": ZLIB}, + members={"ROOT": "6.36-2", "zlib": "1.2-1"}) + self.assertEqual(problems[0], "ROOT loads zlib/1.3-1, not a module of this release") + _ops, problems = ops_of({"ROOT": ROOT}) + self.assertEqual(problems[0], "zlib: no modulefile") + + def test_a_guard_leaving_the_package_is_a_problem(self): + out = ROOT + ("if {[file isdirectory $PKG_ROOT/../../zlib/1.3-1/lib]} " + "{ prepend-path X $PKG_ROOT/../../zlib/1.3-1/lib }\n") + _ops, problems = ops_of({"ROOT": out, "zlib": ZLIB}) + self.assertEqual(problems, ["ROOT: tests a directory outside the package: " + "@B@/zlib/1.3-1/lib"]) + + def test_unbalanced_braces_are_a_problem(self): + _ops, problems = ops_of({"ROOT": ROOT + "if {1} {\n", "zlib": ZLIB}) + self.assertTrue(problems[0].startswith("ROOT: unbalanced braces")) + + +class TestLoads(unittest.TestCase): + + def test_a_load_without_a_version_is_a_problem(self): + # modulecmd takes the tree's default, which may be another release's. + _ops, problems = ops_of({"ROOT": ROOT.replace("module load zlib/1.3-1", "module load zlib"), + "zlib": ZLIB}) + self.assertEqual(problems[0], "ROOT loads zlib without a version") + + def test_prereq_of_several_modules_is_a_problem(self): + # modulecmd reads it as any one of them. + _ops, problems = ops_of({"ROOT": ROOT + "prereq zlib/1.3-1 zstd/1-1\n", "zlib": ZLIB}) + self.assertEqual(problems, ["ROOT: prereq of several modules: prereq zlib/1.3-1 zstd/1-1"]) + + +V = lambda pkg, rel: rel.split("/")[0] in ("bin", "lib", "include") # the view links these + + +class TestSetupEnv(unittest.TestCase): + + def test_collapsed_variables_go_to_the_view_others_to_the_package(self): + ops, _ = ops_of({"ROOT": ROOT, "zlib": ZLIB}) + paths, sets, problems = mfe.setup_env(ops, MEMBERS, V) + self.assertEqual(problems, []) + I = mfe.INHERITED + self.assertEqual(paths, { + "LD_LIBRARY_PATH": ["@V@/lib", I], + "PATH": ["@V@/bin", I], + "CMAKE_PREFIX_PATH": ["@B@/ROOT/6.36-2", I], # the root: V says not all of it + "ROOT_INCLUDE_PATH": ["@V@/include", I], + "ROOT_PLUGIN_PATH": ["@B@/ROOT/6.36-2/etc/plugins", I]}) + # _ROOT and the like keep the package: software finds its files there. + self.assertEqual(sets, { + "ZLIB_ROOT": "@B@/zlib/1.3-1", "ROOT_INCLUDE_DIR": "@B@/ROOT/6.36-2/include", + "ROOT_ROOT": "@B@/ROOT/6.36-2", "ROOTSYS": "@B@/ROOT/6.36-2", + "ZLIB_FROM_ROOT": "@B@/zlib/1.3-1/lib"}) + paths, _sets, _ = mfe.setup_env(ops, MEMBERS, lambda pkg, rel: True) + self.assertEqual(paths["CMAKE_PREFIX_PATH"], ["@V@", I]) + + def test_an_entry_the_view_does_not_link_stays_the_package(self): + ops, _ = ops_of({"ROOT": ROOT, "zlib": ZLIB}) + paths, _sets, _ = mfe.setup_env(ops, MEMBERS, lambda pkg, rel: pkg != "zlib") + self.assertEqual(paths["LD_LIBRARY_PATH"], ["@V@/lib", "@B@/zlib/1.3-1/lib", None]) + + def test_entries_keep_the_order_modulecmd_gives_them(self): + r = "@B@/ROOT/6.36-2" + ops = [("zlib", "prepend-path", "PATH", "@B@/zlib/1.3-1/bin"), + ("ROOT", "prepend-path", "PATH", r + "/wrappers"), + ("ROOT", "prepend-path", "PATH", r + "/bin"), + ("ROOT", "append-path", "PATH", r + "/tools"), + ("ROOT", "prepend-path", "PATH", r + "/wrappers")] # there already: stays + paths, _s, problems = mfe.setup_env(ops, MEMBERS, V) + # modulecmd: bin:wrappers:zlib/bin::tools; collapsed, the first stays. + self.assertEqual(paths["PATH"], ["@V@/bin", r + "/wrappers", None, r + "/tools"]) + self.assertEqual(problems, []) + + def test_values_setup_cannot_reproduce_are_problems(self): + ops = [("zlib", "setenv", "MSG", "a b"), + ("zlib", "setenv", "DATA", "/build/sw/INSTALLROOT/x/share"), + ("zlib", "setenv", "MORE", "@B@/zlib/1.3-1/a:/build/b"), + ("zlib", "prepend-path", "PATH", "@B@/zlib/1.3-1/a:@B@/zlib/1.3-1/b")] + paths, sets, problems = mfe.setup_env(ops, MEMBERS, lambda *a: True) + self.assertEqual((paths, sets), ({}, {})) + self.assertEqual(problems, [ + "zlib: MSG's value needs quoting: a b", + "zlib: DATA is an absolute path: /build/sw/INSTALLROOT/x/share", + "zlib: MORE is an absolute path: @B@/zlib/1.3-1/a:/build/b", + "zlib: PATH gets several entries at once: @B@/zlib/1.3-1/a:@B@/zlib/1.3-1/b"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_multi_store.py b/tests/test_multi_store.py new file mode 100644 index 00000000..c5c9b4b2 --- /dev/null +++ b/tests/test_multi_store.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""Several read and write stores, each bucket with its own credentials (issue #130).""" + +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from bits_helpers import sync + +ARCH = "slc9_x86-64" +SPEC = {"package": "zlib", "version": "1.3", "remote_hashes": ["aa" * 20], "cachedTarball": ""} + + +@patch("bits_helpers.sync.Boto3RemoteSync._s3_init", new=MagicMock()) +@patch.dict(os.environ, {"BITS_AWS_KEYS_FILE": "/nonexistent"}) +class RemoteFromUrlTest(unittest.TestCase): + def test_lists_give_one_helper_per_store(self): + h = sync.remote_from_url("b3://prod, https://s3.cern.ch/pr", "b3://pr,b3://backup", ARCH, "/sw") + self.assertIsInstance(h, sync.MultiRemoteSync) + self.assertEqual([type(r).__name__ for r in h.readers], ["Boto3RemoteSync", "HttpRemoteSync"]) + self.assertEqual(h.readers[1].remoteStore, "https://s3.cern.ch/swift/v1/pr") + self.assertEqual([r.writeStore for r in h.readers], ["", ""]) # readers never upload + self.assertEqual([w.writeStore for w in h.writers], ["pr", "backup"]) + self.assertEqual(h.writeStore, "pr,backup") + + def test_write_stores_alone_are_read_too(self): + h = sync.remote_from_url("", "b3://pr,b3://backup", ARCH, "/sw") + self.assertEqual([r.remoteStore for r in h.readers], ["pr", "backup"]) + + def test_two_buckets_get_two_clients(self): + h = sync.remote_from_url("b3://prod", "b3://pr", ARCH, "/sw") + self.assertIsInstance(h, sync.DualRemoteSync) + self.assertEqual((h.reader.remoteStore, h.reader.writeStore), ("prod", "")) + self.assertEqual((h.writer.remoteStore, h.writer.writeStore), ("pr", "pr")) + self.assertIsInstance(sync.remote_from_url("b3://pr", "b3://pr", ARCH, "/sw"), + sync.Boto3RemoteSync) # one store: as before + + +class MultiRemoteSyncTest(unittest.TestCase): + def setUp(self): + self.work = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.work, True) + self.readers = [MagicMock(remoteStore="prod"), MagicMock(remoteStore="pr")] + self.writers = [MagicMock(writeStore="pr"), MagicMock(writeStore="backup")] + self.multi = sync.MultiRemoteSync(self.readers, self.writers) + self.multi.workdir, self.multi.architecture = self.work, ARCH + + def test_reads_stop_at_the_first_store_that_has_it(self): + def fetch(spec): + d = os.path.join(self.work, sync.resolve_store_path(ARCH, spec["remote_hashes"][0])) + os.makedirs(d) + open(os.path.join(d, "zlib-1.3-1.%s.tar.gz" % ARCH), "w").close() + self.readers[0].fetch_tarball.side_effect = fetch + self.multi.fetch_tarball(SPEC) + self.readers[1].fetch_tarball.assert_not_called() + other = dict(SPEC, remote_hashes=["bb" * 20]) # not in the first store + self.readers[0].fetch_tarball.side_effect = OSError("403") + self.multi.fetch_tarball(other) # a failing store is skipped + self.readers[1].fetch_tarball.assert_called_once_with(other) + self.readers[0].fetch_source.return_value = False + self.readers[1].fetch_source.return_value = True + self.assertTrue(self.multi.fetch_source("c", "f", "/d")) + self.readers[0].list_store_tarballs.return_value = [] + self.readers[1].list_store_tarballs.return_value = ["x"] + self.assertEqual(self.multi.list_store_tarballs(ARCH, "aa"), ["x"]) + self.writers[0].list_store_tarballs.return_value = ["w"] + self.readers[1].list_store_tarballs.return_value = [] + self.assertEqual(self.multi.list_store_tarballs(ARCH, "aa"), []) # write stores not read + + def test_revision_markers_of_every_store_are_merged(self): + for helper, markers in zip(self.readers + self.writers, + ({"1": "a"}, {"1": "b", "2": "c"}, {"3": "d"}, RuntimeError("x"))): + if isinstance(markers, Exception): + helper.read_rev_markers.side_effect = markers + else: + helper.read_rev_markers.return_value = markers + self.assertEqual(self.multi.read_rev_markers("zlib", "1.3", ARCH), + {"1": "a", "2": "c", "3": "d"}) + + def test_uploads_go_to_every_write_store(self): + self.multi.upload_symlinks_and_tarball(SPEC) + self.multi.upload_source("/p", "c", "f") + for w in self.writers: + w.upload_symlinks_and_tarball.assert_called_once_with(SPEC) + w.upload_source.assert_called_once_with("/p", "c", "f") + self.multi.upload_symlinks_and_tarball(dict(SPEC, cachedTarball="/x.tar.gz")) + self.assertEqual(self.writers[0].upload_symlinks_and_tarball.call_count, 1) # recalled + self.writers[0].writestore_has_tarball.return_value = True + self.writers[1].writestore_has_tarball.return_value = False + self.assertFalse(self.multi.writestore_has_tarball(SPEC)) + self.multi.writeStore = "" # a development package: no uploads at all + self.assertEqual([w.writeStore for w in self.writers], ["", ""]) + + def test_unreachable_read_stores_are_dropped_one_by_one(self): + multi = sync.MultiRemoteSync(self.readers, []) + with patch("bits_helpers.sync.store_unreachable", side_effect=["ConnectionError", ""]): + kept, down = sync.drop_unreachable(multi) + self.assertEqual((kept, down), (self.readers[1], [("prod", "ConnectionError")])) + self.assertEqual(sync.drop_unreachable(self.multi), (self.multi, [])) # it writes + + +class S3SettingsTest(unittest.TestCase): + def setUp(self): + self.keys = tempfile.NamedTemporaryFile("w", delete=False) + self.keys.write("AWS_ACCESS_KEY_ID=global-key\nAWS_SECRET_ACCESS_KEY=global-secret\n" + "[bucket cms-pr] # the PR bucket\naws_access_key_id = pr-key\naws_secret_access_key = pr-secret\n" + "endpoint_url = https://pr.example\n") + self.keys.close() + os.chmod(self.keys.name, 0o600) + self.addCleanup(os.unlink, self.keys.name) + env = {"BITS_AWS_KEYS_FILE": self.keys.name, "AWS_ACCESS_KEY_ID": "run-key", + "AWS_SECRET_ACCESS_KEY": "run-secret", "S3_ENDPOINT_URL": "https://run.example"} + p = patch.dict(os.environ, env) + p.start() + self.addCleanup(p.stop) + + def test_a_bucket_section_its_variables_else_the_run(self): + pr = sync.s3_settings("cms-pr") + self.assertEqual((pr["access_key"], pr["secret_key"], pr["endpoint"]), + ("pr-key", "pr-secret", "https://pr.example")) + prod = sync.s3_settings("cms-prod") + self.assertEqual((prod["access_key"], prod["endpoint"]), ("run-key", "https://run.example")) + with patch.dict(os.environ, {"BITS_S3_CMS_PR_ACCESS_KEY_ID": "env-key", + "BITS_S3_CMS_PR_SECRET_ACCESS_KEY": "env-secret"}): + self.assertEqual(sync.s3_settings("cms-pr")["secret_key"], "env-secret") + with patch.dict(os.environ, {"BITS_S3_CMS_PROD_ACCESS_KEY_ID": "half"}): + # A key without its secret is not mixed with another source's secret. + self.assertEqual(sync.s3_settings("cms-prod")["secret_key"], None) + + def test_the_run_settings_ignore_bucket_sections(self): + self.assertEqual(sync._load_aws_keys_file(self.keys.name)["access_key"], "global-key") + + def test_other_sections_still_count_for_the_run(self): + with tempfile.NamedTemporaryFile("w") as f: # as copied from ~/.aws/credentials + f.write("[cern]\naws_access_key_id = k\naws_secret_access_key = s\n") + f.flush() + os.chmod(f.name, 0o600) + self.assertEqual(sync._load_aws_keys_file(f.name)["access_key"], "k") + self.assertEqual(sync._load_aws_keys_file(f.name, "cern"), {}) + + def test_the_client_uses_its_buckets_credentials(self): + try: + import boto3 # noqa: F401 + except ImportError: + self.skipTest("boto3 is not installed") + with patch("boto3.client") as client: + sync.Boto3RemoteSync("b3://cms-pr", "", ARCH, "/sw") + kw = client.call_args.kwargs + self.assertEqual((kw["aws_access_key_id"], kw["endpoint_url"]), ("pr-key", "https://pr.example")) + + +class TemplateTest(unittest.TestCase): + def test_recipes_see_no_s3_secret(self): + import subprocess + with open(os.path.join(os.path.dirname(sync.__file__), "build_template.sh")) as f: + text = f.read() + start = text.index("# Cleanup variables which should not be exposed") + snippet = text[start:text.index("unset _bits_v", start)] + "unset _bits_v\nenv\n" + env = dict(os.environ, AWS_ACCESS_KEY_ID="1", AWS_SESSION_TOKEN="2", + BITS_S3_CMS_PR_SECRET_ACCESS_KEY="3", BITS_AWS_SECRET_ACCESS_KEY="4", + BITS_S3_CMS_PR_ENDPOINT_URL="https://pr.example") + out = subprocess.run(["bash", "-c", snippet], env=env, capture_output=True, + text=True, check=True).stdout + self.assertNotRegex(out, r"(?m)^(AWS_ACCESS_KEY_ID|AWS_SESSION_TOKEN|BITS_S3_CMS_PR_SECRET" + r"_ACCESS_KEY|BITS_AWS_SECRET_ACCESS_KEY)=") + self.assertIn("BITS_S3_CMS_PR_ENDPOINT_URL=https://pr.example", out) # not a secret + + +class SystemCheckEnvTest(unittest.TestCase): + def test_recipe_checks_see_no_s3_secret(self): + from bits_helpers.cmd import DockerRunner + with patch.dict(os.environ, {"BITS_S3_CMS_PR_SECRET_ACCESS_KEY": "3", "AWS_SESSION_TOKEN": "2"}): + with DockerRunner(None) as run: + _, out = run("env") + self.assertNotIn("BITS_S3_CMS_PR_SECRET_ACCESS_KEY", out) + self.assertNotIn("AWS_SESSION_TOKEN", out) + + +class DoctorTest(unittest.TestCase): + def test_a_package_is_looked_up_in_every_store(self): + from bits_helpers import doctor + answers = {"https://a": (doctor.WARN, "not found"), "https://b": (doctor.PASS, "available")} + with patch.object(doctor, "_probe_tarball_in_store", side_effect=lambda s, a, u, i: answers[u]): + self.assertEqual(doctor._probe_in_stores({}, ARCH, "https://a,https://b"), + (doctor.PASS, "available in https://b")) + self.assertEqual(doctor._probe_in_stores({}, ARCH, "https://a"), (doctor.WARN, "not found")) + + + +class HttpsStoresTest(unittest.TestCase): + def test_each_https_store_keeps_its_host(self): + from bits_helpers.publish import _normalize_s3_store + with patch.dict(os.environ, {"BITS_AWS_KEYS_FILE": "/nonexistent"}, clear=False): + for k in [k for k in os.environ if k.startswith("BITS_S3_")]: + os.environ.pop(k) + self.assertEqual(_normalize_s3_store("https://a.example/prod"), "b3://prod") + self.assertEqual(_normalize_s3_store("https://b.example/swift/v1/pr"), "b3://pr") + self.assertEqual(sync.s3_settings("prod")["endpoint"], "https://a.example") + self.assertEqual(sync.s3_settings("pr")["endpoint"], "https://b.example") + + +class OperationsOnEveryStoreTest(unittest.TestCase): + """publish, certify, gc and compliance with several write stores.""" + + def _args(self, **kw): + from argparse import Namespace + return Namespace(**dict(dict(workDir="/sw", architecture=ARCH, dryRun=False, + fromManifest=None, package=None, publishView=None), **kw)) + + def test_publish_goes_to_every_store(self): + from bits_helpers import publish + with patch.object(publish, "_publish_from_manifest") as pub: + publish.doPublish(self._args(publishStore="b3://a, b3://b"), MagicMock()) + self.assertEqual([c.args[2] for c in pub.call_args_list], ["b3://a", "b3://b"]) + + def test_certify_publishes_everywhere_and_certifies_the_first(self): + from bits_helpers import publish + bom = lambda sha: [(ARCH, {"packages": [{"hash": "h1", "tarball_sha256": sha}]})] + args = self._args(publishStore="b3://a,b3://b", approval="passkey", certifier=None, + gitlabToken=None, certifyGroup=None, manifestsRemote=None, certifyRef=None) + with patch.object(publish, "_orchestrator_marker", return_value={"manifests": "m"}) as mark, \ + patch.object(publish, "_publish_from_manifest", + side_effect=[("bid", bom("x"), {}), ("bid", bom("y"), {})]) as pub, \ + patch.object(publish, "_preapprove_build") as approve, \ + patch.object(publish, "warning") as warn, \ + patch("bits_helpers.orchestrated.describe", return_value="the orchestrator"): + publish.doCertify(args, MagicMock()) + self.assertEqual(mark.call_args.args[0], "b3://a") + self.assertEqual([c.args[2] for c in pub.call_args_list], ["b3://a", "b3://b"]) + self.assertEqual(approve.call_args.args[3], bom("x")) # the first store's + self.assertIn("h1@" + ARCH, warn.call_args.args[3]) # b holds another tarball + + def test_gc_sweeps_every_store(self): + from bits_helpers import gc + writers = {"b3://a": MagicMock(remoteStore="a"), "b3://b": MagicMock(remoteStore="b")} + with patch("bits_helpers.sync.remote_from_url", side_effect=lambda r, w, a, d: writers[r]), \ + patch.object(gc, "_localize_manifest", return_value="/m.json"), \ + patch.object(gc, "gather_s3_store_objects", return_value=[]) as gather, \ + patch.object(gc, "collect_garbage", + return_value=dict(verified=True, delete=[], kept=0, young=0, unsafe=0)): + gc.doGc(self._args(gcStore="b3://a,b3://b", trustManifest="ma,mb", graceDays=7, + allowEmpty=False), MagicMock()) + parser = MagicMock(error=MagicMock(side_effect=SystemExit)) + with self.assertRaises(SystemExit): # one manifest for two stores + gc.doGc(self._args(gcStore="b3://a,b3://b", trustManifest="m", graceDays=7, + allowEmpty=False), parser) + self.assertEqual([c.args[1] for c in gather.call_args_list], ["a", "b"]) + self.assertIn("one --trust-manifest per store", parser.error.call_args.args[0]) + + def test_compliance_enforces_on_every_store(self): + from bits_helpers import compliance + with tempfile.TemporaryDirectory() as recipes: + with open(os.path.join(recipes, "zlib.sh"), "w") as f: + f.write("package: zlib\nversion: '1'\nlicense: Zlib\n---\n") + args = self._args(packages=[], recipesDir=recipes, enforce=True, noStoreCheck=False, + complianceStore="b3://a,b3://b", enforceKey=None) + with patch.object(compliance, "enforce_store", side_effect=[0, 2]) as enforce, \ + patch.object(compliance, "_console", return_value=None): + self.assertEqual(compliance.doCompliance(args, MagicMock()), 2) + self.assertEqual([c.args[0] for c in enforce.call_args_list], ["b3://a", "b3://b"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_orchestrated.py b/tests/test_orchestrated.py new file mode 100644 index 00000000..ed7fb3fd --- /dev/null +++ b/tests/test_orchestrated.py @@ -0,0 +1,348 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""bits in a store whose certified set bits-orchestrator owns (MANIFESTS/orchestrator.json).""" + +import io +import json +import os +import shutil +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from bits_helpers import certify, compliance, orchestrated, publish, stale_boms + +MARKER = {"issuer": "bits-orchestrator/bkt", "manifests": "MANIFESTS/common-manifest", + "since": "2026-10-20T08:00:00Z"} + + +class _Err(Exception): + def __init__(self, code): + super().__init__(code) + self.response = {"Error": {"Code": code}} + + +class _S3: + def __init__(self, objects=None, fail=None, listable=False): + self.objects, self.fail, self.listable = dict(objects or {}), fail, listable + + def get_object(self, Bucket, Key): + if self.fail: + raise _Err(self.fail) + if Key not in self.objects: + raise _Err("NoSuchKey") + return {"Body": io.BytesIO(self.objects[Key])} + + def list_objects_v2(self, Bucket, Prefix, MaxKeys): + if not self.listable: + raise _Err("AccessDenied") + return {"Contents": [{"Key": k} for k in self.objects if k.startswith(Prefix)]} + + +class _Parser: + class Error(Exception): + pass + + def error(self, msg): + raise self.Error(msg) + + +class ReadTest(unittest.TestCase): + + def test_no_marker(self): + self.assertIsNone(orchestrated.read(_S3(), "bkt")) + + def test_the_marker(self): + s3 = _S3({orchestrated.MARKER: json.dumps(MARKER).encode()}) + self.assertEqual(orchestrated.read(s3, "bkt"), MARKER) + + def test_an_unreadable_marker_still_says_the_orchestrator_owns_the_set(self): + m = orchestrated.read(_S3({orchestrated.MARKER: b"{not json"}), "bkt") + self.assertEqual(m["issuer"], "bits-orchestrator") + + def test_a_store_that_cannot_say(self): + with self.assertRaises(orchestrated.Unknown): + orchestrated.read(_S3(fail="AccessDenied"), "bkt") + + def test_a_403_for_a_marker_the_listing_does_not_show_is_no_marker(self): + self.assertIsNone(orchestrated.read(_S3(fail="AccessDenied", listable=True), "bkt")) + + def test_a_store_that_is_not_s3_has_no_orchestrator(self): + self.assertIsNone(orchestrated.for_store("rsync://host/path")) + + +class ConsoleTest(unittest.TestCase): + + def setUp(self): + self.calls = [] + self.answers = [] + + def call(url, obj=None, bearer=None): + self.calls.append((url, obj, bearer)) + return self.answers.pop(0) + self.console = orchestrated.Console("https://console.example/", "tok") + self.console._call = call + + def test_decertify_is_requested_as_the_token_holder(self): + self.answers = [(201, {"id": 7, "state": "approved"})] + a = self.console.decertify("lcg", "b1", "el9", "line one\nline two") + self.assertEqual(a["id"], 7) + url, body, bearer = self.calls[0] + self.assertEqual((url, bearer), ("https://console.example/ops/actions", "tok")) + self.assertEqual(body, {"kind": "decertify", "params": { + "group": "lcg", "build_id": "b1", "arch": "el9", "reason": "line one line two"}}) + + def test_a_refused_request_says_why(self): + self.answers = [(403, {"detail": "bob is not an admin for lcg"})] + with self.assertRaisesRegex(orchestrated.RequestError, "403.*not an admin"): + self.console.decertify("lcg", "b1") + + def test_settle_follows_the_action(self): + self.answers = [(200, {"id": 7, "state": "running"}), (200, {"id": 7, "state": "done"})] + with patch("bits_helpers.orchestrated.time.sleep"): + a = self.console.settle({"id": 7, "state": "approved"}) + self.assertEqual(a["state"], "done") + self.assertEqual(self.calls[-1][0], "https://console.example/ops/actions/7") + + def test_decertify_package_is_requested_as_the_token_holder(self): + self.answers = [(201, {"id": 8, "state": "approved"})] + self.console.decertify_package("lcg", "Secret", "compliance") + url, body, bearer = self.calls[0] + self.assertEqual((url, bearer), ("https://console.example/ops/actions", "tok")) + self.assertEqual(body, {"kind": "decertify-package", "params": { + "group": "lcg", "package": "Secret", "reason": "compliance"}}) + + def test_the_groups_of_the_set(self): + self.answers = [(200, {"groups": {"ship": {}, "lcg": {}}, "entries": []})] + self.assertEqual(self.console.groups(), ["lcg", "ship"]) + + def test_the_outcome_of_a_decertify(self): + out = orchestrated.Console.outcome + self.assertIn("2 BOM(s) without it", out({"state": "done", "result": { + "boms_replaced": 2, "boms_removed": 1}})[1]) + self.assertEqual(out({"state": "done", "result": None}), (True, "done")) + self.assertTrue(out({"state": "approved"})[0]) + ok, words = out({"state": "done", "result": json.dumps({"still_certified_by": ["ship"]})}) + self.assertTrue(ok) + self.assertIn("still certified by ship", words) + ok, words = out({"state": "failed", "error": "not in lcg's certified set"}) + self.assertFalse(ok) + self.assertIn("not in lcg", words) + + def test_a_token_does_not_follow_a_redirect(self): + import http.server + import threading + seen = [] + + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + seen.append((self.path, self.headers.get("Authorization"))) + self.send_response(302) + self.send_header("Location", "/elsewhere") + self.end_headers() + + def log_message(self, *a): + pass + srv = http.server.HTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=srv.handle_request, daemon=True).start() + from bits_helpers import sign_console + code, _ = sign_console._json_call("http://127.0.0.1:%d/ops/x" % srv.server_port, + bearer="tok") + srv.server_close() + self.assertEqual((code, seen), (302, [("/ops/x", "Bearer tok")])) + + def test_a_token_is_never_sent_in_clear(self): + console = orchestrated.Console("http://console.example", "tok") + with self.assertRaisesRegex(orchestrated.RequestError, "non-https"): + console.decertify("lcg", "b1") + + +class CertifyTest(unittest.TestCase): + + def args(self, **over): + base = dict(fromManifest="latest", dryRun=False, workDir="/tmp/sw", architecture="el9", + publishStore="https://s3.example/bkt", certifyGroup="lcg", + manifestsRemote=None, certifyRef=None, gitlabToken=None, + approval="passkey", console="https://c", consoleCafile=None, + consoleInsecure=False, certifier=None) + base.update(over) + return SimpleNamespace(**base) + + def run_certify(self, marker, **over): + bom = [("el9", {"build_id": "bid", "packages": []})] + with patch.object(publish, "_orchestrator_marker", return_value=marker), \ + patch.object(publish, "_publish_from_manifest", return_value=("bid", bom, {})), \ + patch.object(publish, "_submit_certification_mr") as mr, \ + patch("bits_helpers.forge.resolve_gitlab_token", return_value=None), \ + patch("bits_helpers.sign_console.preapprove_via_console") as pre: + publish.doCertify(self.args(**over), _Parser()) + return pre, mr + + def test_the_preapproval_is_the_end_no_token_no_mr(self): + pre, mr = self.run_certify(MARKER) + pre.assert_called_once() + mr.assert_not_called() + + def test_approval_none_only_publishes(self): + pre, mr = self.run_certify(MARKER, approval="none") + pre.assert_not_called() + mr.assert_not_called() + + def test_unmarked_still_needs_the_token_for_the_mr(self): + with self.assertRaises(_Parser.Error): + self.run_certify(None) + + def test_a_store_that_cannot_say_is_certified_through_an_mr(self): + with patch("bits_helpers.orchestrated.for_store", + side_effect=orchestrated.Unknown("cannot read")), \ + patch.object(publish, "warning") as warn: + self.assertIsNone(publish._orchestrator_marker("b3://bkt", "/tmp/sw", "el9")) + self.assertIn("merge request", warn.call_args[0][0]) + + +class SignTest(unittest.TestCase): + + def test_sign_refuses_a_store_the_orchestrator_signs(self): + with patch("bits_helpers.orchestrated.for_store", return_value=MARKER): + with self.assertRaisesRegex(_Parser.Error, "bits-orchestrator/bkt signs"): + certify._refuse_orchestrated("b3://bkt", "/tmp/sw", "el9", _Parser()) + + def test_sign_refuses_a_store_that_cannot_say(self): + with patch("bits_helpers.orchestrated.for_store", + side_effect=orchestrated.Unknown("x")): + with self.assertRaisesRegex(_Parser.Error, "cannot tell"): + certify._refuse_orchestrated("b3://bkt", "/tmp/sw", "el9", _Parser()) + + +class EnforceTest(unittest.TestCase): + """--enforce in a store the orchestrator certifies: decertify, never sign.""" + + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d, True) + with open(os.path.join(self.d, "secret.sh"), "w") as fh: + fh.write("package: Secret\nversion: \"1\"\nlicense: LicenseRef-Secret\n" + "redistributable: false\n---\n") + self.rec = compliance.scan_recipes(self.d) + self.objects = { + orchestrated.MARKER: json.dumps(MARKER), + "MANIFESTS/b1/h-1.el9.json": json.dumps({"build_id": "b1", "packages": [ + {"package": "Secret", "hash": "beef01", "effective_architecture": "el9"}, + {"package": "Good", "hash": "aa11", "effective_architecture": "el9"}]}), + } + self.deleted = [] + outer = self + + class S3: + def get_paginator(self, _m): + def paginate(Bucket, Prefix=""): + return [{"Contents": [{"Key": k} for k in sorted(outer.objects) + if k.startswith(Prefix) and k not in outer.deleted]}] + return SimpleNamespace(paginate=paginate) + + def get_object(self, Bucket, Key): + if Key not in outer.objects: + raise _Err("NoSuchKey") + return {"Body": io.BytesIO(outer.objects[Key].encode())} + + def delete_object(self, Bucket, Key): + outer.deleted.append(Key) + + def put_object(self, Bucket, Key, Body): + outer.objects[Key] = Body.decode() + + c = patch.object(compliance, "_s3_client", return_value=(S3(), "bkt", "https://s3.x")) + c.start() + self.addCleanup(c.stop) + env = patch.dict(os.environ, {"AWS_ACCESS_KEY_ID": "k", "AWS_SECRET_ACCESS_KEY": "s"}) + env.start() + self.addCleanup(env.stop) + + class Console: + def __init__(self, groups=("lcg",), state="done"): + self._groups, self.state, self.asked = list(groups), state, [] + + def groups(self): + return self._groups + + def check(self): + pass + + outcome = staticmethod(orchestrated.Console.outcome) + + def decertify_package(self, group, package, reason): + self.asked.append((group, package)) + return {"id": len(self.asked), "state": "approved"} + + def settle(self, action, timeout=60): + return dict(action, state=self.state, + result={"boms_replaced": 1, "boms_removed": 0}) + + def test_the_package_is_decertified_in_each_group(self): + console = self.Console(groups=("lcg", "ship")) + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + console=console), 0) + self.assertEqual(console.asked, [("lcg", "Secret"), ("ship", "Secret")]) + kept = json.loads(self.objects["MANIFESTS/b1/h-1.el9.json"]) # purged as before + self.assertEqual([p["package"] for p in kept["packages"]], ["Good"]) + + def test_a_package_only_the_signed_manifests_still_list_is_decertified(self): + del self.objects["MANIFESTS/b1/h-1.el9.json"] + self.objects["MANIFESTS/common-manifest-el9.json"] = json.dumps({"packages": [ + {"package": "secret", "hash": "beef01", "group": "lcg"}, + {"package": "Good", "hash": "aa11", "group": "lcg"}]}) + console = self.Console() + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + console=console), 0) + self.assertEqual(console.asked, [("lcg", "secret")]) + + def test_a_decertify_still_waiting_for_the_worker_is_requested(self): + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + console=self.Console(state="approved")), 0) + + def test_no_group_to_decertify_in_fails_before_anything_is_touched(self): + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + console=self.Console(groups=())), 1) + self.assertEqual(json.loads(self.objects["MANIFESTS/b1/h-1.el9.json"])["packages"][0] + ["package"], "Secret") + + def test_a_decertify_that_fails_fails_the_enforcement(self): + console = self.Console(state="failed") + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + console=console), 1) + + def test_a_key_is_refused_before_anything_is_touched(self): + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + key_pem="k.pem", console=self.Console()), 1) + self.assertEqual(self.deleted, []) + + def test_no_console_is_refused_before_anything_is_touched(self): + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d), 1) + self.assertEqual(self.deleted, []) + + def test_a_dry_run_needs_no_console(self): + self.assertEqual(compliance.enforce_store("b3://bkt", self.rec, self.d, + dry_run=True), 0) + self.assertEqual(self.deleted, []) + + +class StaleBomTargetsTest(unittest.TestCase): + + def test_group_from_the_directory_arch_as_the_set_files_it(self): + bom = {"build_id": "b1", "packages": [{"hash": "h", "tarball_sha256": "s", + "effective_architecture": "el9"}]} + wanted, bad = stale_boms.decertify_targets({ + "/c/manifests/lcg/b1.el9.x.json": bom, + "/c/manifests/lcg/b1.el9.y.json": bom, + "/c/elsewhere/b1.json": bom, + "/c/manifests/lcg/noid.json": dict(bom, build_id="noid.json"), + "/c/manifests/lcg/empty.json": {"build_id": "b2", "packages": []}}) + self.assertEqual(wanted, {("lcg", "b1", "el9"): ["/c/manifests/lcg/b1.el9.x.json", + "/c/manifests/lcg/b1.el9.y.json"]}) + self.assertEqual(bad, ["/c/elsewhere/b1.json", "/c/manifests/lcg/empty.json", + "/c/manifests/lcg/noid.json"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_overlay.py b/tests/test_overlay.py index 827ed9b6..0fb73beb 100644 --- a/tests/test_overlay.py +++ b/tests/test_overlay.py @@ -58,6 +58,28 @@ def test_family_packages_are_found(self): self.assertEqual(sorted(records), ["ROOT", "evtgen"]) self.assertTrue(records["evtgen"][0].endswith("MCGenerators/evtgen/1-1")) + def test_closure_only(self): + # Another build's packages and versions in the work dir stay out. + from bits_helpers.overlay.lcg import closure_of, collect + with tempfile.TemporaryDirectory() as wd: + a = os.path.join(wd, "x86_64-el9-gcc14-opt") + for pkg, ver in (("ROOT", "1"), ("ROOT", "2"), ("Athena", "1")): + d = os.path.join(a, pkg, ver + "-1") + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + json.dump({"package": {"name": pkg, "version": ver, "revision": "1"}}, fh) + view = os.path.join(wd, "view.json") + with open(view, "w") as fh: + json.dump({"dependencies": {"recursive": {"build": [], "runtime": [ + {"name": "ROOT", "version": "1", "revision": "1"}, + {"name": "GCC-Toolchain", "version": "14", "revision": "1"}]}}}, fh) + records, warnings, errors = collect(wd, "x86_64-el9-gcc14-opt", closure_of(view)) + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + self.assertIn("GCC-Toolchain 14-1", warnings[0]) + self.assertEqual(list(records), ["ROOT"]) + self.assertTrue(records["ROOT"][0].endswith("ROOT/1-1")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pack_checksum.py b/tests/test_pack_checksum.py index e6266efe..41a830e1 100644 --- a/tests/test_pack_checksum.py +++ b/tests/test_pack_checksum.py @@ -1,6 +1,7 @@ """The tarball checksum written while packing (build_template.sh) and read back by the manifest and the store upload (checksum.tarball_checksum).""" +from pathlib import Path import hashlib import os import shutil @@ -14,7 +15,7 @@ def _pack_block(): """build_template.sh's packing branch, as bash (the template is %-formatted).""" - src = open(os.path.join(os.path.dirname(bits_helpers.__file__), "build_template.sh")).read() + src = Path(os.path.join(os.path.dirname(bits_helpers.__file__), "build_template.sh")).read_text() start = src.index('elif [ -z "$CACHED_TARBALL" ]; then') end = src.index('\nfi\nwait "$rsync_pid"', start) return "if false; then :\n" + src[start:end].replace("%%", "%") + "\nfi\n" diff --git a/tests/test_package_data.py b/tests/test_package_data.py new file mode 100644 index 00000000..5f3b2a64 --- /dev/null +++ b/tests/test_package_data.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Every file bits_helpers needs at run time is installed with it.""" + +import ast +from fnmatch import fnmatch +import os +import re +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class PackageDataTest(unittest.TestCase): + def test_non_python_files_are_package_data(self): + with open(os.path.join(ROOT, "pyproject.toml")) as f: + m = re.search(r"^bits_helpers = (\[.*\])$", f.read(), re.M) + patterns = ast.literal_eval(m.group(1)) + top = os.path.join(ROOT, "bits_helpers") + missing = [] + for directory, dirs, files in os.walk(top): + dirs[:] = [d for d in dirs if d != "__pycache__"] + for name in files: + rel = os.path.relpath(os.path.join(directory, name), top) + if not rel.endswith((".py", ".pyc")) and not any(fnmatch(rel, p) for p in patterns): + missing.append(rel) + self.assertEqual(missing, [], "not in pyproject.toml's package-data") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packagelist.py b/tests/test_packagelist.py index aec53575..f00b18b6 100644 --- a/tests/test_packagelist.py +++ b/tests/test_packagelist.py @@ -8,6 +8,8 @@ from unittest.mock import patch import tempfile +from shlex import quote + from bits_helpers.cmd import getstatusoutput from bits_helpers.packages import getPackageList @@ -58,6 +60,43 @@ prefer_system_replacement_specs: {} --- """), + "CONFIG_DIR/pin-owner.sh": dedent("""\ + package: pin-owner + version: "1" + requires: + - pin-dep = 2.0 + - pin-ovr + --- + """), + "CONFIG_DIR/pin-dep.sh": dedent("""\ + package: pin-dep + version: "1.0" + --- + """), + "CONFIG_DIR/pin-ovr.sh": dedent("""\ + package: pin-ovr + version: "1.0" + --- + """), + "CONFIG_DIR/sys-libxml2.sh": dedent("""\ + package: sys-libxml2 + version: "2.10.4" + prefer_system: '.*' + prefer_system_check: | + echo 2.9.13 + references: + - cpe:2.3:a:xmlsoft:libxml2:%(version)s + --- + """), + "CONFIG_DIR/sys-marked.sh": dedent("""\ + package: sys-marked + version: "1" + sbom: tool + prefer_system: '.*' + prefer_system_check: | + echo "found it, see 3.1 notes"; echo 'bits_system_version: 7.4.0' + --- + """), "CONFIG_DIR/sentinel-command.sh": dedent("""\ package: sentinel-command version: v1 @@ -136,7 +175,8 @@ def __call__(self): return self._contents -def getPackageListWithDefaults(packages, force_rebuild=(), satisfied=None): +def getPackageListWithDefaults(packages, force_rebuild=(), satisfied=None, system_info=None, + overrides=None, defaults_meta=None): specs = {} # getPackageList will mutate this def performPreferCheckWithTempDir(pkg, cmd): with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir: @@ -157,11 +197,13 @@ def performPreferCheckWithTempDir(pkg, cmd): performPreferCheck=performPreferCheckWithTempDir, performRequirementCheck=performPreferCheckWithTempDir, performValidateDefaults=lambda spec: (True, "", ["release"]), - overrides={"defaults-release": {}}, + overrides=dict({"defaults-release": {}}, **(overrides or {})), + defaults_meta=defaults_meta, taps={}, log=lambda *_: None, force_rebuild=force_rebuild, satisfied_requirements=satisfied, + system_info=system_info, ) return (specs, *return_values) @@ -195,6 +237,75 @@ def test_unfiltered_requires_keep_system_edges(self): self.assertIn("disable", specs[pkg]["unfiltered_requires"]["runtime"], pkg) self.assertIn("disable", specs["sbom-b"]["unfiltered_requires"]["build"]) + def _package_managers(self, rpm="", dpkg=""): + """Fake rpm and dpkg-query on PATH that answer *rpm* / *dpkg* for any name + ending in -libs, and log every call.""" + bindir = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, bindir, True) + for tool, answer in (("rpm", rpm), ("dpkg-query", dpkg)): + with open(os.path.join(bindir, tool), "w") as f: + f.write('#!/bin/sh\nfor a; do last=$a; done\n' + 'echo "$last" >> "%s/calls"\n' % bindir + + 'case "$last" in *-libs) printf "%%s\\n" "%s";; *) echo "package $last is not installed"; exit 1;; esac\n' + % answer) + os.chmod(os.path.join(bindir, tool), 0o755) + patcher = mock.patch.dict(os.environ, PATH=bindir + os.pathsep + os.environ["PATH"]) + patcher.start() + self.addCleanup(patcher.stop) + return os.path.join(bindir, "calls") + + def test_system_versions_come_from_the_checks(self): + self._package_managers() + info = {} + getPackageListWithDefaults(["sys-libxml2", "sys-marked", "disable"], system_info=info) + self.assertEqual(info["sys-libxml2"], {"version": "2.9.13", + "references": ["cpe:2.3:a:xmlsoft:libxml2:2.9.13:*:*:*:*:*:*:*"]}) + self.assertEqual(info["sys-marked"]["version"], "7.4.0") + self.assertEqual(info["sys-marked"]["sbom"], "tool") # sbom: tool + self.assertEqual(info["disable"], {"version": "", "references": []}) # it said nothing + + def test_the_package_manager_is_asked_first(self): + calls = self._package_managers( + rpm="rpm libxml2 (none) 2.9.13 6.el9_4 x86_64 libxml2-2.9.13-6.el9_4.src.rpm") + info = {} + getPackageListWithDefaults(["sys-libxml2", "disable"], system_info=info) + self.assertEqual(info["sys-libxml2"]["version"], "2.9.13-6.el9_4") + self.assertRegex(info["sys-libxml2"]["references"][0], + r"^pkg:rpm/[^/]+/libxml2@2\.9\.13-6\.el9_4\?arch=x86_64(&distro=[^&]+)?" + r"&upstream=libxml2-2\.9\.13-6\.el9_4\.src\.rpm$") + with open(calls) as f: # one query, for both packages + self.assertEqual(sorted(ln.strip() for ln in f), sorted( + ["sys-libxml2", "sys-libxml2-libs", "libsys-libxml2", "disable", "disable-libs", + "libdisable"] * 2)) + + def test_the_query_works_without_os_release(self): + # macOS has no /etc/os-release; "." of a missing file would end sh's script. + from bits_helpers.cmd import getstatusoutput + from bits_helpers.packages import _system_query, distro_packages + self._package_managers( + rpm="rpm libxml2 (none) 2.9.13 6.el9_4 x86_64 libxml2-2.9.13-6.el9_4.src.rpm") + script = _system_query([{"package": "sys-libxml2"}]).replace( + "/etc/os-release", os.path.join(tempfile.mkdtemp(), "no-os-release")) + err, out = getstatusoutput(script) + self.assertEqual(err, 0, out) + self.assertEqual(distro_packages(out)["sys-libxml2"][0], "2.9.13-6.el9_4") + + def test_dpkg_is_asked_too_and_held_packages_count(self): + self._package_managers(dpkg="hi deb sys-marked - 1:7.4.0-2 - amd64 marked@1:7.4.0-2") + info = {} + getPackageListWithDefaults(["sys-marked"], system_info=info) + self.assertEqual((info["sys-marked"]["version"], info["sys-marked"]["sbom"]), ("1:7.4.0-2", "tool")) + self.assertRegex(info["sys-marked"]["references"][0], + r"^pkg:deb/[^/]+/sys-marked@1:7\.4\.0-2\?arch=amd64(&distro=[^&]+)?&upstream=marked$") + + def test_who_pinned_a_version_is_recorded(self): + specs, *_ = getPackageListWithDefaults( + ["pin-owner"], overrides={"pin-ovr": {"version": "3.0"}}, + defaults_meta={"_version_profiles": {"pin-ovr": "lcg"}}) + self.assertEqual((specs["pin-dep"]["version"], specs["pin-dep"]["pinned_by"]), ("2.0", "requires:pin-owner")) + self.assertEqual((specs["pin-ovr"]["version"], specs["pin-ovr"]["pinned_by"]), ("3.0", "defaults:lcg")) + self.assertNotIn("pinned_by", specs["pin-owner"]) + def test_satisfied_system_requirements_are_collected(self): satisfied = set() specs, systemPkgs, _, failed, _ = getPackageListWithDefaults(["sbom-uses-sysreq"], @@ -286,10 +397,6 @@ def test_force_rebuild_command_line(self) -> None: -if __name__ == '__main__': - unittest.main() - - RECIPES["CONFIG_DIR/pinned.sh"] = dedent("""\ package: pinned version: v1 @@ -361,3 +468,38 @@ def test_source_override_sets_the_dir(self): def test_other_override_does_not(self): self.assertIsNone(self._dir({"pinned": {"env": {"X": "1"}}})) self.assertIsNone(self._dir({})) + + +class SystemVersionTestCase(unittest.TestCase): + def test_a_version_line_or_the_marker(self): + from bits_helpers.packages import system_version + self.assertEqual(system_version("2.9.13\n"), "2.9.13") + self.assertEqual(system_version("cmake version 3.26.5\n3.26.5"), "3.26.5") + self.assertEqual(system_version("bits_system_version: 1.2\n9.9.9"), "1.2") + self.assertEqual(system_version("OpenSSL 3.0.7 1 Nov 2022"), "") # not only a version + self.assertEqual(system_version(""), "") + self.assertEqual(system_version("bits_system_version:\nv2.1"), "2.1") # no value: the line after + + def test_the_distribution_packages(self): + from bits_helpers.packages import distro_packages, _query_names, _system_query + self.assertEqual(distro_packages( + "== zlib\npackage zlib is not installed\nrpm zlib 1 1.2.11 41.el9 x86_64 zlib-1.2.11-41.el9.src.rpm\n" + "rpm zlib-libs 1 1.2.11 41.el9 x86_64 zlib-1.2.11-41.el9.src.rpm\n== none\n" + "== libxml2\ndeb libxml2 - 1:2.9.14+dfsg-1.3 - amd64 libxml2@1:2.9.14+dfsg-1.3\nos debian 12\n"), { + "zlib": ("1:1.2.11-41.el9", "pkg:rpm/debian/zlib@1.2.11-41.el9?arch=x86_64&distro=debian-12" + "&epoch=1&upstream=zlib-1.2.11-41.el9.src.rpm"), + "libxml2": ("1:2.9.14+dfsg-1.3", "pkg:deb/debian/libxml2@1:2.9.14+dfsg-1.3?arch=amd64&distro=debian-12")}) + self.assertEqual(distro_packages("== libssl3\ndeb libssl3 - 3.0.13-0u3 - amd64 openssl@3.0.13-0u3\n"), + {"libssl3": ("3.0.13-0u3", "pkg:deb/debian/libssl3@3.0.13-0u3?arch=amd64&upstream=openssl")}) + self.assertEqual(distro_packages("== a\ndeb a - 2-1 - all b@1-1\n")["a"][1], + "pkg:deb/debian/a@2-1?arch=all&upstream=b%401-1") + self.assertEqual(distro_packages("os ubuntu 24.04\n"), {}) + self.assertEqual(distro_packages("== a\ndeb a - 1 - all\nos debian \n")["a"][1], + "pkg:deb/debian/a@1?arch=all") # sid: no VERSION_ID + self.assertEqual(_query_names({"package": "OpenSSL"}), ["OpenSSL", "openssl", "openssl-libs", "libopenssl"]) + self.assertEqual(_query_names({"package": "zlib", "system_package": "zlib1g"}), ["zlib1g"]) + self.assertIn("echo %s\nq zlib1g\n" % quote("== it's"), + _system_query([{"package": "it's", "system_package": "zlib1g"}])) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_plan.py b/tests/test_plan.py index 0eeaf54c..3a715df4 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -2,6 +2,7 @@ import os import tempfile import unittest +from types import SimpleNamespace from unittest.mock import patch from bits_helpers import plan @@ -70,7 +71,10 @@ def test_remote(self): def test_remote_unsigned(self): store = _Store(["zlib-1.3-4.%s.tar.gz" % ARCH]) self.assertEqual(self._classify(store, trusted={})[0], plan.REMOTE_UNSIGNED) - self.assertEqual(self._classify(store, trusted={RH: "sha"})[0], plan.REMOTE) + self.assertEqual(self._classify(store, trusted={(ARCH, RH): "sha"})[0], plan.REMOTE) + # Certified for another architecture only: not this tarball. + self.assertEqual(self._classify(store, trusted={("other-arch", RH): "sha"})[0], + plan.REMOTE_UNSIGNED) def test_local_preferred_over_remote(self): self._tarball(RH, "2") @@ -105,7 +109,7 @@ def test_hash_policy_unsigned_remote_falls_back_to_local_hash(self): store = _Store(["zlib-1.3-%s.%s.tar.gz" % (RH, ARCH)]) self.assertEqual(self._classify(store, trusted={}, **spec), (plan.REMOTE_UNSIGNED, LH, LH)) - self.assertEqual(self._classify(store, trusted={RH: "sha"}, **spec), + self.assertEqual(self._classify(store, trusted={(ARCH, RH): "sha"}, **spec), (plan.REMOTE, RH, RH)) def test_hash_policy_fallback_does_not_change_virtual_package_hash(self): @@ -126,11 +130,67 @@ def test_summary(self): with patch("bits_helpers.plan.banner") as b: plan.print_plan(rows, store_checked=True) text = b.call_args[0][0] % b.call_args[0][1:] - self.assertIn("2 reused (1 installed, 0 local tarball, 1 from remote store), 2 to build", text) + self.assertIn("2 reused (1 installed, 0 local tarball, 1 from store), 2 to build", text) + self.assertIn("1 to build is in the remote store but not reused: no verified signed manifest", text) self.assertNotIn("not checked", text) with patch("bits_helpers.plan.banner") as b: plan.print_plan(rows, store_checked=False) self.assertIn("remote store not checked", b.call_args[0][0] % b.call_args[0][1:]) + with patch("bits_helpers.plan.banner") as b, \ + patch("bits_helpers.build._try_import_trust", return_value=None): + plan.print_plan(rows, store_checked=None) + text = b.call_args[0][0] % b.call_args[0][1:] + self.assertIn("no remote store used", text) + self.assertIn("its signature cannot be checked without the Python package cryptography", text) + + +class NarrowPlanTest(unittest.TestCase): + def test_fits_a_terminal(self): + rows = [("jupyter_contrib_nbextensions_and_more", "0.7.0", plan.REMOTE, "", "lcg.bits@94a3bce60b"), + ("dask_histogram", "2025.2.0", plan.REMOTE, "", "lcg.bits@94a3bce60b"), + ("astropy_iers_data", "0.2026.5.25.1.14.13", plan.REMOTE, "", "lcg.bits@94a3bce60b"), + ("LCG", "1", plan.BUILD, "", ".@43badb4")] + with patch("bits_helpers.plan.banner") as b: + plan.print_plan(rows, store_checked=True) + lines = (b.call_args[0][0] % b.call_args[0][1:]).splitlines() + self.assertIn("Recipes from lcg.bits@94a3bce60b, unless shown", lines) + self.assertEqual(max(len(x) for x in lines if x.startswith("dask")), 58) + self.assertTrue([x for x in lines if x.startswith("LCG ")][0].endswith("build [.@43badb4]")) + self.assertTrue(all(len(x) <= 76 for x in lines if not x.startswith(("jupyter", "astropy", "0 reused")))) + + +class ListingsTest(unittest.TestCase): + def test_listed_once_in_parallel_then_from_the_cache(self): + calls = [] + + class Helper: + def list_store_tarballs(self, arch, h): + calls.append((arch, h)) + if h == "bad": + raise OSError("timeout") + return ["%s.tar.gz" % h] + lister = plan._Listings(Helper()) + lister.prefetch([("a", "1"), ("a", "2"), ("a", "1"), ("a", "bad")]) + self.assertEqual(sorted(calls), [("a", "1"), ("a", "2"), ("a", "bad")]) + self.assertEqual(lister.list_store_tarballs("a", "2"), ["2.tar.gz"]) + self.assertEqual(lister.list_store_tarballs("a", "3"), ["3.tar.gz"]) # not foreseen + with self.assertRaises(OSError): # failed: asked again + lister.list_store_tarballs("a", "bad") + self.assertEqual(len(calls), 5) + + + def test_packages_already_here_are_not_listed(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + work = tmp.name + d = os.path.join(work, "TARS", ARCH, "store", RH[:2], RH) + os.makedirs(d) + open(os.path.join(d, "zlib-1.3-1.%s.tar.gz" % ARCH), "w").close() + specs = {"zlib": _spec(), "curl": _spec(package="curl", remote_hashes=["cc" + "0" * 38])} + args = SimpleNamespace(defaults=["release"], architecture=ARCH) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.hashing.storeHashes"): + keys = plan._foreseen(["zlib", "curl"], specs, args, "slc9_x86-64", work, False) + self.assertEqual(keys, [(ARCH, "cc" + "0" * 38)]) class StoreListableTest(unittest.TestCase): @@ -173,7 +233,7 @@ def test_overlay(self): class PlanBuildTest(unittest.TestCase): def test_devel_disables_write_store_for_later_packages(self): import types - specs = {"a": _spec(package="a", is_devel_pkg=True), "b": _spec(package="b")} + specs = {"a": _spec(package="a", is_devel_pkg=True), "b": _spec(package="b", requires=["a"])} args = types.SimpleNamespace(defaults=["release"], architecture=ARCH) cfg = types.SimpleNamespace(require_signed_reuse=False, reuse_overlay=None) store = types.SimpleNamespace(writeStore="b3://bucket") @@ -185,5 +245,104 @@ def test_devel_disables_write_store_for_later_packages(self): self.assertEqual(specs["b"]["hash"], LH) # local hash: uploads are off after a devel pkg +class NeededTest(unittest.TestCase): + """Issue #141: a reused package's build_requires are not fetched.""" + + def _specs(self): + sp = lambda name, runtime=(), build=(): { + "package": name, "version": "1", "runtime_requires": list(runtime), + "build_requires": list(build), "untracked_requires": [], + "requires": list(runtime) + list(build) + ["defaults-release"]} + return {"defaults-release": sp("defaults-release"), + "gcc-prerequisites": sp("gcc-prerequisites", build=["rocm-sources"]), + "rocm-sources": sp("rocm-sources"), + "zlib": sp("zlib"), + "gcc": sp("gcc", runtime=["zlib"], build=["gcc-prerequisites"]), + "app": sp("app", runtime=["gcc"], build=["cmake"]), "cmake": sp("cmake")} + + ORDER = ["defaults-release", "rocm-sources", "gcc-prerequisites", "zlib", "gcc", "cmake", "app"] + + def test_a_reused_packages_build_requires_are_not_needed(self): + states = dict.fromkeys(self.ORDER, plan.BUILD) + states.update(gcc=plan.REMOTE, zlib=plan.REMOTE) + need = plan.needed(self.ORDER, self._specs(), states, ["app"]) + self.assertEqual(need, {"defaults-release", "app", "cmake", "gcc", "zlib"}) + + def test_a_built_package_needs_everything_and_reuse_of_all_needs_runtime_only(self): + specs = self._specs() + built = plan.needed(self.ORDER, specs, dict.fromkeys(self.ORDER, plan.BUILD), ["app"]) + self.assertEqual(built, set(self.ORDER)) + reused = plan.needed(self.ORDER, specs, dict.fromkeys(self.ORDER, plan.INSTALLED), ["app"]) + self.assertEqual(reused, {"defaults-release", "app", "gcc", "zlib"}) + # --only-deps: the main package is built later, so its build_requires stay. + deps = plan.needed(self.ORDER, specs, dict.fromkeys(self.ORDER, plan.INSTALLED), ["app"], + build_targets={"app"}) + self.assertIn("cmake", deps) + + def test_the_plan_names_who_does_not_need_it(self): + specs = self._specs() + states = dict.fromkeys(self.ORDER, plan.REMOTE) + need = plan.needed(self.ORDER, specs, states, ["app"], build_targets={"app"}) + rows = {r[0]: r for r in plan.plan_rows(self.ORDER, specs, states, need, + source=lambda p: "lcg.bits@abc")} + self.assertNotIn("defaults-release", rows) + self.assertEqual(rows["gcc-prerequisites"][2:4], + (plan.NOT_NEEDED, " (build-only, for reused gcc)")) + self.assertEqual(rows["gcc"][2:], (plan.REMOTE, "", "lcg.bits@abc")) + with patch("bits_helpers.plan.banner") as b: + plan.print_plan(list(rows.values()), store_checked=True, title="Build plan") + text = b.call_args[0][0] % b.call_args[0][1:] + self.assertTrue(text.startswith("Build plan")) + self.assertIn("\nRecipes from lcg.bits@abc, unless shown\n", text) + self.assertNotIn("[lcg.bits@abc]", text) + self.assertIn("4 reused (0 installed, 0 local tarball, 4 from store), 0 to build," + " 2 not needed", text) + with patch("bits_helpers.plan.banner") as b: + plan.print_plan(list(rows.values()), store_checked=None) + self.assertNotIn("not checked", b.call_args[0][0] % b.call_args[0][1:]) + + + def test_what_defaults_and_development_packages_require_is_needed(self): + specs = self._specs() + specs["defaults-release"]["requires"] = ["zlib"] + specs["cmake"]["is_devel_pkg"] = True + order = ["zlib", "defaults-release"] + self.ORDER[1:] + need = plan.needed(order, specs, dict.fromkeys(order, plan.REMOTE), ["app"]) + self.assertEqual(need, {"defaults-release", "zlib", "app", "gcc", "cmake"}) + + def test_fetch_build_requires_needs_everything(self): + specs, states = self._specs(), dict.fromkeys(self.ORDER, plan.REMOTE) + for fetch, not_needed in ((False, {"cmake", "gcc-prerequisites", "rocm-sources"}), + (True, set())): + cfg = SimpleNamespace(fetch_build_requires=fetch) + with patch.object(plan, "plan", return_value=(states, set(), True)), \ + patch.object(plan, "banner"): + _, _, need = plan.decide(self.ORDER, specs, None, "/sw", None, "", cfg, None, + targets=["app"]) + self.assertEqual(set(self.ORDER) - need, not_needed) + + def test_not_needed_packages_adopt_the_planned_hash(self): + specs, planned = self._specs(), self._specs() + planned["cmake"].update(hash="c" * 40, revision="2", deps_hash="x") + plan.adopt(specs, planned, {"cmake"}) + self.assertEqual((specs["cmake"]["hash"], specs["cmake"]["revision"], specs["cmake"]["not_needed"]), + ("c" * 40, "2", True)) + self.assertNotIn("deps_hash", specs["cmake"]) + + def test_a_reused_package_hashes_as_planned_without_its_build_requires_built(self): + import copy + from bits_helpers.hashing import storeHashes + sp = lambda name, build=(): {"package": name, "version": "1", "recipe": name, + "commit_hash": "0", "is_devel_pkg": False, + "requires": list(build), "build_requires": list(build)} + specs = {"cmake": sp("cmake"), "app": sp("app", ["cmake"])} + planned = copy.deepcopy(specs) + for p in ("cmake", "app"): + storeHashes(p, planned, considerRelocation=False) + planned[p]["hash"], planned[p]["revision"] = planned[p]["remote_revision_hash"], "1" + plan.adopt(specs, planned, {"cmake"}) # the loop then starts at app + storeHashes("app", specs, considerRelocation=False) + self.assertEqual(specs["app"]["remote_hashes"], planned["app"]["remote_hashes"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_prepub.py b/tests/test_prepub.py index 016d228a..2a1a6c24 100644 --- a/tests/test_prepub.py +++ b/tests/test_prepub.py @@ -5,6 +5,7 @@ import hashlib import json +import shutil import os import tarfile import tempfile @@ -169,6 +170,7 @@ def setUp(self): def tearDown(self): self.server.shutdown() + self.server.server_close() def _make_tar(self, content=b"fake payload"): fd, path = tempfile.mkstemp(suffix=".tar.gz") @@ -277,6 +279,7 @@ def do_POST(self): finally: os.unlink(tar) reject_server.shutdown() + reject_server.server_close() # --------------------------------------------------------------------------- @@ -293,6 +296,7 @@ def test_returns_published_immediately(self): self.assertEqual(result, "published") finally: server.shutdown() + server.server_close() def test_raises_on_failed(self): server, url = _start_fake_prepub(state="failed") @@ -302,6 +306,7 @@ def test_raises_on_failed(self): poll_job(url, "", "test-job-id-123", poll_interval=1, timeout=10) finally: server.shutdown() + server.server_close() def test_raises_on_aborted(self): server, url = _start_fake_prepub(state="aborted") @@ -311,6 +316,7 @@ def test_raises_on_aborted(self): poll_job(url, "", "test-job-id-123", poll_interval=1, timeout=10) finally: server.shutdown() + server.server_close() def test_raises_on_timeout(self): """When the job never reaches a terminal state, poll_job must time out.""" @@ -322,6 +328,7 @@ def test_raises_on_timeout(self): poll_job(url, "", "test-job-id-123", poll_interval=1, timeout=1) finally: server.shutdown() + server.server_close() def test_transitions_from_intermediate_to_published(self): """Simulate uploading → published across two polls.""" @@ -339,6 +346,7 @@ def _flip(): result = poll_job(url, "", "test-job-id-123", poll_interval=0, timeout=5) self.assertEqual(result, "published") server.shutdown() + server.server_close() if __name__ == "__main__": @@ -410,6 +418,7 @@ def test_submission_signs_the_prefixed_uri(self): os.unlink(tar) finally: server.shutdown() + server.server_close() class TestSignedRequestsAreNotReplayed(unittest.TestCase): @@ -474,3 +483,32 @@ def do_POST(self): finally: os.unlink(tar) server.shutdown() + server.server_close() + + +class MultipartFileTest(unittest.TestCase): + """The streamed upload body: what requests' files= sent, read in pieces.""" + + def test_parts_length_and_rewind(self): + import email.parser + from bits_helpers.prepub import MultipartFile + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + tar = os.path.join(d, "p.tar") + payload = os.urandom(300000) + with open(tar, "wb") as fh: + fh.write(payload) + fields = {"repo": (None, "r"), "tar": ("p.tar", tar, "application/x-tar"), + "tar_sha256": (None, "abc")} + with MultipartFile(fields) as body: + first = b"".join(iter(lambda: body.read(4096), b"")) + self.assertEqual(len(first), len(body)) + body.seek(0) # urllib3's retry rewind + self.assertEqual(body.read(), first) + msg = email.parser.BytesParser().parsebytes( + b"Content-Type: " + body.content_type.encode() + b"\r\n\r\n" + first) + parts = {p.get_param("name", header="content-disposition"): p for p in msg.get_payload()} + self.assertEqual(list(parts), ["repo", "tar_sha256", "tar"]) # the file last + self.assertEqual(parts["repo"].get_payload(decode=True), b"r") + self.assertEqual(parts["tar"].get_payload(decode=True), payload) + self.assertEqual(parts["tar"].get_content_type(), "application/x-tar") diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 63c0f811..2856cc0e 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -122,6 +122,16 @@ def test_cvmfs_layout_recorded_when_present(self): self.assertEqual(rec["cvmfs_layout"]["views_dir"], "Views") self.assertEqual(rec["cvmfs_layout"]["views_path"], "/cvmfs/x/Views") + def test_relaxed_symlinks_record_loose_provenance_independently_of_reuse(self): + for policy, expected in (("strict", "pure"), ("relaxed", "loose")): + rec = self._record(SimpleNamespace( + annotate={}, architecture="arch", defaults=["release"], + reusePolicy="strict", symlinkPackagesFrom="/deployment", + symlinkPolicy=policy)) + self.assertEqual(rec["symlink_policy"], policy) + self.assertEqual(rec["reuse_policy"], "strict") + self.assertEqual(rec["provenance"], expected) + def test_reuse_policy_defaults_to_strict_when_arg_absent(self): # args without a reuse_policy attribute (the aliBuild simple case) rec = self._record(SimpleNamespace(annotate={}, architecture="arch", diff --git a/tests/test_publish_manifest.py b/tests/test_publish_manifest.py index 9e2c8e65..79cfeb1f 100644 --- a/tests/test_publish_manifest.py +++ b/tests/test_publish_manifest.py @@ -73,6 +73,9 @@ def _write_store_tarball(work_dir, arch, h, name): class TestPublishFromManifest(unittest.TestCase): def setUp(self): + marker = patch.object(publish, "_orchestrator_marker", return_value=None) + marker.start() # the stores here have no orchestrator + self.addCleanup(marker.stop) self.tmp = tempfile.mkdtemp() self.work = os.path.join(self.tmp, "sw") man_dir = os.path.join(self.work, "MANIFESTS") @@ -155,6 +158,14 @@ def test_bom_has_provenance_and_excludes_config(self): self.assertTrue(top["tarball"]) self.assertTrue(top["tarball_sha256"].startswith("sha256:")) + def test_bom_records_the_builds_architecture_not_the_hosts(self): + self.writer = _FakeWriter() + with patch.object(sync, "remote_from_url", return_value=self.writer): + publish._publish_from_manifest( + "x86_64-el9", self.work, "https://s3.example/mybucket", _Parser(), + manifest=self.man_path) + self.assertEqual(self.writer.s3.uploads[-1][2]["architecture"], ARCH) + def test_bom_carries_what_a_release_needs(self): """A release is made later from the BOM alone: the build's templates, what it was asked to build, per package its view rules and requires; @@ -412,6 +423,7 @@ def test_certify_passkey_needs_console_and_target(self): patch.object(publish, "_submit_certification_mr") as sub, \ patch("bits_helpers.forge.resolve_gitlab_token", return_value="tok"), \ patch("bits_helpers.sign_console.preapprove_via_console") as pre, \ + patch("bits_helpers.site.get", return_value=None), \ patch.dict("os.environ", {"BITS_CONSOLE_URL": ""}): with self.assertRaises(_Parser._Err): # no group / manifests repo publish.doCertify(self._certify_args(console="https://c"), _Parser()) @@ -421,6 +433,15 @@ def test_certify_passkey_needs_console_and_target(self): pre.assert_not_called() sub.assert_not_called() + def test_the_console_url_defaults_to_the_site_s(self): + from bits_helpers import orchestrated + with patch.dict("os.environ", {"BITS_CONSOLE_URL": ""}): + self.assertEqual(orchestrated.console_url(), "https://bits.cern.ch") # 10-CERN.yaml + self.assertEqual(orchestrated.console_url(system={"console_url": "https://c"}), + "https://c") + with patch("bits_helpers.site.get", return_value=None): + self.assertIsNone(orchestrated.console_url()) + def test_certify_dry_run_touches_nothing(self): with patch.object(publish, "_publish_from_manifest", return_value=None), \ patch.object(publish, "_submit_certification_mr") as sub, \ @@ -435,6 +456,165 @@ def test_run_leaf_is_unique_per_call(self): self.assertRegex(leaf, r"^[A-Za-z0-9._-]+-\d{8}T\d{6}Z-[0-9a-f]+\.json$") +class _StoreS3: + """An S3 stand-in that keeps what is written and serves it back.""" + + def __init__(self): + self.objects = {} + + def upload_file(self, local, bucket, key): + with open(local, "rb") as fh: + self.objects[key] = fh.read() + + def put_object(self, Bucket, Key, Body): + self.objects[Key] = Body if isinstance(Body, bytes) else Body.encode() + + def get_object(self, Bucket, Key): + import io + if Key not in self.objects: + raise KeyError(Key) + return {"Body": io.BytesIO(self.objects[Key])} + + +class _StoreWriter: + def __init__(self, s3, delay=None): + import threading + self.s3, self.writeStore, self.tarballs = s3, "mybucket", [] + self.delay, self.lock = delay or {}, threading.Lock() + self.active = self.peak = 0 + + def upload_symlinks_and_tarball(self, spec): + import time + with self.lock: + self.active += 1 + self.peak = max(self.peak, self.active) + time.sleep(self.delay.get(spec["hash"], 0.01)) + with self.lock: + self.active -= 1 + self.tarballs.append(spec["hash"]) + spec["store_tarball_sha256"] = "sha256:" + spec["hash"] + + +class TestPublishOnceAndConcurrently(unittest.TestCase): + """The per-package store checks run a few at a time, and `bits certify` in + the publish job uses the BOM the job's BOM step already published.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.work = os.path.join(self.tmp, "sw") + os.makedirs(os.path.join(self.work, "MANIFESTS")) + self.hashes = ["%02x%s" % (i, "ab" * 3) for i in range(12)] + self.manifest = {"defaults": ["release"], "architecture": ARCH, "packages": [ + {"package": "P%d" % i, "version": "1", "revision": "1", "hash": h, + "effective_architecture": ARCH, "view": {"include": ["bin"]}} + for i, h in enumerate(self.hashes)]} + self.man_path = os.path.join(self.work, "MANIFESTS", "bits-manifest-P-20260101T000000Z.json") + with open(self.man_path, "w") as fh: + json.dump(self.manifest, fh) + for i, h in enumerate(self.hashes): + _write_store_tarball(self.work, ARCH, h, "P%d-1-1.%s.tar.gz" % (i, ARCH)) + self.s3 = _StoreS3() + env = patch.dict(os.environ, {"CI_PIPELINE_ID": "4242"}) + env.start() + self.addCleanup(env.stop) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _publish(self, writer, **kw): + with patch.object(sync, "remote_from_url", return_value=writer), \ + patch("bits_helpers.notice.upload_release_compliance"): + return publish._publish_from_manifest(ARCH, self.work, "https://s3.example/mybucket", + _Parser(), manifest=self.man_path, **kw) + + def _boms(self): + return [json.loads(v) for k, v in sorted(self.s3.objects.items()) + if k.startswith("MANIFESTS/") and k.endswith(".json")] + + def test_concurrent_checks_keep_the_bom_in_manifest_order(self): + # The first packages finish last: the BOM still follows the manifest. + w = _StoreWriter(self.s3, {h: 0.05 * (12 - i) for i, h in enumerate(self.hashes)}) + with patch.dict(os.environ, {"BITS_PUBLISH_JOBS": "4"}): + build_id, bom, _system = self._publish(w) + self.assertEqual(w.peak, 4) + self.assertEqual(sorted(w.tarballs), sorted(self.hashes)) + self.assertNotEqual(w.tarballs, self.hashes) # finished out of order + self.assertEqual([p["hash"] for p in bom[0][1]["packages"]], self.hashes) + self.assertEqual(bom[0][1]["packages"][0]["tarball_sha256"], "sha256:" + self.hashes[0]) + + def test_certify_uses_the_bom_the_publish_step_wrote(self): + first = self._publish(_StoreWriter(self.s3)) + boms = len(self._boms()) + again = _StoreWriter(self.s3) + build_id, bom, _system = self._publish(again, reuse_pipeline_bom=True) + self.assertEqual(again.tarballs, []) # nothing checked again + self.assertEqual(len(self._boms()), boms) # no second BOM + self.assertEqual((build_id, bom), (first[0], first[1])) + + def test_a_bom_that_is_not_this_publish_is_not_reused(self): + def tamper(doc): + doc["packages"][0]["view"] = {"include": ["everything"]} + def drop(doc): + doc["packages"].pop() + def twice(doc): + doc["packages"][-1] = dict(doc["packages"][0]) + def other_pipeline(doc): + doc["source_pipeline_id"] = "1" + def extra_group(doc): + doc["packages"][0]["group"] = "common" + def other_layout(doc): + doc["cvmfs_templates"] = {"package": "/elsewhere"} + def other_tarball(doc): + doc["packages"][0]["tarball"] = "P0-1-2.%s.tar.gz" % ARCH + def other_label(doc): + doc["effective_architecture"] = "x86_64-el9" + for change in (tamper, drop, twice, other_pipeline, extra_group, other_layout, + other_tarball, other_label): + with self.subTest(change=change.__name__): + self.s3.objects.clear() + self._publish(_StoreWriter(self.s3)) + key = [k for k in self.s3.objects if k.startswith("MANIFESTS/") + and k.endswith(ARCH + ".json")][0] + doc = json.loads(self.s3.objects[key]) + change(doc) + self.s3.objects[key] = json.dumps(doc).encode() + again = _StoreWriter(self.s3) + self._publish(again, reuse_pipeline_bom=True) + self.assertEqual(sorted(again.tarballs), sorted(self.hashes)) # published afresh + + def test_a_stored_sha256_is_what_the_reused_bom_carries(self): + # The one thing taken from the store: its sha256 of each tarball. + self._publish(_StoreWriter(self.s3)) + build_id, bom, _system = self._publish(_StoreWriter(self.s3), reuse_pipeline_bom=True) + self.assertEqual([p["tarball_sha256"] for p in bom[0][1]["packages"]], + ["sha256:" + h for h in self.hashes]) + + def test_an_interrupt_stops_the_queue(self): + class Interrupting(_StoreWriter): + def upload_symlinks_and_tarball(self, spec): + if spec["hash"] == self_hashes[0]: + raise KeyboardInterrupt + super().upload_symlinks_and_tarball(spec) + self_hashes = self.hashes + w = Interrupting(self.s3, {h: 0.2 for h in self.hashes}) + with patch.dict(os.environ, {"BITS_PUBLISH_JOBS": "2"}): + with self.assertRaises(KeyboardInterrupt): + self._publish(w) + self.assertLess(len(w.tarballs), 4) # not all twelve + self.assertEqual([k for k in self.s3.objects if k.startswith("MANIFESTS/")], []) + + def test_no_reuse_outside_a_pipeline_or_for_bits_publish(self): + self._publish(_StoreWriter(self.s3)) + plain = _StoreWriter(self.s3) + self._publish(plain) # bits publish: always + self.assertEqual(len(plain.tarballs), 12) + with patch.dict(os.environ, {"CI_PIPELINE_ID": ""}): + cli = _StoreWriter(self.s3) + self._publish(cli, reuse_pipeline_bom=True) + self.assertEqual(len(cli.tarballs), 12) + + class TestStoreUploadRedistributable(unittest.TestCase): """H1 mitigation: `bits store upload` (_publish_s3) must refuse a package whose recipe forbids binary redistribution — mirroring the build-time and bulk gates @@ -447,7 +627,8 @@ def _run(self, entry, store="b3://bucket"): patch.dict(os.environ, {}, clear=False): publish._publish_s3(entry["package"], entry.get("version"), ARCH, "/wd", store, _Parser()) - self.endpoint = os.environ.get("BITS_S3_ENDPOINT_URL") + from bits_helpers.sync import bucket_endpoint # the store's own + self.endpoint = bucket_endpoint(rfu.call_args[0][0][5:]) if rfu.call_args else None self.store_used = rfu.call_args[0][0] if rfu.call_args else None return writer @@ -455,7 +636,7 @@ def test_https_store_is_normalised_like_bits_publish(self): # The default store is an https:// URL; without normalisation the upload # fell through to the rsync backend. with patch.dict(os.environ, {}, clear=False): - os.environ.pop("BITS_S3_ENDPOINT_URL", None) + os.environ.pop("BITS_S3_ALIBUILD_REPO_ENDPOINT_URL", None) w = self._run({"package": "ROOT", "version": "6.30", "revision": "1", "hash": "h3"}, store="https://s3.cern.ch/swift/v1/alibuild-repo") self.assertEqual(self.store_used, "b3://alibuild-repo") diff --git a/tests/test_release_view.py b/tests/test_release_view.py new file mode 100644 index 00000000..5e651981 --- /dev/null +++ b/tests/test_release_view.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""bits setenv / enter / printenv of a release with a complete view on CVMFS: +its setup.sh stands in for loading the modules, with no modulecmd and no Python. + +python3 and modulecmd are stubs that record being run, so the tests also show +when the modules path is taken. +""" + +import os +import shutil +import subprocess +import tempfile +import unittest + +from bits_helpers import cvmfs_publish as cp + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ARCH = "x86_64-el9-gcc14-opt" + +TOP_MF = """#%Module1.0 +if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 } +if ![ is-loaded "dep/2-1" ] { module load dep/2-1 } +setenv TOP_ROOT $::env(BASEDIR)/top/1-1 +setenv BITS_VIEW 1 +""" + + +def _write(path, text, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + os.chmod(path, mode) + + +class ReleaseViewTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.inst = os.path.join(self.tmp, "inst") + os.makedirs(self.inst) + for f in ("bits", "bitsBuild"): + shutil.copy2(os.path.join(ROOT, f), self.inst) + self.calls = os.path.join(self.tmp, "calls") + stubs = os.path.join(self.tmp, "stubs") + for name in ("python3", "modulecmd"): + _write(os.path.join(stubs, name), + '#!/bin/sh\necho "%s $*" >> "%s"\nexit 1\n' % (name, self.calls), 0o755) + self.work = os.path.join(self.tmp, "work") + os.makedirs(os.path.join(self.work, "sw", ARCH)) + # The community's tree, as bits cvmfs publish leaves it. + self.cvmfs = os.path.join(self.tmp, "cvmfs", "community") + self.mods = os.path.join(self.cvmfs, ARCH, "Modules", "modulefiles") + _write(os.path.join(self.mods, "BASE", "1.0"), "#%Module1.0\n") + _write(os.path.join(self.mods, "top", "1-1"), TOP_MF) + _write(os.path.join(self.mods, "dep", "2-1"), "#%Module1.0\n") + for pkg in ("top/1-1", "dep/2-1"): + os.makedirs(os.path.join(self.cvmfs, ARCH, "Packages", pkg, "bin")) + self.view = os.path.join(self.cvmfs, "views", "R", ARCH) + os.makedirs(os.path.join(self.view, "bin")) + env = {"paths": {}, "sets": {"TOP_ROOT": "@B@/top/1-1"}, + "base": (os.path.relpath(os.path.join(self.cvmfs, ARCH, "Packages"), self.view), + "/cvmfs/community/%s/Packages" % ARCH)} + cp.write_view_setup(self.view, "/cvmfs/community/views/R/" + ARCH, env) + self.marker = os.path.join(self.view, cp.setup_marker( + {"package": "top", "version": "1", "revision": "1"})) + _write(self.marker, "#\n") + self.pointer = os.path.join(self.mods, "top", ".1-1.view") + os.symlink(os.path.relpath(self.marker, os.path.dirname(self.pointer)), self.pointer) + self.env = {"HOME": self.tmp, "PATH": stubs + ":/usr/bin:/bin", + "BITS_CVMFS_PREFIX": self.cvmfs, "MODULES_SHELL": "bash"} + + def _bits(self, *args, **kw): + return subprocess.run([os.path.join(self.inst, "bits"), "-a", ARCH] + list(args), + cwd=self.work, env=self.env, capture_output=True, text=True, **kw) + + def _ran(self): + try: + with open(self.calls) as f: + return f.read().split("\n")[:-1] + except OSError: + return [] + + def _setenv(self, *mods): + return self._bits("setenv", *mods, "-c", "sh", "-c", + 'echo "$TOP_ROOT|${PATH%%:*}|${BITS_VIEW-unset}"') + + def test_setenv_sources_the_view(self): + r = self._setenv("top/1-1") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.strip().split("|"), [ + os.path.join(self.cvmfs, ARCH, "Packages", "top", "1-1"), + os.path.join(self.view, "bin"), "unset"]) + self.assertEqual(self._ran(), []) + + def test_printenv_prints_sourcing_the_view(self): + # load / printenv collapse onto a view only with --view, as the modules path. + r = self._bits("--view", "printenv", "top/1-1") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout, '. "%s/setup.sh";\nunset BITS_VIEW;\n' % self.view) + self.env["MODULES_SHELL"] = "tcsh" + r = self._bits("--view", "printenv", "top/1-1") + self.assertEqual(r.stdout, 'source "%s/setup.csh";\nunsetenv BITS_VIEW;\n' % self.view) + self.assertEqual(self._ran(), []) + self._bits("printenv", "top/1-1") + self.assertTrue(self._ran()) + + def test_enter_sources_the_view(self): + r = self._bits("enter", "top/1-1", input='echo "ROOT=$TOP_ROOT"\nexit\n') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("ROOT=" + os.path.join(self.cvmfs, ARCH, "Packages", "top", "1-1"), + r.stdout) + self.assertIn("Set up from the release view %s" % self.view, r.stderr) + self.assertEqual(self._ran(), []) + + def test_the_modules_otherwise(self): + cases = [] + # Two modules, a bare name: modulecmd resolves them. + cases.append(lambda: self._setenv("top/1-1,dep/2-1")) + cases.append(lambda: self._setenv("top")) + # A package without view: true, unless --view. + def plain(): + _write(os.path.join(self.mods, "top", "1-1"), TOP_MF.replace("setenv BITS_VIEW 1\n", "")) + return self._setenv("top/1-1") + cases.append(plain) + # The view is not complete (no .setup-complete), or not pointed to. + def incomplete(): + os.unlink(self.marker) + return self._setenv("top/1-1") + cases.append(incomplete) + # The view was made again for another revision: this one's link leads nowhere. + def remade(): + os.rename(self.marker, self.marker.replace(".1-1", ".1-2")) + return self._setenv("top/1-1") + cases.append(remade) + def no_csh(): + os.unlink(os.path.join(self.view, "setup.csh")) + return self._setenv("top/1-1") + cases.append(no_csh) + def no_pointer(): + os.unlink(self.pointer) + return self._setenv("top/1-1") + cases.append(no_pointer) + # A local build of it wins over CVMFS. + def local(): + _write(os.path.join(self.work, "sw", ARCH, "top", "1-1", "etc", "modulefiles", "top"), + TOP_MF) + return self._setenv("top/1-1") + cases.append(local) + for i, case in enumerate(cases): + self.setUp() + case() + self.assertTrue(any(c.startswith("modulecmd ") for c in self._ran()), i) + + def test_view_option_uses_it_for_a_plain_package(self): + _write(os.path.join(self.mods, "top", "1-1"), TOP_MF.replace("setenv BITS_VIEW 1\n", "")) + r = self._bits("--view", "setenv", "top/1-1", "-c", "sh", "-c", 'echo "$TOP_ROOT"') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.strip(), os.path.join(self.cvmfs, ARCH, "Packages", "top", "1-1")) + self.assertEqual(self._ran(), []) + + + +def _modulecmd(): + m = os.environ.get("BITS_TEST_MODULECMD") or shutil.which("modulecmd") + return m if m and os.access(m, os.X_OK) else None + + +@unittest.skipUnless(_modulecmd(), "needs modulecmd (Environment Modules)") +class SameAsTheModulesTest(unittest.TestCase): + """What setup.sh sets is what loading the modules sets, but for the entries + the view collapses (those are the view's).""" + + def test_setup_sets_what_modulecmd_sets(self): + import sys + sys.path.insert(0, os.path.join(ROOT, "tests")) + from test_modulefile_env import ROOT as ROOT_MF, ZLIB as ZLIB_MF, MEMBERS + from bits_helpers import modulefile_env as mfe + tmp = os.path.realpath(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, tmp, True) + pkgs = os.path.join(tmp, ARCH, "Packages") + mods = os.path.join(tmp, ARCH, "Modules", "modulefiles") + _write(os.path.join(mods, "BASE", "1.0"), "#%%Module1.0\nsetenv BASEDIR %s\n" % pkgs) + _write(os.path.join(mods, "ROOT", "6.36-2"), ROOT_MF) + _write(os.path.join(mods, "zlib", "1.3-1"), ZLIB_MF) + dirs = {("ROOT", "bin"), ("ROOT", "lib"), ("ROOT", "include"), ("ROOT", "etc/plugins"), + ("zlib", "lib")} + for pkg, rel in dirs: + os.makedirs(os.path.join(pkgs, pkg, MEMBERS[pkg], rel)) + texts = {"ROOT": ROOT_MF, "zlib": ZLIB_MF} + ops, problems = mfe.release_ops(["ROOT"], MEMBERS, texts.get, lambda p, r: (p, r) in dirs) + linked = ("bin", "lib", "include") + paths, sets, more = mfe.setup_env(ops, MEMBERS, lambda p, r: r.split("/")[0] in linked) + self.assertEqual(problems + more, []) + view = os.path.join(tmp, "views", "R", ARCH) + for d in linked: + os.makedirs(os.path.join(view, d)) + cp.write_view_setup(view, "/cvmfs/x", {"paths": paths, "sets": sets, + "base": (os.path.relpath(pkgs, view), pkgs)}) + show = 'env | LC_ALL=C sort | grep -vE "^(_|SHLVL|PWD|OLDPWD)="' + clean = {"PATH": "/usr/bin:/bin", "HOME": tmp} + by_modules = subprocess.run( + ["bash", "-c", 'eval "$(%s bash load ROOT/6.36-2)"; %s' % (_modulecmd(), show)], + env=dict(clean, MODULEPATH=mods), capture_output=True, text=True).stdout + by_view = subprocess.run(["bash", "-c", '. "%s/setup.sh"; %s' % (view, show)], + env=dict(clean, MODULEPATH=mods), capture_output=True, + text=True).stdout + + def parse(out): + return dict(l.split("=", 1) for l in out.splitlines() if "=" in l) + mod, vw = parse(by_modules), parse(by_view) + collapse = set(mfe.COLLAPSE_VARS) + ignore = {"BITS_VIEW", "BASEDIR", "LOADEDMODULES", "_LMFILES_", "MODULEPATH", + "MODULES_LMALTNAME", "MODULES_LMCONFLICT", "MODULES_LMPREREQ"} + for var in set(mod) | set(vw): + if var in ignore or var.startswith("__MODULES") or var.endswith("_modshare"): + continue + if var in collapse: + # Each package entry the modules add is the view's entry for it. + for entry in mod.get(var, "").split(":"): + if entry.startswith(pkgs + "/"): + pkg, ver, *rest = os.path.relpath(entry, pkgs).split("/") + want = os.path.join(view, *rest) if not rest or rest[0] in linked else entry + self.assertIn(want.rstrip("/"), vw[var].split(":"), (var, entry)) + else: + self.assertEqual(vw.get(var), mod.get(var), var) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_relocate_deps.py b/tests/test_relocate_deps.py new file mode 100644 index 00000000..d543dc8e --- /dev/null +++ b/tests/test_relocate_deps.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""bits cvmfs publish: paths to dependencies become their own CVMFS paths.""" +from pathlib import Path +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from unittest import mock + +import bits_helpers +import bits_helpers.cvmfs_publish as cp + +W, A, PH = "/w/sw", "el9-gcc14-opt", "abc" +PP = "%s/Athena/25.0.1-1" % A +TM = {"prefix": "/cvmfs/r/atlas", + "packages": "{prefix}/{arch}/Packages/{family}{pkg}/{tag}", + "shared": "{prefix}/noarch/{pkg}/{tag}"} +DEPS = [ # (name, family, effective_architecture, version, revision) + ("ROOT", "", A, "6.36", "1"), + ("Python", "lcg", A, "3.13", "1"), # a family + ("GCC", "", "el9", "14.3", "1"), # own_hash toolchain arch + ("cmake-modules", "", "share", "2.0", "1"), # noarch + ("zlib", "", A, "1.0", "1"), + ("onlybuilt", "", A, "1", ""), # no revision, never referenced +] +FILES = { + "env_setup.sh": + "export ROOTSYS=%(W)s/%(A)s/ROOT/6.36-1\n" + "export PATH=%(W)s/%(PP)s/bin:%(W)s/%(A)s/lcg/Python/3.13-1/bin:$PATH\n" + "export OLD=/cvmfs/r/atlas/%(A)s/Packages/ROOT/6.36-1/lib\n", + "bin/tool": "#!%(W)s/%(A)s/lcg/Python/3.13-1/bin/python3\nprint(1)\n", + "lib/cmake/AthenaConfig.cmake": + 'set(GCC_ROOT "%(W)s/el9/GCC/14.3-1")\n' + "include(%(W)s/share/cmake-modules/2.0-1/x.cmake)\n", + "lib/libAthena.la": + "dependency_libs=' -L%(W)s/%(A)s/ROOT/6.36-1/lib " + "%(W)s/INSTALLROOT/%(PH)s/%(PP)s/lib/libz.la'\n", + "zlib.txt": "%(W)s/%(A)s/zlib/1.0-1/lib;%(W)s/%(A)s/zlib/1.0-10/lib\n", +} +OWN = "/cvmfs/r/atlas/%s/Packages/Athena/25.0.1-1" % A +CTX = {"repo": "r", "arch": A, "tmpl_prefix": "", "templates": TM, + "prefix_fallback": "/cvmfs/r", "submit": False} + + +def _pkgroot(base): + root = os.path.join(base, PP) + os.makedirs(os.path.join(root, "etc", "profile.d")) + for rel, text in FILES.items(): + os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True) + with open(os.path.join(root, rel), "w") as fh: + fh.write(text % {"W": W, "A": A, "PP": PP, "PH": PH}) + os.chmod(os.path.join(root, "bin", "tool"), 0o755) + os.symlink("tool", os.path.join(root, "bin", "link")) + with open(os.path.join(root, "etc", "profile.d", ".bits-pkginfo"), "w") as fh: + fh.write('OP=%s\nPP=${PKGPATH:-%s}\nPH=%s\nPKG_DIR="%s"\n' % (PP, PP, PH, W)) + with open(os.path.join(root, "etc", "profile.d", ".bits-relocate"), "w") as fh: + fh.write("\n".join(sorted(FILES)) + "\nbin/link\n") + deps = [{"name": n, "pkg_family": f, "effective_architecture": a, + "version": v, "revision": r} for n, f, a, v, r in DEPS] + with open(os.path.join(root, ".meta.json"), "w") as fh: + json.dump({"dependencies": {"recursive": {"build": deps, "runtime": deps[:2]}}}, fh) + shutil.copy(os.path.join(os.path.dirname(bits_helpers.__file__), "relocate-me.sh"), root) + return root + + +def _read(root, rel): + with open(os.path.join(root, rel)) as fh: + return fh.read() + + +class TestRelocateDependencies(unittest.TestCase): + def setUp(self): + self.t = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.t, True) + self.root = _pkgroot(os.path.join(self.t, "w")) + + def _relocate(self, ctx=CTX): + cp.relocate_dependencies(self.root, cp.dependency_targets(self.root, ctx)) + + def test_each_dependency_gets_its_own_path(self): + self._relocate() + P = "/cvmfs/r/atlas/%s/Packages" % A + self.assertEqual(_read(self.root, "env_setup.sh"), + "export ROOTSYS=%s/ROOT/6.36-1\n" + "export PATH=%s/bin:%s/lcg/Python/3.13-1/bin:$PATH\n" + "export OLD=%s/ROOT/6.36-1/lib\n" % (P, W, P, P)) + self.assertEqual(_read(self.root, "bin/tool").splitlines()[0], + "#!%s/lcg/Python/3.13-1/bin/python3" % P) + self.assertEqual(_read(self.root, "lib/cmake/AthenaConfig.cmake"), + 'set(GCC_ROOT "/cvmfs/r/atlas/el9/Packages/GCC/14.3-1")\n' + "include(/cvmfs/r/atlas/noarch/cmake-modules/2.0-1/x.cmake)\n") + # The package's own paths are left to relocate-me.sh. + self.assertIn("%s/INSTALLROOT/%s/%s/lib" % (W, PH, PP), + _read(self.root, "lib/libAthena.la")) + self.assertIn("PATH=%s/bin:" % W, _read(self.root, "env_setup.sh")) + # zlib/1.0-1 is the dependency, zlib/1.0-10 is not. + self.assertEqual(_read(self.root, "zlib.txt"), + "%s/zlib/1.0-1/lib;%s/%s/zlib/1.0-10/lib\n" % (P, W, A)) + self.assertEqual(os.stat(os.path.join(self.root, "bin", "tool")).st_mode & 0o777, 0o755) + self.assertTrue(os.path.islink(os.path.join(self.root, "bin", "link"))) + + def test_a_referenced_dependency_without_a_path_stops_the_publish(self): + ctx = dict(CTX, templates=dict(TM, packages="{prefix}/{release}/{pkg}/{tag}")) + with self.assertRaisesRegex(SystemExit, r"refers to .* no CVMFS path: unresolved \{release\}"): + self._relocate(ctx) + + def test_an_unreferenced_dependency_without_a_path_is_fine(self): + targets = cp.dependency_targets(self.root, CTX) + targets["%s/%s/onlybuilt/1" % (W, A)] = ValueError("unresolved {release}") + cp.relocate_dependencies(self.root, targets) + self.assertNotIn(W + "/" + A + "/ROOT", _read(self.root, "env_setup.sh")) + + def test_what_else_names_the_build_area_is_reported(self): + with open(os.path.join(self.root, "lib", "cmake", "AthenaConfig.cmake"), "a") as fh: + fh.write("# %s/BUILD/abc/Athena/x.cc\n" % W) + self.assertEqual(cp.relocate_dependencies(self.root, cp.dependency_targets(self.root, CTX)), + ["lib/cmake/AthenaConfig.cmake", "zlib.txt"]) + + def test_old_meta_and_odd_entries(self): + with open(os.path.join(self.root, ".meta.json"), "w") as fh: + json.dump({"dependencies": {"recursive": {"build": [ + {"name": "ROOT", "version": "6.36", "revision": 1}, {"version": "x"}]}}}, fh) + self.assertEqual(cp.dependency_targets(self.root, CTX)["%s/%s/ROOT/6.36-1" % (W, A)], + "/cvmfs/r/atlas/%s/Packages/ROOT/6.36-1" % A) + os.remove(os.path.join(self.root, ".meta.json")) + self.assertEqual(cp.dependency_targets(self.root, CTX), {"%s/%s" % (W, PP): W}) + + def test_files_outside_the_package_and_read_only_files(self): + outside = os.path.join(self.t, "outside.txt") + with open(outside, "w") as fh: + fh.write("%s/%s/ROOT/6.36-1\n" % (W, A)) + with open(os.path.join(self.root, "etc", "profile.d", ".bits-relocate"), "a") as fh: + fh.write("%s\n../../../../outside.txt\n" % outside) + os.chmod(os.path.join(self.root, "env_setup.sh"), 0o444) + self._relocate() + self.assertEqual(_read(self.t, "outside.txt"), "%s/%s/ROOT/6.36-1\n" % (W, A)) + self.assertNotIn(W + "/" + A, _read(self.root, "env_setup.sh")) + self.assertEqual(os.stat(os.path.join(self.root, "env_setup.sh")).st_mode & 0o777, 0o444) + + def test_a_cvmfs_path_containing_the_build_dir(self): + # relocate-me.sh maps the package's own path in one pass; a dependency's + # would be rewritten again by it, so that stops the publish. + ctx = dict(CTX, templates=dict(TM, packages="{prefix}%s/{pkg}/{tag}" % W)) + with self.assertRaisesRegex(SystemExit, "contains the build dir"): + self._relocate(ctx) + + def test_without_pkginfo_nothing_changes(self): + os.remove(os.path.join(self.root, "etc", "profile.d", ".bits-pkginfo")) + self.assertEqual(cp.dependency_targets(self.root, CTX), {}) + + def test_publish_one_leaves_no_build_path(self): + if "GNU tar" not in subprocess.run(["tar", "--version"], capture_output=True, + text=True).stdout: + self.skipTest("publish packs with GNU tar --hard-dereference (not bsdtar)") + tar = os.path.join(self.t, "Athena.tar.gz") + subprocess.run(["tar", "-czf", tar, "-C", os.path.join(self.t, "w"), A], check=True) + out = os.path.join(self.t, "out") + + def publish(c, p, tarfn, label, **k): + with tarfile.open(tarfn) as tf: + tf.extractall(out) + return "J" + spec = {"package": "Athena", "version": "25.0.1", "revision": "1"} + with mock.patch.object(cp, "tar_path", lambda s, r, a: tar), \ + mock.patch.object(cp, "_publish_tar", publish): + cp.publish_one(spec, dict(CTX, tars_root=self.t, tmp_dir=self.t)) + for dp, _dn, fns in os.walk(out): + for f in fns: + p = os.path.join(dp, f) + if not os.path.islink(p) and f != "zlib.txt": # 1.0-10: not a dependency + text = Path(p).read_text(errors="replace") + self.assertNotIn(W + "/", text, p) + self.assertNotIn(OWN + "/" + A, text, p) + self.assertIn("-L/cvmfs/r/atlas/%s/Packages/ROOT/6.36-1/lib %s/lib/libz.la" % (A, OWN), + _read(out, "lib/libAthena.la")) + self.assertIn('PKG_DIR="%s"' % OWN, _read(out, "etc/profile.d/.bits-pkginfo")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_relocate_me.py b/tests/test_relocate_me.py index 0d119ec3..854dd3fd 100644 --- a/tests/test_relocate_me.py +++ b/tests/test_relocate_me.py @@ -50,6 +50,22 @@ def test_install_base_containing_pkg_dir_is_not_rewritten_again(self): ["/w/sw/stage/pkg/1.0/lib", "/w/sw/stage/pkg/1.0/other/lib", "/w/sw/stage/pkg/1.0/%s/bin" % PP]) + def test_a_staging_dir_under_the_work_dir_is_not_doubled(self): + # A cached tarball is relocated first into INSTALLROOT/, under PKG_DIR. + stage = "/w/sw/INSTALLROOT/%s" % PH + self.assertEqual(self._relocate("/w/sw", stage, False), + ["%s/%s/lib" % (stage, PP), "%s/other/lib" % stage, + "%s/%s/bin" % (stage, PP)]) + + def test_a_cached_tarball_is_relocated_with_this_bits_script(self): + # The tarball's relocate-me.sh is from the bits that packed it (older ones + # double INSTALLROOT/ for a staging dir under the work dir). + with open(os.path.join(os.path.dirname(SCRIPT), "build_template.sh")) as f: + t = f.read() + cached = t[t.index('tar -xzf "$CACHED_TARBALL"'):] + install = cached.index('install "${BITS_SCRIPT_DIR}/bits_helpers/relocate-me.sh"') + self.assertLess(install, cached.index('/bin/bash -ex "$INSTALLROOT/relocate-me.sh"')) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_relocate_references.py b/tests/test_relocate_references.py new file mode 100644 index 00000000..3a9b36a3 --- /dev/null +++ b/tests/test_relocate_references.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +from bits_helpers.reuse_symlinks import serialize_relocation_table + + +RELOCATE = Path(__file__).parents[1] / "bits_helpers/relocate-references.sh" + + +def relocate_references(root, replacements): + subprocess.run(["bash", str(RELOCATE), str(root), + serialize_relocation_table(replacements)], check=True) + + +class ReferenceRelocationTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() # macOS: /var is /private/var + self.stage = self.root / "stage" + self.stage.mkdir() + + def test_all_dependency_paths_in_consumer_configs(self): + text = ('-I/ref/sw/arch/external/hdf5/1/include ' + '-L/ref/sw/arch/external/zlib/1/lib ' + '-L/ref/sw/arch/external/openmpi/1/lib\n') + for name in ("yoda.pc", "libYODA.la", "yoda-config", "YODAConfig.cmake"): + (self.stage / name).write_text(text) + relocate_references(self.stage, {"/ref/sw": "/local/sw"}) + for path in self.stage.iterdir(): + self.assertEqual(path.read_text(), text.replace("/ref/sw", "/local/sw")) + relocate_references(self.stage, {"/ref/sw": "/local/sw"}) + + def test_mapping_file_avoids_passing_table_as_argument(self): + config = self.stage / "yoda.pc" + config.write_text("-L/ref/sw/lib\n") + mappings = self.root / "reuse-relocations.tsv" + mappings.write_text(serialize_relocation_table({"/ref/sw": "/local/sw"}) + "\n") + subprocess.run(["bash", str(RELOCATE), str(self.stage), "--file", str(mappings)], + check=True) + self.assertEqual(config.read_text(), "-L/local/sw/lib\n") + + def test_no_symlinks_or_binary_files_are_followed_or_modified(self): + original = self.root / "reference" + original.mkdir() + target = original / "config" + target.write_text("/ref/sw/lib\n") + (self.stage / "linked-file").symlink_to(target) + (self.stage / "linked-directory").symlink_to(original) + binary = self.stage / "library.so" + binary.write_bytes(b"\x7fELF\0/ref/sw/lib") + invalid = self.stage / "other-binary" + invalid.write_bytes(b"\xff/ref/sw/lib") + relocate_references(self.stage, {"/ref/sw": "/local/sw"}) + self.assertEqual(target.read_text(), "/ref/sw/lib\n") + self.assertEqual(binary.read_bytes(), b"\x7fELF\0/ref/sw/lib") + self.assertEqual(invalid.read_bytes(), b"\xff/local/sw/lib") + result = subprocess.run(["bash", str(RELOCATE), str(self.stage / "linked-directory"), + serialize_relocation_table({"/ref/sw": "/local/sw"})], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.assertNotEqual(result.returncode, 0) + + def test_hardlinks_are_broken_without_changing_the_other_link(self): + original = self.root / "source-config" + original.write_text("/ref/sw/lib\n") + original.chmod(0o555) + local = self.stage / "wrapper" + os.link(original, local) + relocate_references(self.stage, {"/ref/sw": "/local/sw"}) + self.assertEqual(original.read_text(), "/ref/sw/lib\n") + self.assertEqual(local.read_text(), "/local/sw/lib\n") + + def test_longest_prefix_and_sequential_literal_replacement_with_spaces(self): + path = self.stage / "a config" + path.write_text('/ref/Packages/gcc/1/lib /ref/Packages/other/lib /ref/Packages-old/lib\n') + relocate_references(self.stage, { + "/ref/Packages": "/local/arch", + "/ref/Packages/gcc/1": "/local/toolchain/external/gcc/1", + }) + self.assertEqual(path.read_text(), + '/local/toolchain/external/gcc/1/lib ' + '/local/arch/other/lib ' + '/local/arch-old/lib\n') + + def test_bsd_grep_and_xargs_options_are_not_used(self): + # BSD grep reads -Z as "decompress", BSD xargs may lack -r: shims refuse both. + shims = self.root / "bin" + shims.mkdir() + for tool, refused in (("grep", "Z"), ("xargs", "r")): + real = shutil.which(tool) + (shims / tool).write_text( + '#!/bin/bash\nfor a in "$@"; do case "$a" in --*) ;; -*%s*) exit 9 ;; esac; done\n' + 'exec %s "$@"\n' % (refused, real)) + (shims / tool).chmod(0o755) + (self.stage / "yoda.pc").write_text("-L/ref/sw/lib\n") + env = dict(os.environ, PATH="%s:%s" % (shims, os.environ["PATH"])) + subprocess.run(["bash", str(RELOCATE), str(self.stage), + serialize_relocation_table({"/ref/sw": "/local/sw"})], check=True, env=env) + self.assertEqual((self.stage / "yoda.pc").read_text(), "-L/local/sw/lib\n") + + @unittest.skipUnless(shutil.which("perl") and shutil.which("grep"), "CMS reference commands required") + def test_matches_cms_relocate_reference_shell_macro(self): + fixtures = { + "paths": b'/ref/sw/lib /ref/sw-other /ref/sw/BUILD/x\n', + "latin": b'\xff /ref/sw/lib\n', + "binary": b'ELF\0/ref/sw/lib', + } + oracle = self.root / "cms" + oracle.mkdir() + outside = self.root / "outside" + outside.write_bytes(b'/ref/sw/lib\n') + for directory in (oracle, self.stage): + for name, content in fixtures.items(): + (directory / name).write_bytes(content) + (directory / "link").symlink_to(outside) + # SPEC_REFERENCE_REPO from cms-sw/pkgtools V00-34-XX, expanded. + subprocess.run(["bash", "-e", "-c", + 'find "$1" -type f | xargs --no-run-if-empty grep -Il "/ref/sw" ' + '| xargs -n1 --no-run-if-empty perl -p -i -e ' + "'s|\\Q/ref/sw\\E|/local/sw|g;'", "cms", str(oracle)], check=True, + env=dict(os.environ, LC_ALL="C")) + relocate_references(self.stage, {"/ref/sw": "/local/sw"}) + for name in fixtures: + self.assertEqual((self.stage / name).read_bytes(), (oracle / name).read_bytes()) + self.assertTrue((self.stage / "link").is_symlink()) + self.assertEqual(outside.read_bytes(), b'/ref/sw/lib\n') + + def test_template_rewrites_before_relocation_list_and_later_move(self): + checkout = Path(__file__).parents[1] + template = (checkout / "bits_helpers/build_template.sh").read_text() + # Execute the actual post-install block, including relativization, + # relocation-list collection and view generation, without compiling. + block = template.split("# Make pkg-config and CMake package files relocation-independent:", 1)[1] + block = "# Make pkg-config and CMake package files relocation-independent:" + block + block = block.split("# Archive creation", 1)[0] + for enabled in (False, True): + with self.subTest(symlinks=enabled): + work = self.root / ("on" if enabled else "off") + pkgpath = "arch/external/yoda/1-hash" + install = work / "INSTALLROOT/hash" / pkgpath + (install / "etc/profile.d").mkdir(parents=True) + (install / "lib/pkgconfig").mkdir(parents=True) + flags = install / "lib/pkgconfig/yoda.pc" + flags.write_text('Libs: -L/ref/sw/arch/external/hdf5/1/lib ' + '-L/ref/sw/arch/external/zlib/1/lib\n') + mappings_file = work / "reuse-relocations.tsv" + if enabled: + mappings_file.write_text(serialize_relocation_table( + {"/ref/sw": str(work)}) + "\n") + env = dict(os.environ, WORK_DIR=str(work), INSTALLROOT=str(install), + PKGHASH="hash", PKGPATH=pkgpath, PKGNAME="yoda", + BITS_SCRIPT_DIR=str(checkout), + BITS_REUSE_RELOCATIONS_FILE=str(mappings_file) if enabled else "") + subprocess.run(["bash", "-e", "-c", block], env=env, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + manifest = (install / "etc/profile.d/.bits-relocate").read_text() + self.assertEqual("lib/pkgconfig/yoda.pc" in manifest, enabled) + self.assertEqual("/ref/sw" in flags.read_text(), not enabled) + if not enabled: + continue + (install / "etc/profile.d/.bits-pkginfo").write_text( + 'PKG_DIR="%s"\nPP="%s"\nPH=hash\n' % (work, pkgpath)) + shutil.copy(checkout / "bits_helpers/relocate-me.sh", install) + moved = self.root / "downloaded" / pkgpath + moved.parent.mkdir(parents=True) + shutil.move(str(install), moved) + env["WORK_DIR"] = str(self.root / "downloaded") + subprocess.run(["bash", str(moved / "relocate-me.sh")], check=True, env=env) + self.assertEqual((moved / "lib/pkgconfig/yoda.pc").read_text(), + 'Libs: -L%s/arch/external/hdf5/1/lib -L%s/arch/external/zlib/1/lib\n' + % (self.root / "downloaded", self.root / "downloaded")) diff --git a/tests/test_remote_reuse.py b/tests/test_remote_reuse.py index ba6983a4..cddf5714 100644 --- a/tests/test_remote_reuse.py +++ b/tests/test_remote_reuse.py @@ -20,6 +20,7 @@ class RevisionResolved(BaseException): ctx.args.defaults = ["release"] ctx.args.develPrefix = ctx.develPackageBranch = "" ctx.cfg.reuse_overlay = None + ctx.cfg.symlink_packages_from = None ctx.syncHelper.writeStore = "" ctx.mainPackage = "fftw" link = "fftw-3.3.10-local1.%s.tar.gz" % arch diff --git a/tests/test_repo_provider.py b/tests/test_repo_provider.py index ebc2d166..1dc6568c 100644 --- a/tests/test_repo_provider.py +++ b/tests/test_repo_provider.py @@ -274,6 +274,125 @@ def test_commit_pin_blank_is_no_pin(self, MockGit, mock_logged_scm, mock_update_ spec, self.work_dir, self.ref_dir, fetch_repos=False) self.assertEqual(got_hash, commit) + # ── a tag that names nothing in the repository ────────────────────────── + def _cloned(self, scm): + return any("clone" in (c[0][0] if c[0] else []) for c in scm.exec.call_args_list) + + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_unknown_tag_dies_before_cloning(self, MockGit, mock_logged_scm, mock_update_ref): + scm = self._mock_scm() + scm.parseRefs.return_value = {"refs/heads/LCG_110": "41623d8010d5631143882c13"} + MockGit.return_value = scm + spec = self._spec() + spec["tag"] = "LCG_110b" + with patch("bits_helpers.log.error") as err, self.assertRaises(SystemExit): + clone_or_update_provider(spec, self.work_dir, self.ref_dir, fetch_repos=False) + self.assertIn("no branch or tag 'LCG_110b'", err.call_args[0][0] % err.call_args[0][1:]) + self.assertFalse(self._cloned(scm)) + self.assertFalse(os.path.exists(os.path.join( + _provider_cache_root(self.work_dir, spec["package"]), "LCG_110b"))) + + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec", + side_effect=lambda _r, _p, spec, **_k: spec.__setitem__("reference", "/mirror")) + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_tag_newer_than_the_mirror_is_found_upstream(self, MockGit, mock_logged_scm, _ref): + scm = self._mock_scm() + scm.listRefsCmd.side_effect = lambda repo: ["ls-remote", repo] + scm.parseRefs.side_effect = lambda out: out + spec = self._spec() + spec["tag"] = "v2" + refs = {"/mirror": {"refs/tags/v1": "1111111111aa"}, + spec["source"]: {"refs/tags/v2": "2222222222bb"}} + mock_logged_scm.side_effect = lambda _s, _p, _r, cmd, *a, **k: refs[cmd[-1]] + MockGit.return_value = scm + _, got = clone_or_update_provider(spec, self.work_dir, self.ref_dir, fetch_repos=False) + self.assertEqual(got, "2222222222bb") + + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm", side_effect=SystemExit(1)) + @patch("bits_helpers.repo_provider.Git") + def test_unlisted_refs_and_a_name_dies(self, MockGit, mock_logged_scm, mock_update_ref): + scm = self._mock_scm() + MockGit.return_value = scm + spec = self._spec() + spec["tag"] = "LCG_110b" + with self.assertRaises(SystemExit): + clone_or_update_provider(spec, self.work_dir, self.ref_dir, fetch_repos=False) + self.assertFalse(self._cloned(scm)) + + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_commit_as_tag_still_clones(self, MockGit, mock_logged_scm, mock_update_ref): + scm = self._mock_scm() + scm.parseRefs.return_value = {"refs/heads/main": "43badb4296a8cd324a90"} + MockGit.return_value = scm + spec = self._spec() + spec["tag"] = "deadbeef1234" + checkout_dir, got = clone_or_update_provider( + spec, self.work_dir, self.ref_dir, fetch_repos=False) + self.assertEqual(got, "deadbeef1234") + self.assertTrue(self._cloned(scm)) + + # ── an unfinished checkout, and another bits fetching the provider ─────── + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_unfinished_checkout_is_redone(self, MockGit, mock_logged_scm, mock_update_ref): + commit = "abcdef1234567890" + scm = self._mock_scm(commit) + MockGit.return_value = scm + spec = self._spec() + stale = os.path.join(_provider_cache_root(self.work_dir, spec["package"]), commit[:10]) + os.makedirs(os.path.join(stale, ".git")) # cloned, never checked out + checkout_dir, _ = clone_or_update_provider( + spec, self.work_dir, self.ref_dir, fetch_repos=False) + self.assertEqual(checkout_dir, stale) + self.assertTrue(self._cloned(scm)) + self.assertFalse(os.path.exists(os.path.join(stale, ".git"))) # removed before the clone + self.assertTrue(os.path.exists(os.path.join(stale, ".bits_provider_ok"))) + + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_waits_for_another_bits_fetching_it(self, MockGit, mock_logged_scm, mock_update_ref): + import fcntl + import threading + scm = self._mock_scm() + MockGit.return_value = scm + spec = self._spec() + root = _provider_cache_root(self.work_dir, spec["package"]) + os.makedirs(root) + holder = open(os.path.join(root, ".lock"), "a") + fcntl.flock(holder, fcntl.LOCK_EX) # the other bits + t = threading.Thread(target=clone_or_update_provider, daemon=True, + args=(spec, self.work_dir, self.ref_dir, False)) + try: + t.start() + t.join(0.5) + self.assertTrue(t.is_alive()) + self.assertFalse(self._cloned(scm)) + finally: + holder.close() # the other bits is done + t.join(5) + self.assertFalse(t.is_alive()) + self.assertTrue(self._cloned(scm)) + + @patch("bits_helpers.repo_provider.warning") + @patch("bits_helpers.repo_provider.fcntl.flock", side_effect=OSError(37, "No locks available")) + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") + @patch("bits_helpers.repo_provider.logged_scm") + @patch("bits_helpers.repo_provider.Git") + def test_no_lock_goes_on_without(self, MockGit, mock_logged_scm, _ref, _flock, warn): + scm = self._mock_scm() + MockGit.return_value = scm + clone_or_update_provider(self._spec(), self.work_dir, self.ref_dir, fetch_repos=False) + self.assertTrue(self._cloned(scm)) + self.assertIn("going on without it", warn.call_args[0][0]) + @patch("bits_helpers.repo_provider.updateReferenceRepoSpec") @patch("bits_helpers.repo_provider.logged_scm") @patch("bits_helpers.repo_provider.Git") diff --git a/tests/test_reuse_report.py b/tests/test_reuse_report.py new file mode 100644 index 00000000..2bd1f5e4 --- /dev/null +++ b/tests/test_reuse_report.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The end-of-build reuse summary: what was reused, what was built, and why.""" + +import os +import shutil +import sys +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from bits_helpers import reuse_report +from bits_helpers.manifest import BuildManifest + + +def _e(pkg, outcome, why=None): + e = {"package": pkg, "outcome": outcome} + if why: + e["rebuilt_because"] = why + return e + + +PACKAGES = ([_e("defaults-release", "built_from_source")] + + [_e("p%d" % i, "already_installed") for i in range(249)] + + [_e("cldconfig", "from_store"), _e("k4rectracker", "from_store"), + _e("key4hep", "from_store"), + _e("fccanalyses", "built_from_source", "store tarball differs from the certified one"), + _e("lcg.bits", "built_from_source", "never stored")]) + + +class TestSummary(unittest.TestCase): + + def test_counts_names_and_reasons(self): + text = reuse_report.format_summary(PACKAGES) + self.assertEqual(text.splitlines(), [ + "Reuse: 254 package(s) — 249 already installed, 3 from the store, 2 built", + " from the store: cldconfig, k4rectracker, key4hep", + " built: fccanalyses (store tarball differs from the certified one), " + "lcg.bits (never stored)", + " built, by reason: 1 store tarball differs from the certified one; 1 never stored"]) + + def test_long_lists_are_cut_and_reasons_counted(self): + built = [_e("b%d" % i, "built_from_source", "not in the store") for i in range(20)] + text = reuse_report.format_summary(built, limit=3) + self.assertIn("built: b0 (not in the store), b1 (not in the store), " + "b2 (not in the store), … 17 more", text) + self.assertIn("built, by reason: 20 not in the store", text) + + def test_nothing_recorded_prints_nothing(self): + self.assertEqual(reuse_report.format_summary([]), "") + + def test_metrics(self): + lines = reuse_report.metric_lines(PACKAGES, "h-1", "x86_64-el9-gcc14-opt", "key4hep") + base = 'instance="h-1",job="bits",arch="x86_64-el9-gcc14-opt",package="key4hep"' + self.assertIn('bits_build_packages{%s,outcome="already_installed"} 249' % base, lines) + self.assertIn('bits_build_packages{%s,outcome="from_store"} 3' % base, lines) + self.assertIn('bits_build_rebuilt{%s,reason="never stored"} 1' % base, lines) + + +class TestRecorded(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_the_manifest_keeps_why_a_package_was_built(self): + m = BuildManifest(work_dir=self.tmp, requested_packages=["X"], architecture="a", + defaults=["release"], config_dir=self.tmp, config_commit="c") + spec = {"package": "X", "version": "1", "revision": "1", "hash": "h" * 40, + "commit_hash": "0", "reuse_note": "not in the store"} + m.add_package(spec, "built_from_source") + m.add_package(dict(spec, package="Y"), "from_store") + got = {e["package"]: e.get("rebuilt_because") for e in m.packages()} + self.assertEqual(got, {"X": "not in the store", "Y": None}) + + def test_reasons(self): + from bits_helpers import build + from bits_helpers.sync import NoRemoteSync + store = SimpleNamespace(writeStore="b3://bucket") # e.g. a CVMFS read + S3 write sync + base = {"package": "P", "is_devel_pkg": False} + cases = [ + (dict(base, is_devel_pkg=True), store, "development package"), + (dict(base, force_rebuild=True), store, "forced rebuild"), + (dict(base, redistributable="none"), store, "licence: never stored"), + (base, NoRemoteSync(), "no remote store"), + (base, store, "not found in the store"), + ] + for spec, helper, want in cases: + with self.subTest(want=want): + with patch.object(build, "is_virtual_package", lambda s: False): + self.assertEqual(build._rebuild_reason(spec, helper), want) + with patch.object(build, "is_virtual_package", lambda s: True): + self.assertEqual(build._rebuild_reason(base, store), "never stored") + + def test_the_report_never_fails_a_build(self): + from bits_helpers import build + args = SimpleNamespace(manifest=SimpleNamespace(packages=lambda: PACKAGES), + architecture="a") + shown = [] + with patch.object(build, "banner", lambda fmt, *a: shown.append(fmt % a)), \ + patch("bits_helpers.metrics.push_prometheus", side_effect=OSError("down")): + build._report_reuse(args, "key4hep", "http://metrics:8428", lambda: "h-1") + self.assertTrue(shown and shown[0].startswith("Reuse: 254 package(s)")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reuse_symlinks.py b/tests/test_reuse_symlinks.py new file mode 100644 index 00000000..991e49a5 --- /dev/null +++ b/tests/test_reuse_symlinks.py @@ -0,0 +1,679 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Exercise reuse through local paths without modifying the deployment.""" + +from argparse import Namespace +import hashlib +import json +import os +from pathlib import Path +from shlex import quote +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from bits_helpers.build import _BuildLoopCtx +from bits_helpers.build import build_one_package +from bits_helpers.build_config import BuildConfig +from bits_helpers.initdotsh import generate_initdotsh +from bits_helpers.reuse_symlinks import stage_reused_package, find_symlink_package +from bits_helpers.reuse_symlinks import stage_hook_scripts + +# What cms.bits' hooks/scram-project does (CMS REFERENCE_PACKAGE_POST). +SCRAM_HOOK = """\ +private() { + [ -L "$1" ] || return 0 + rm "$1" + cp -RH "$SOURCE_ROOT/$1" "$1" + chmod -R u+w "$1" +} +if [ -d "$SOURCE_ROOT/etc/scram.d" ]; then private etc/scram.d; fi +if [ -d "$SOURCE_ROOT/tools/selected" ]; then private tools; fi +if [ -d "$SOURCE_ROOT/.SCRAM" ]; then + private .SCRAM + private config + config/SCRAM/projectAreaRename.pl "$REFERENCE_ROOT" "$WORK_DIR" "$ARCHITECTURE" "$PKG_ROOT" +fi +""" + + +class DeploymentSelectionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) / "deploy" + self.work = Path(self.tmp.name) / "work" + self.arch = "el9_amd64_gcc14" + self.spec = dict(package="data-example", version="V00-03-00", force_revision="", + pkg_family="cms", architecture="share", requires=[], + hash="expected", remote_revision_hash="expected", + local_revision_hash="local", remote_hashes=["expected"], + local_hashes=["local"], is_devel_pkg=False) + + def deploy(self, arch="share", family="cms", label="V00-03-00", hash_value="expected"): + source = self.root / arch / family / self.spec["package"] / label + (source / "etc/profile.d").mkdir(parents=True) + (source / "payload").mkdir() + (source / "payload/data").write_text("unchanged") + if hash_value is not None: + (source / ".build-hash").write_text(hash_value + "\n") + (source / "etc/profile.d/init.sh").write_text( + 'export DATA_EXAMPLE_ROOT="%s"\nexport DATA_EXAMPLE_REVISION=""\n' % source) + return source + + def test_shared_family_package_without_modulefiles_builds_by_symlinking(self): + source = self.deploy() + original = (source / "etc/profile.d/init.sh").read_bytes() + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=None) + cfg = BuildConfig.from_args(args) + self.assertIsNone(cfg.reuse_overlay) + spec = dict(self.spec) + ctx = _BuildLoopCtx( + args=args, cfg=cfg, specs={spec["package"]: spec}, workDir=str(self.work), + syncHelper=None, scheduler=None, mainPackage=spec["package"], + raw_architecture=self.arch, develPackageBranch="", defaultsMeta={}, + buildTargets="", packages=[], prefetch_executor=None, cmake_prefix_env=False, + specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"): + build_one_package(spec["package"], ctx) + local = self.work / "share/cms/data-example/V00-03-00" + self.assertTrue(spec["reuse_symlinked"]) + self.assertEqual(spec["revision"], "") + self.assertNotIn("reuse_module_id", spec) + self.assertTrue((local / "payload").is_symlink()) + self.assertEqual((local / "payload/data").read_text(), "unchanged") + consumer = generate_initdotsh("App", {"App": dict(package="App", requires=["data-example"]), + "data-example": spec}, self.arch, workDir=str(self.work)) + output = subprocess.check_output( + ["bash", "-ec", consumer + '\nprintf "%s" "$DATA_EXAMPLE_ROOT"'], text=True) + self.assertEqual(output, str(local)) + self.assertEqual((source / "etc/profile.d/init.sh").read_bytes(), original) + + def test_manifest_records_the_linked_deployment(self): + from bits_helpers.manifest import BuildManifest + source = self.deploy() + self.work.mkdir() + manifest = BuildManifest(str(self.work), ["data-example"], self.arch, ["release"], + str(self.work), "abc") + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=manifest) + spec = dict(self.spec) + ctx = _BuildLoopCtx( + args=args, cfg=BuildConfig.from_args(args), specs={spec["package"]: spec}, + workDir=str(self.work), syncHelper=None, scheduler=None, + mainPackage=spec["package"], raw_architecture=self.arch, develPackageBranch="", + defaultsMeta={}, buildTargets="", packages=[], prefetch_executor=None, + cmake_prefix_env=False, specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"): + build_one_package(spec["package"], ctx) + entry, = manifest.packages() + self.assertEqual(entry["outcome"], "already_installed") + self.assertEqual(entry["symlinked_from"], str(source)) + self.assertEqual(entry["symlink_policy"], "strict") + + def test_build_runs_the_recipe_stage_hooks(self): + self.deploy() + hooks = Path(self.tmp.name) / "cms.bits/hooks" + hooks.mkdir(parents=True) + (hooks / "mark").write_text('echo "$PKGNAME" > "$PKG_ROOT/hooked"\n') + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=None, + configDir=str(hooks.parent)) + spec = dict(self.spec, hook={"SYMLINK_STAGE": "mark"}) + ctx = _BuildLoopCtx( + args=args, cfg=BuildConfig.from_args(args), specs={spec["package"]: spec}, + workDir=str(self.work), syncHelper=None, scheduler=None, + mainPackage=spec["package"], raw_architecture=self.arch, develPackageBranch="", + defaultsMeta={}, buildTargets="", packages=[], prefetch_executor=None, + cmake_prefix_env=False, specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"), patch.dict(os.environ, {"BITS_PATH": ""}): + build_one_package(spec["package"], ctx) + local = self.work / "share/cms/data-example/V00-03-00" + self.assertEqual((local / "hooked").read_text(), "data-example\n") + + def test_matching_local_install_is_kept(self): + from bits_helpers.plan import classify, INSTALLED + from bits_helpers.reuse_symlinks import select_symlink_package + source = self.deploy() + local = self.work / "share/cms/data-example/V00-03-00" + local.parent.mkdir(parents=True) + shutil.copytree(source, local) + cfg = BuildConfig.from_args(Namespace(symlinkPackagesFrom=str(self.root))) + spec = dict(self.spec) + self.assertEqual(classify(spec, self.arch, str(self.work), None, False, False, + None, symlink=lambda s: select_symlink_package(s, cfg, self.arch)), + (INSTALLED, "expected", "")) + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=None) + ctx = _BuildLoopCtx( + args=args, cfg=cfg, specs={spec["package"]: spec}, workDir=str(self.work), + syncHelper=None, scheduler=None, mainPackage=spec["package"], + raw_architecture=self.arch, develPackageBranch="", defaultsMeta={}, + buildTargets="", packages=[], prefetch_executor=None, cmake_prefix_env=False, + specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"): + build_one_package(spec["package"], ctx) + self.assertFalse(local.is_symlink()) + self.assertNotIn("reuse_symlinked", spec) + self.assertEqual(spec["hash"], "expected") + + def test_shared_package_sources_arch_specific_dependency(self): + source = self.deploy() + (source / "etc/profile.d/init.sh").write_text( + ': "${BITS_ARCH_PREFIX:=el9_amd64_gcc14}"\n' + '. "$WORK_DIR/$BITS_ARCH_PREFIX/cms/cms-common/1/etc/profile.d/init.sh"\n' + 'export DATA_EXAMPLE_ROOT="$WORK_DIR/share/cms/data-example/V00-03-00"\n') + common = self.work / self.arch / "cms/cms-common/1/etc/profile.d" + common.mkdir(parents=True) + (common / "init.sh").write_text( + 'export CMS_COMMON_ROOT="$WORK_DIR/$BITS_ARCH_PREFIX/cms/cms-common/1"\n') + match = find_symlink_package(self.spec, self.root, self.arch) + local = stage_reused_package(self.spec, match["install_base"], self.work, + self.arch, source_path=match["source"]) + output = subprocess.check_output( + ["bash", "-e", "-c", + 'BITS_ARCH_PREFIX=caller; . "$1"; ' + 'printf "%s\\n%s\\n" "$CMS_COMMON_ROOT" "$BITS_ARCH_PREFIX"', + "bash", str(Path(local) / "etc/profile.d/init.sh")], text=True) + self.assertEqual(output.splitlines(), + [str(self.work / self.arch / "cms/cms-common/1"), "caller"]) + + def test_stale_shared_stage_is_refreshed_before_build(self): + source = self.deploy() + (source / "etc/profile.d/init.sh").write_text( + ': "${BITS_ARCH_PREFIX:=el9_amd64_gcc14}"\n' + '. "$WORK_DIR/$BITS_ARCH_PREFIX/defaults-release/1/etc/profile.d/init.sh"\n') + defaults = self.work / self.arch / "defaults-release/1/etc/profile.d" + defaults.mkdir(parents=True) + (defaults / "init.sh").write_text('export DEFAULTS_RELEASE_REVISION=loaded\n') + match = find_symlink_package(self.spec, self.root, self.arch) + local = Path(stage_reused_package(self.spec, match["install_base"], self.work, + self.arch, source_path=match["source"])) + local_init = local / "etc/profile.d/init.sh" + fresh = local_init.read_text() + stale = fresh.replace("BITS_ARCH_PREFIX=%s\n" % self.arch, + "BITS_ARCH_PREFIX=share\n", 1) + self.assertNotEqual(stale, fresh) + local_init.write_text(stale) + marker = local / ".bits-reuse-symlinks.json" + old_identity = json.loads(marker.read_text()) + old_identity["init_sha256"] = hashlib.sha256(stale.encode()).hexdigest() + marker.write_text(json.dumps(old_identity, sort_keys=True) + "\n") + + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=None) + cfg = BuildConfig.from_args(args) + spec = dict(self.spec) + ctx = _BuildLoopCtx( + args=args, cfg=cfg, specs={spec["package"]: spec}, workDir=str(self.work), + syncHelper=None, scheduler=None, mainPackage=spec["package"], + raw_architecture=self.arch, develPackageBranch="", defaultsMeta={}, + buildTargets="", packages=[], prefetch_executor=None, cmake_prefix_env=False, + specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"): + build_one_package(spec["package"], ctx) + self.assertTrue(spec["reuse_symlinked"]) + self.assertEqual(local_init.read_text(), fresh) + output = subprocess.check_output( + ["bash", "-e", "-c", '. "$1"; printf "%s" "$DEFAULTS_RELEASE_REVISION"', + "bash", str(local_init)], text=True) + self.assertEqual(output, "loaded") + + def test_strict_checks_hash_relaxed_uses_path(self): + source = self.deploy(hash_value="different") + self.assertIsNone(find_symlink_package(self.spec, self.root, self.arch)) + self.assertEqual(find_symlink_package(self.spec, self.root, self.arch, "relaxed")["source"], str(source)) + (source / ".build-hash").unlink() + self.assertIsNone(find_symlink_package(self.spec, self.root, self.arch)) + self.assertIsNotNone(find_symlink_package(self.spec, self.root, self.arch, "relaxed")) + (source / ".build-hash").write_text("local\n") + self.assertIsNone(find_symlink_package(self.spec, self.root, self.arch)) + self.assertEqual(find_symlink_package(self.spec, self.root, self.arch, "relaxed")["hash"], "local") + + def test_architecture_and_version_are_required_in_both_policies(self): + self.deploy(arch="wrong-arch") + self.deploy(label="V00-03-01") + for policy in ("strict", "relaxed"): + self.assertIsNone(find_symlink_package(self.spec, self.root, self.arch, policy)) + + def test_arch_specific_and_flat_layouts_and_symlinked_package(self): + self.spec.pop("architecture") + self.spec["pkg_family"] = "external" + source = self.deploy(arch=self.arch, family="") + moved = source.with_name("payload-directory") + source.rename(moved) + source.symlink_to(moved, target_is_directory=True) + match = find_symlink_package(self.spec, self.root, self.arch) + self.assertEqual(match["source"], str(source)) + local = Path(stage_reused_package(self.spec, match["install_base"], self.work, + self.arch, source_path=match["source"])) + self.assertEqual(local, self.work / self.arch / "external/data-example/V00-03-00") + self.assertEqual((local / "payload/data").read_text(), "unchanged") + self.spec["pkg_family"] = "" + self.assertIsNotNone(find_symlink_package(self.spec, self.root, self.arch)) + + def test_revision_policy_and_dry_run_match_selection(self): + from bits_helpers.plan import classify, SYMLINK + from bits_helpers.reuse_symlinks import select_symlink_package + source = self.deploy(label="V00-03-00-older", hash_value="different") + # An explicit empty revision must not accidentally match a suffixed one. + self.assertIsNone(find_symlink_package(self.spec, self.root, self.arch, "relaxed")) + self.spec.update(force_revision="expected", _revision_policy_hash_injected=True) + cfg = BuildConfig.from_args(Namespace( + symlinkPackagesFrom=str(self.root), symlinkPolicy="relaxed")) + result = classify(self.spec, self.arch, str(self.work), None, False, False, None, + symlink=lambda spec: select_symlink_package(spec, cfg, self.arch)) + self.assertEqual(result, (SYMLINK, "different", "older")) + self.assertFalse(self.work.exists()) + self.assertEqual(select_symlink_package(self.spec, cfg, self.arch)["source"], str(source)) + blocked = BuildConfig.from_args(Namespace( + symlinkPackagesFrom=str(self.root), buildLocal=[self.spec["package"]])) + self.assertIsNone(select_symlink_package(self.spec, blocked, self.arch)) + for changes in ({"is_devel_pkg": True}, {"provides_repository": True}, + {"package": "defaults-release"}): + self.assertIsNone(select_symlink_package(dict(self.spec, **changes), cfg, self.arch)) + + def test_strict_selection_needs_a_certified_hash(self): + from types import SimpleNamespace + from bits_helpers import plan + from bits_helpers.reuse_symlinks import select_symlink_package + from bits_helpers.sync import NoRemoteSync + self.deploy() + certified = {("share", "expected"): "sha256"} + cfg = SimpleNamespace(symlink_packages_from=str(self.root), symlink_policy="strict", + build_local=[], require_signed_reuse=True) + self.assertIsNone(select_symlink_package(self.spec, cfg, self.arch, trusted={})) + self.assertIsNotNone(select_symlink_package(self.spec, cfg, self.arch, trusted=certified)) + relaxed = SimpleNamespace(**dict(vars(cfg), symlink_policy="relaxed")) + self.assertIsNotNone(select_symlink_package(self.spec, relaxed, self.arch, trusted={})) + # The plan asks for the signed index and classifies as the build does. + args = SimpleNamespace(defaults=["release"], architecture=self.arch) + for trusted, state in (({}, plan.BUILD), (certified, plan.SYMLINK)): + spec = dict(self.spec) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.hashing.storeHashes"), \ + patch.object(plan, "_overlay_predicate", lambda cfg: None): + states, _, _ = plan.plan([spec["package"]], {spec["package"]: spec}, args, + str(self.work), NoRemoteSync(), self.arch, cfg, + lambda: trusted) + self.assertEqual(states[spec["package"]], state) + + def test_plan_does_not_list_symlinked_packages_in_the_store(self): + from types import SimpleNamespace + from bits_helpers import plan + from bits_helpers.sync import RemoteSync + self.deploy() + listed = [] + + class Store(RemoteSync): + remoteStore = "b3://store" + + def list_store_tarballs(self, arch, pkg_hash): + listed.append(pkg_hash) + return [] + cfg = SimpleNamespace(symlink_packages_from=str(self.root), symlink_policy="strict", + build_local=[], require_signed_reuse=False) + args = SimpleNamespace(defaults=["release"], architecture=self.arch) + spec = dict(self.spec) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.hashing.storeHashes"), \ + patch.object(plan, "_overlay_predicate", lambda cfg: None): + states, _, _ = plan.plan([spec["package"]], {spec["package"]: spec}, args, + str(self.work), Store(), self.arch, cfg, None) + self.assertEqual(states[spec["package"]], plan.SYMLINK) + self.assertEqual(listed, []) + + def test_build_selects_with_the_signed_index(self): + class Selected(Exception): + pass + seen = [] + + def select(spec, cfg, architecture, trusted=None): + seen.append(trusted) + raise Selected() + for signed in (True, False): + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + symlinkPackagesFrom=str(self.root), manifest=None, + requireSignedReuse=signed) + spec = dict(self.spec) + ctx = _BuildLoopCtx( + args=args, cfg=BuildConfig.from_args(args), specs={spec["package"]: spec}, + workDir=str(self.work), syncHelper=None, scheduler=None, + mainPackage=spec["package"], raw_architecture=self.arch, develPackageBranch="", + defaultsMeta={}, buildTargets="", packages=[], prefetch_executor=None, + cmake_prefix_env=False, specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"), \ + patch("bits_helpers.build.trusted_reuse_index", return_value={"k": "v"}), \ + patch("bits_helpers.reuse_symlinks.select_symlink_package", select), \ + self.assertRaises(Selected): + build_one_package(spec["package"], ctx) + self.assertEqual(seen, [{"k": "v"}, None]) + + +class ReuseSymlinksTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() # macOS: /var is /private/var + self.base = self.root / "deployment" / "Packages" + self.work = self.root / "local" + self.arch = "el9_amd64_gcc14" + self.spec = dict(package="gcc", version="14", revision="hash", hash="hash", + commit_hash="commit", requires=[], pkg_family="external", + reuse_module_id="gcc/14-hash", is_devel_pkg=False, + remote_revision_hash="hash", remote_hashes=["hash"], local_hashes=[]) + self.source = self.base / "external/gcc/14-hash" + (self.source / "etc/profile.d").mkdir(parents=True) + (self.source / "bin").mkdir() + (self.source / "lib").mkdir() + (self.source / "lib/libgcc.so").write_text("unchanged library") + (self.source / "bin/gcc").write_text("#!/bin/sh\nprintf 'compiler works\\n'\n") + (self.source / "bin/gcc").chmod(0o755) + self.original = generate_initdotsh( + "gcc", {"gcc": self.spec}, self.arch, workDir=str(self.root / "old-build"), + post_build=True) + # Cover a literal deployment path as well as WORK_DIR-based paths. + self.original += '\nexport GCC_EXTRA="%s/include"\n' % self.source + (self.source / "etc/profile.d/init.sh").write_text(self.original) + (self.source / "etc/profile.d/.bits-pkginfo").write_text( + 'PKG_DIR="%s"\n' % (self.root / "old-build")) + + def stage(self, **kwargs): + return Path(stage_reused_package(self.spec, str(self.base), str(self.work), + self.arch, source_path=self.source, **kwargs)) + + def environment(self, script): + result = subprocess.run(["bash", "-e", "-c", script + "\nenv -0"], + env={"PATH": "/usr/bin:/bin"}, check=True, + stdout=subprocess.PIPE) + return dict(v.split("=", 1) for v in result.stdout.decode().split("\0") if "=" in v) + + def test_local_environment_links_payload_and_preserves_source(self): + local = self.stage() + self.spec["reuse_symlinked"] = True + app = dict(package="App", requires=["gcc"]) + consumer = generate_initdotsh("App", {"App": app, "gcc": self.spec}, self.arch, + workDir=str(self.work), reuse_cvmfs_base=str(self.base)) + env = self.environment(consumer) + self.assertEqual(env["GCC_ROOT"], str(local)) + self.assertEqual(env["GCC_EXTRA"], str(local / "include")) + self.assertEqual(env["PATH"].split(":")[0], str(local / "bin")) + self.assertIn(str(local / "lib"), env["LD_LIBRARY_PATH"].split(":")) + self.assertNotIn(str(self.base), consumer) + self.assertTrue((local / "bin").is_symlink()) + self.assertFalse((local / "etc/profile.d/init.sh").is_symlink()) + self.assertEqual((local / "lib/libgcc.so").read_text(), "unchanged library") + self.assertEqual((self.source / "etc/profile.d/init.sh").read_text(), self.original) + output = subprocess.check_output([str(local / "bin/gcc")], text=True) + self.assertEqual(output.strip(), "compiler works") + + def test_new_consumer_paths_are_handled_by_existing_relocator(self): + local = self.stage() + env = self.environment('. %s' % quote(str(local / "etc/profile.d/init.sh"))) + suffix = self.arch + "/external/App/1-hash" + app = self.work / suffix + (app / "etc/profile.d").mkdir(parents=True) + (app / "flags.pc").write_text("Libs: -L%s/lib\n" % env["GCC_ROOT"]) + (app / "etc/profile.d/.bits-relocate").write_text("flags.pc\n") + (app / "etc/profile.d/.bits-pkginfo").write_text( + 'PKG_DIR="%s"\nPP="%s"\nPH=hash\n' % (self.work, suffix)) + shutil.copy(Path(__file__).parents[1] / "bits_helpers/relocate-me.sh", app) + relocated = self.root / "downloaded" / suffix + relocated.parent.mkdir(parents=True) + shutil.move(str(app), relocated) + subprocess.run(["bash", str(relocated / "relocate-me.sh")], check=True, + env={"PATH": "/usr/bin:/bin", "WORK_DIR": str(self.root / "downloaded")}) + self.assertEqual((relocated / "flags.pc").read_text(), + "Libs: -L%s/%s/external/gcc/14-hash/lib\n" + % (self.root / "downloaded", self.arch)) + + def test_idempotent_and_adopts_compatible_existing_layouts(self): + local = self.stage() + self.assertEqual(self.stage(), local) + (local / ".bits-reuse-symlinks.json").unlink() + # A marker can be lost while the prepared symlink tree remains. Rebuild + # its private files and adopt its compatible source links. + self.assertEqual(self.stage(), local) + self.assertFalse((local / "etc/profile.d/init.sh").is_symlink()) + shutil.rmtree(local) + local.symlink_to(self.source) + self.assertEqual(self.stage(), local) + self.assertFalse(local.is_symlink()) + self.assertTrue((local / "bin").is_symlink()) + self.assertFalse((local / "etc/profile.d/init.sh").is_symlink()) + self.assertEqual((self.source / "etc/profile.d/init.sh").read_text(), self.original) + + def test_compatible_partial_links_are_adopted_but_foreign_links_rejected(self): + local = self.work / self.arch / "external/gcc/14-hash" + local.mkdir(parents=True) + (local / "bin").symlink_to(self.source / "bin") + (local / "lib").symlink_to(self.source / "lib") + self.assertEqual(self.stage(), local) + self.assertTrue((local / "bin").is_symlink()) + self.assertFalse((local / "etc/profile.d/init.sh").is_symlink()) + + shutil.rmtree(local) + local.mkdir(parents=True) + foreign = self.root / "foreign" + foreign.mkdir() + (local / "bin").symlink_to(foreign) + with self.assertRaisesRegex(ValueError, r"first conflict: .*/bin\)"): + self.stage() + self.assertEqual((self.source / "etc/profile.d/init.sh").read_text(), self.original) + + def test_missing_init_fails_without_staging(self): + (self.source / "etc/profile.d/init.sh").unlink() + with self.assertRaisesRegex(ValueError, "needs the deployed init.sh"): + self.stage() + self.assertFalse(self.work.exists()) + + def test_linked_destination_parent_is_rejected(self): + (self.work / self.arch).mkdir(parents=True) + (self.work / self.arch / "external").symlink_to(self.base / "external") + with self.assertRaisesRegex(ValueError, "linked destination parent"): + self.stage() + self.assertEqual((self.source / "etc/profile.d/init.sh").read_text(), self.original) + + def test_flat_source_can_stage_under_local_family(self): + flat = self.base / "gcc/14-hash" + flat.parent.mkdir(parents=True) + shutil.move(str(self.source), flat) + self.source = flat + (flat / "etc/profile.d/init.sh").write_text( + 'export GCC_ROOT="%s"\nexport GCC_REVISION=hash\n' % flat) + local = self.stage() + env = self.environment('. %s' % quote(str(local / "etc/profile.d/init.sh"))) + self.assertEqual(env["GCC_ROOT"], str(local)) + + def test_effective_arch_and_caller_arch_restoration(self): + for effective in ("share", "el9-toolchain"): + with self.subTest(effective=effective): + if effective == "share": + self.spec["architecture"] = "share" + else: + self.spec.pop("architecture", None) + self.spec["_own_hash_arch"] = effective + # The source init.sh must reflect the effective architecture + # of the package being staged, as a real deployment does. + (self.source / "etc/profile.d/init.sh").write_text(generate_initdotsh( + "gcc", {"gcc": self.spec}, self.arch, + workDir=str(self.root / "old-build"), post_build=True)) + local = self.stage() + env = self.environment('export BITS_ARCH_PREFIX=consumer\n. %s' + % quote(str(local / "etc/profile.d/init.sh"))) + self.assertEqual(env["GCC_ROOT"], str(local)) + self.assertEqual(env["BITS_ARCH_PREFIX"], "consumer") + + def test_container_init_uses_container_workdir(self): + local = self.stage(runtime_work_dir="/container/bits/sw") + init = (local / "etc/profile.d/init.sh").read_text() + self.assertIn('/container/bits/sw/%s/external/gcc/14-hash/include' % self.arch, init) + self.assertNotIn(str(self.base), init) + self.assertNotIn(str(self.root / "old-build"), init) + + def test_transitive_initialization_uses_local_dependencies(self): + tools = dict(self.spec, package="Tools", pkg_family="", version="1", + reuse_module_id="Tools/1-hash", requires=[]) + source = self.base / "Tools/1-hash" + (source / "etc/profile.d").mkdir(parents=True) + (source / "bin").mkdir() + (source / "etc/profile.d/init.sh").write_text(generate_initdotsh( + "Tools", {"Tools": tools}, self.arch, workDir=str(self.base), post_build=True)) + self.spec["requires"] = ["Tools"] + (self.source / "etc/profile.d/init.sh").write_text(generate_initdotsh( + "gcc", {"gcc": self.spec, "Tools": tools}, self.arch, + workDir=str(self.base), post_build=True)) + tool_root = stage_reused_package(tools, str(self.base), str(self.work), self.arch, + source_path=source) + local = self.stage() + env = self.environment('export BITS_ARCH_PREFIX=consumer\n. %s' + % quote(str(local / "etc/profile.d/init.sh"))) + self.assertEqual(env["TOOLS_ROOT"], tool_root) + self.assertEqual(env["GCC_ROOT"], str(local)) + self.assertEqual(env["BITS_ARCH_PREFIX"], "consumer") + + def test_changed_source_is_not_silently_reused(self): + self.stage() + (self.source / "etc/profile.d/init.sh").write_text(self.original + "\n# changed\n") + with self.assertRaisesRegex(ValueError, "incompatible existing content"): + self.stage() + + def scram_hook(self): + """A recipe repository's SYMLINK_STAGE hook doing what CMS does for SCRAM.""" + hooks = self.root / "cms.bits/hooks" + hooks.mkdir(parents=True, exist_ok=True) + (hooks / "scram-project").write_text(SCRAM_HOOK) + spec = dict(self.spec, hook={"SYMLINK_STAGE": "scram-project"}) + with patch.dict(os.environ, {"BITS_PATH": ""}): + return stage_hook_scripts(spec, self.root / "cms.bits") + + def test_stage_hooks_are_found_as_build_hooks_are(self): + self.assertEqual(stage_hook_scripts(self.spec, "/nowhere"), []) + self.assertEqual(self.scram_hook(), [str(self.root / "cms.bits/hooks/scram-project")]) + with patch.dict(os.environ, {"BITS_PATH": "other"}): + with self.assertRaisesRegex(ValueError, "missing is in no hooks/ of other, cms"): + stage_hook_scripts(dict(self.spec, hook={"SYMLINK_STAGE": "missing"}), + self.root / "cms.bits") + + def test_stage_hook_is_not_part_of_the_hash(self): + from bits_helpers.hashing import storeHashes + hashes = [] + for hook in ({}, {"SYMLINK_STAGE": "scram-project"}, {"POST_INSTALL": "x"}): + specs = {"p": dict(package="p", version="1", recipe="", requires=[], + is_devel_pkg=False, commit_hash="0", hook=hook)} + storeHashes("p", specs, False) + hashes.append(specs["p"]["remote_revision_hash"]) + self.assertEqual(hashes[0], hashes[1]) + self.assertNotEqual(hashes[0], hashes[2]) + + def test_cms_copy_exceptions_and_reference_relocation(self): + copies = ["etc/profile.d/init.csh", "etc/profile.d/extra.sh", + "etc/scram.d/tool.xml", "tools/selected/tool.xml", "tools/other"] + links = ["etc/other/file", "config/file"] + for name in copies + links: + path = self.source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(str(self.source) + "/lib\n") + path.chmod(0o444) + # rsync -a keeps links inside copied directories as links. + (self.source / "etc/profile.d/link").symlink_to(self.source / "etc/other/file") + # bits itself copies only etc/profile.d; the rest is the hook's. + plain = self.stage() + self.assertTrue((plain / "etc/scram.d").is_symlink()) + self.assertTrue((plain / "tools").is_symlink()) + self.assertFalse((plain / "etc/profile.d/extra.sh").is_symlink()) + shutil.rmtree(plain) + local = self.stage(stage_hooks=self.scram_hook()) + for name in copies: + self.assertFalse((local / name).is_symlink()) + self.assertEqual((local / name).read_text(), str(local) + "/lib\n") + self.assertTrue(os.stat(local / name).st_mode & 0o200) + self.assertEqual((self.source / name).read_text(), str(self.source) + "/lib\n") + self.assertTrue((local / "etc/other").is_symlink()) + self.assertTrue((local / "config").is_symlink()) + self.assertTrue((local / "etc/profile.d/link").is_symlink()) + self.assertEqual(Path(self.stage(stage_hooks=self.scram_hook())), local) + + def test_scram_project_rename_runs_before_text_replacement(self): + (self.source / ".SCRAM").mkdir() + (self.source / ".SCRAM/Environment").write_text(str(self.base) + "/payload\n") + script = self.source / "config/SCRAM/projectAreaRename.pl" + script.parent.mkdir(parents=True) + script.write_text('#!/bin/sh\n' + 'test "$(cat "$4/.SCRAM/Environment")" = "$1/payload" || exit 1\n' + 'printf "%s\\n" "$1" "$2" "$3" "$4" > "$4/rename-args"\n' + 'cat "$4/.SCRAM/Environment" > "$4/before-rewrite"\n') + script.chmod(0o555) + local = self.stage(stage_hooks=self.scram_hook()) + self.assertFalse((local / ".SCRAM").is_symlink()) + self.assertFalse((local / "config").is_symlink()) + self.assertEqual((local / "before-rewrite").read_text(), + str(self.work / self.arch) + "/payload\n") + self.assertEqual((self.source / ".SCRAM/Environment").read_text(), + str(self.base) + "/payload\n") + self.assertFalse((self.source / "rename-args").exists()) + self.assertIn(str(local), (local / "rename-args").read_text()) + + def test_scram_hook_copies_a_linked_directory_not_the_link(self): + real = self.source / "tools-1" + (real / "selected").mkdir(parents=True) + (real / "selected/tool.xml").write_text(str(self.source) + "/lib\n") + (self.source / "tools").symlink_to("tools-1") + local = self.stage(stage_hooks=self.scram_hook()) + self.assertFalse((local / "tools").is_symlink()) + self.assertEqual((local / "tools/selected/tool.xml").read_text(), str(local) + "/lib\n") + self.assertEqual((real / "selected/tool.xml").read_text(), str(self.source) + "/lib\n") + + def test_interrupted_hook_stage_is_made_again(self): + hooks = self.scram_hook() + local = self.stage(stage_hooks=hooks) + marker = local / ".bits-reuse-symlinks.json" + # As left by a process killed in a hook: unfinished, deployed init.sh. + marker.write_text(json.dumps(dict(json.loads(marker.read_text()), complete=False))) + (local / "etc/profile.d/init.sh").write_text(self.original) + self.assertEqual(self.stage(stage_hooks=hooks), local) + self.assertNotIn("complete", json.loads(marker.read_text())) + self.assertNotEqual((local / "etc/profile.d/init.sh").read_text(), self.original) + # Someone else's content there is still refused. + (local / "etc/profile.d/init.sh").write_text(self.original) + with self.assertRaisesRegex(ValueError, "incompatible existing content"): + self.stage(stage_hooks=hooks) + + def test_failed_stage_hook_leaves_no_stage(self): + hook = self.root / "fail" + hook.write_text("exit 3\n") + with self.assertRaisesRegex(ValueError, "stage hook .* failed for gcc"): + self.stage(stage_hooks=[str(hook)]) + self.assertFalse(os.path.lexists(self.work / self.arch / "external/gcc/14-hash")) + + def test_module_overlay_does_not_stage_local_symlinks(self): + spec = dict(self.spec) + spec.pop("reuse_module_id") + args = Namespace(defaults=["release"], architecture=self.arch, docker=False, + reuseOverlay="overlay", manifest=None, + reuseCvmfsBase=str(self.base)) + ctx = _BuildLoopCtx( + args=args, cfg=BuildConfig.from_args(args), specs={"gcc": spec}, + workDir=str(self.work), syncHelper=None, scheduler=None, + mainPackage="gcc", raw_architecture=self.arch, develPackageBranch="", + defaultsMeta={}, buildTargets="", packages=[], prefetch_executor=None, + cmake_prefix_env=False, specs_for_checksum_phase=[], monitoredDirs={}) + with patch("bits_helpers.build.storeHook"), \ + patch("bits_helpers.build.storeHashes"), \ + patch("bits_helpers.build.log_current_package"), \ + patch("bits_helpers.cvmfs_import.overlay_reuse_module", return_value="gcc/14-hash"): + build_one_package("gcc", ctx) + self.assertNotIn("reuse_symlinked", spec) + self.assertFalse(self.work.exists()) + init = generate_initdotsh("App", {"App": dict(package="App", requires=["gcc"]), "gcc": spec}, + self.arch, workDir=str(self.work), + reuse_cvmfs_base=str(self.base)) + self.assertIn(str(self.base), init) diff --git a/tests/test_rev_marker_s3.py b/tests/test_rev_marker_s3.py index f487b437..26e3cdb5 100644 --- a/tests/test_rev_marker_s3.py +++ b/tests/test_rev_marker_s3.py @@ -189,14 +189,15 @@ class _BareReader(sync.RemoteSync): class DualDelegationTestCase(unittest.TestCase): """DualRemoteSync routes store-metadata reads to reader, then writer.""" - def test_delegates_read_rev_markers_to_reader(self): + def test_merges_read_rev_markers_the_reader_winning(self): + # Shared numbering: a revision either store uses is never handed out again. reader = MagicMock() reader.read_rev_markers.return_value = {"2": "bb"} writer = MagicMock() + writer.read_rev_markers.return_value = {"2": "cc", "3": "dd"} dual = sync.DualRemoteSync(reader=reader, writer=writer) - self.assertEqual(dual.read_rev_markers("fftw", "3.3.10", ARCH), {"2": "bb"}) + self.assertEqual(dual.read_rev_markers("fftw", "3.3.10", ARCH), {"2": "bb", "3": "dd"}) reader.read_rev_markers.assert_called_once_with("fftw", "3.3.10", ARCH) - writer.read_rev_markers.assert_not_called() def test_falls_back_to_writer_when_reader_lacks_method(self): # REGRESSION: `--remote-store https://…::rw` builds DualRemoteSync(Http, Boto3). diff --git a/tests/test_runtime_dir.py b/tests/test_runtime_dir.py index 8440ef78..e879c477 100644 --- a/tests/test_runtime_dir.py +++ b/tests/test_runtime_dir.py @@ -32,14 +32,12 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.inst = os.path.join(self.tmp, "inst") os.makedirs(self.inst) - for f in ("bits", "bitsBuild", "bitsStore", "bitsenv"): + for f in ("bits", "bitsBuild", "bitsStore"): shutil.copy2(os.path.join(ROOT, f), self.inst) shutil.copytree(os.path.join(ROOT, "bits_helpers"), os.path.join(self.inst, "bits_helpers"), ignore=shutil.ignore_patterns("__pycache__")) self.work = os.path.join(self.tmp, "work") os.makedirs(os.path.join(self.work, "sw", "MODULES", "slc7_x86-64")) - self.tree = os.path.join(self.tmp, "tree") # a published modules tree - os.makedirs(os.path.join(self.tree, "x86_64-el9-gcc15-opt", "Modules", "modulefiles")) self.path_log = os.path.join(self.tmp, "path.log") self.runtime_log = os.path.join(self.tmp, "runtime.log") fakebin = os.path.join(self.tmp, "fakebin") @@ -60,11 +58,6 @@ def _bits(self, *args): return subprocess.run([os.path.join(self.inst, "bits")] + list(args), cwd=self.work, env=self.env, capture_output=True, text=True, timeout=120) - def _bitsenv(self, *args): - return subprocess.run([os.path.join(self.inst, "bitsenv"), "-p", "x86_64-el9-gcc15-opt"] + list(args), - cwd=self.work, env=dict(self.env, BITS_MODULEDIR=self.tree), - capture_output=True, text=True, timeout=120) - def _log(self, path): if not os.path.exists(path): return "" @@ -75,12 +68,10 @@ def test_without_runtime_python_and_modulecmd_come_from_path(self): self._bits("use") self._bits("modulecmd", "--version") self._bits("store", "--help") - self._bitsenv("q") log = self._log(self.path_log) self.assertIn("fakebin/python3 -c", log) self.assertIn("fakebin/modulecmd --version", log) self.assertIn("fakebin/python3 - --help", log) - self.assertIn("fakebin/modulecmd bash -t avail", log) def test_with_runtime_bits_uses_it_and_not_path(self): self._add_runtime() @@ -89,13 +80,11 @@ def test_with_runtime_bits_uses_it_and_not_path(self): self.assertEqual(r.returncode, 0, r.stderr) self._bits("modulecmd", "--version") self._bits("store", "--help") - self._bitsenv("q") log = self._log(self.runtime_log) self.assertIn("runtime/bin/python3 -E -s -c", log) self.assertIn("runtime/bin/python3 -E -s %s/bitsBuild version" % self.inst, log) self.assertIn("runtime/bin/modulecmd --version", log) self.assertIn("runtime/bin/python3 -E -s - --help", log) - self.assertIn("runtime/bin/modulecmd bash -t avail", log) self.assertEqual(self._log(self.path_log), "") def test_with_runtime_the_users_pythonpath_is_ignored(self): diff --git a/tests/test_runtime_without_python.py b/tests/test_runtime_without_python.py new file mode 100644 index 00000000..02942952 --- /dev/null +++ b/tests/test_runtime_without_python.py @@ -0,0 +1,237 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""The environment commands (q, enter, load, printenv, setenv) start no Python +unless they need it, e.g. for a grid job using a community's CVMFS modules. + +python3 is a stub that records being run (and fails). +""" + +import hashlib +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ARCH = "x86_64-el9-gcc14-opt" + + +def _write(path, text, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + os.chmod(path, mode) + + +class _Bits(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.inst = os.path.join(self.tmp, "inst") + os.makedirs(self.inst) + for f in ("bits", "bitsBuild"): + shutil.copy2(os.path.join(ROOT, f), self.inst) + self.calls = os.path.join(self.tmp, "calls") + stubs = os.path.join(self.tmp, "stubs") + _write(os.path.join(stubs, "python3"), + '#!/bin/sh\necho "python3 $*" >> "%s"\nexit 1\n' % self.calls, 0o755) + _write(os.path.join(stubs, "modulecmd"), "#!/bin/sh\nexit 0\n", 0o755) + self.work = os.path.join(self.tmp, "work") + _write(os.path.join(self.work, "sw", ARCH, "A", "1-1", "etc", "modulefiles", "A"), + "#%Module1.0\n") + self.env = {"HOME": self.tmp, "PATH": stubs + ":/usr/bin:/bin", "MODULES_SHELL": "bash"} + + def _bits(self, *args, cwd=None): + return subprocess.run([os.path.join(self.inst, "bits")] + list(args), + cwd=cwd or self.work, env=self.env, capture_output=True, text=True) + + def _python_ran(self): + return os.path.exists(self.calls) + + def _host(self): + import platform + sys.path.insert(0, os.path.join(ROOT, "tests")) + from test_cvmfs_modules import _host_os + if not _host_os(): + self.skipTest("bits picks the host's arch on el and Ubuntu hosts only") + return "%s-%s-gcc14" % (platform.machine(), _host_os()) + + def _real_modulecmd(self): + sys.path.insert(0, os.path.join(ROOT, "tests")) + from test_release_view import _modulecmd + m = _modulecmd() + if not m: + self.skipTest("needs modulecmd (Environment Modules)") + os.unlink(os.path.join(self.tmp, "stubs", "modulecmd")) + os.symlink(m, os.path.join(self.tmp, "stubs", "modulecmd")) + + +class RuntimeWithoutPythonTest(_Bits): + def test_a_profile_of_another_directory_starts_no_python(self): + _write(os.path.join(self.tmp, ".bits", "use", "0123456789abcdef.use"), "[common]\n") + r = self._bits("-a", ARCH, "q") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("A/1-1", r.stdout) + self.assertFalse(self._python_ran()) + + def test_this_directorys_profile_is_read(self): + key = hashlib.sha256(self.work.encode()).hexdigest()[:16] + _write(os.path.join(self.tmp, ".bits", "use", key + ".use"), "[common]\n") + self._bits("-a", ARCH, "q") + self.assertTrue(self._python_ran()) + os.unlink(self.calls) + _write(os.path.join(self.work, ".bitsuse"), "[common]\n") + self._bits("-a", ARCH, "q") + self.assertTrue(self._python_ran()) + + def test_the_arch_published_for_this_host_without_python(self): + host = self._host() + shutil.rmtree(os.path.join(self.work, "sw", ARCH)) # nothing installed + cvmfs = os.path.join(self.tmp, "cvmfs", "community") + for arch in (host + "-opt", "x86_64-el1-gcc9-opt"): + _write(os.path.join(cvmfs, arch, "Modules", "modulefiles", "K", "1-1"), "#%Module1.0\n") + self.env["BITS_CVMFS_PREFIX"] = cvmfs + r = self._bits("q") + self.assertEqual((r.returncode, r.stdout.split()), (0, ["K/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + + def test_a_riscv64_install_without_python(self): + os.rename(os.path.join(self.work, "sw", ARCH), os.path.join(self.work, "sw", "ubuntu2604_riscv64")) + r = self._bits("q") + self.assertEqual((r.returncode, r.stdout.split()), (0, ["A/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + + def test_of_several_installed_the_hosts_without_python(self): + host = self._host() + "-opt" + for arch, pkg in ((host, "H"), ("x86_64-el1-gcc9-opt", "G")): + _write(os.path.join(self.work, "sw", arch, pkg, "1-1", "etc", "modulefiles", pkg), + "#%Module1.0\n") + os.makedirs(os.path.join(self.work, "sw", "MODULES", arch)) + shutil.rmtree(os.path.join(self.work, "sw", ARCH)) + r = self._bits("q") + self.assertEqual((r.returncode, r.stdout.split()), (0, ["H/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + + +class WithoutWorkDirTest(_Bits): + """A grid job: a community's CVMFS modules, no sw directory.""" + + def setUp(self): + super().setUp() + self.job = os.path.join(self.tmp, "job") + os.makedirs(self.job) + self.cvmfs = os.path.join(self.tmp, "cvmfs", "community") + self._publish(ARCH) + self.env["BITS_CVMFS_PREFIX"] = self.cvmfs + + def _publish(self, arch): + mods = os.path.join(self.cvmfs, arch, "Modules", "modulefiles") + _write(os.path.join(mods, "BASE", "1.0"), "#%Module1.0\n") + _write(os.path.join(mods, "K", "1-1"), + "#%Module1.0\nmodule load BASE/1.0\nsetenv K_ROOT $::env(BASEDIR)/K/1-1\n") + + def test_query_needs_no_work_dir(self): + r = self._bits("-a", ARCH, "q", cwd=self.job) + self.assertEqual((r.returncode, r.stdout.split()), (0, ["K/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + + def test_the_trees_of_a_cvmfs_yaml(self): + del self.env["BITS_CVMFS_PREFIX"] + _write(os.path.join(self.job, "cvmfs.yaml"), "prefix: %s\n" % self.cvmfs) + r = self._bits("-a", ARCH, "q", cwd=self.job) + self.assertEqual((r.returncode, r.stdout.split()), (0, ["K/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + + def test_the_hosts_arch(self): + host = self._host() + shutil.rmtree(os.path.join(self.cvmfs, ARCH)) + self._publish(host + "-opt") + self._publish("x86_64-el1-gcc9-opt") + r = self._bits("q", cwd=self.job) + self.assertEqual((r.returncode, r.stdout.split()), (0, ["K/1-1"]), r.stderr) + self.assertFalse(self._python_ran()) + # Two compilers for this host: bits cannot choose. + self._publish(host.replace("gcc14", "gcc13") + "-opt") + r = self._bits("q", cwd=self.job) + self.assertEqual(r.returncode, 1) + self.assertIn("Choose one: bits -a q", r.stderr) + self.assertIn(host + "-opt", r.stderr) + self.assertFalse(self._python_ran()) + + def test_other_commands_still_need_one(self): + def no_work_dir(*args): + r = self._bits(*args, cwd=self.job) + self.assertNotEqual(r.returncode, 0, args) + self.assertIn("no work dir", r.stdout + r.stderr, args) + self.env["BITS_CVMFS_PREFIX"] = "" # off + no_work_dir("-a", ARCH, "q") + del self.env["BITS_CVMFS_PREFIX"] + no_work_dir("-a", ARCH, "q") + # A cvmfs.yaml that gives no module trees (not below the prefix). + _write(os.path.join(self.job, "cvmfs.yaml"), + "prefix: %s\ncvmfs_modules_template: \"{install_dir}/etc/modulefiles/{pkg}\"\n" % self.cvmfs) + no_work_dir("-a", ARCH, "q") + os.unlink(os.path.join(self.job, "cvmfs.yaml")) + self.env["BITS_CVMFS_PREFIX"] = self.cvmfs + r = self._bits("-a", ARCH, "--dev", "load", "K/1-1", cwd=self.job) + self.assertEqual(r.returncode, 1) + self.assertIn("--dev needs a work dir", r.stderr) + + def test_printenv_with_modulecmd(self): + self._real_modulecmd() + for action in ("printenv", "load"): + r = self._bits("-a", ARCH, action, "K/1-1", cwd=self.job) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn(os.path.join(self.cvmfs, ARCH, "Packages", "K", "1-1"), r.stdout) + self.assertFalse(self._python_ran()) + + def test_setenv_with_modulecmd(self): + self._real_modulecmd() + r = self._bits("-a", ARCH, "setenv", "K/1-1", "-c", "sh", "-c", 'echo "$K_ROOT"', + cwd=self.job) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.strip(), os.path.join(self.cvmfs, ARCH, "Packages", "K", "1-1")) + self.assertFalse(self._python_ran()) + + +class CommunityOptionTest(_Bits): + """bits --community NAME … is $BITS_COMMUNITY=NAME.""" + + def setUp(self): + super().setUp() + _write(os.path.join(self.tmp, "stubs", "python3"), + '#!/bin/sh\necho "${BITS_COMMUNITY-unset} $*" >> "%s"\n' % self.calls, 0o755) + + def _seen(self): + with open(self.calls) as f: + return f.read().split() + + def test_the_option_sets_the_variable(self): + for opt in (["--community", "key4hep"], ["--community=key4hep"]): + self.env["BITS_COMMUNITY"] = "LHCB" + r = self._bits(*opt, "build", "K") + self.assertEqual(r.returncode, 0, r.stderr) + seen = self._seen() + self.assertEqual(seen[0], "KEY4HEP") + self.assertEqual(seen[-2:], ["build", "K"]) # and is not passed on + os.unlink(self.calls) + + def test_after_the_sub_command_it_is_the_sub_commands(self): + self._bits("init", "--community", "lhcb") + self.assertEqual(self._seen()[0], "unset") + self.assertEqual(self._seen()[-3:], ["init", "--community", "lhcb"]) + + def test_a_bad_name_is_an_error(self): + for opt in (["--community"], ["--community", ""], ["--community=a/b"], + ["--community", "-a", "q"]): + r = self._bits(*opt) + self.assertEqual(r.returncode, 1, opt) + self.assertIn("--community needs a community name", r.stderr) + self.assertFalse(self._python_ran()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 7c428f9c..72baa436 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -7,12 +7,14 @@ they run on any platform without requiring podman to be installed. """ +from pathlib import Path import os import sys import types import unittest from unittest.mock import patch, mock_open, MagicMock +from bits_helpers import sandbox from bits_helpers.sandbox import ( detect_dind, podman_available, @@ -170,8 +172,9 @@ def test_explicit_podman_docker_missing_in_image_raises(self, _p): self.assertIn("builder image", str(ctx.exception)) @patch("sys.platform", "darwin") + @patch("bits_helpers.sandbox.sandbox_exec_refusal", return_value=None) @patch("bits_helpers.sandbox.sandbox_exec_available", return_value=True) - def test_explicit_sandbox_exec_macos(self, _s): + def test_explicit_sandbox_exec_macos(self, _s, _r): self.assertEqual("sandbox-exec", resolve_sandbox_mode("sandbox-exec", False)) @patch("sys.platform", "linux") @@ -195,8 +198,9 @@ def test_auto_linux_no_docker_off_without_probe(self, _p): # auto, no docker, macOS @patch("sys.platform", "darwin") + @patch("bits_helpers.sandbox.sandbox_exec_refusal", return_value=None) @patch("bits_helpers.sandbox.sandbox_exec_available", return_value=True) - def test_auto_macos_sandbox_exec(self, _s): + def test_auto_macos_sandbox_exec(self, _s, _r): self.assertEqual("sandbox-exec", resolve_sandbox_mode("auto", False)) @patch("sys.platform", "darwin") @@ -391,7 +395,8 @@ def test_profile_receives_allow_network_false_by_default(self, mock_profile, _r) workdir="/sw", ) # make_sbpl_profile should have been called with allow_network=False - mock_profile.assert_called_once_with(False, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (False, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -400,7 +405,8 @@ def test_profile_receives_allow_network_true_when_off(self, mock_profile, _r): LOCAL_CMD, _spec(sandbox_network="off"), _opts(sandbox="sandbox-exec"), workdir="/sw", ) - mock_profile.assert_called_once_with(True, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (True, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -411,7 +417,8 @@ def test_yaml_boolean_off_enables_network(self, mock_profile, _r): LOCAL_CMD, _spec(sandbox_network=False), _opts(sandbox="sandbox-exec"), workdir="/sw", ) - mock_profile.assert_called_once_with(True, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (True, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -421,7 +428,8 @@ def test_yaml_boolean_on_blocks_network(self, mock_profile, _r): LOCAL_CMD, _spec(sandbox_network=True), _opts(sandbox="sandbox-exec"), workdir="/sw", ) - mock_profile.assert_called_once_with(False, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (False, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -432,7 +440,8 @@ def test_recipe_system_block_is_honoured(self, mock_profile, _r): LOCAL_CMD, {"package": "P", "system": {"sandbox_network": "off"}}, _opts(sandbox="sandbox-exec", sandbox_network="on"), workdir="/sw", ) - mock_profile.assert_called_once_with(True, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (True, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -442,7 +451,8 @@ def test_top_level_field_beats_system_block(self, mock_profile, _r): "system": {"sandbox_network": "off"}}, _opts(sandbox="sandbox-exec", sandbox_network="off"), workdir="/sw", ) - mock_profile.assert_called_once_with(False, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (False, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -452,7 +462,8 @@ def test_global_default_off_allows_when_recipe_silent(self, mock_profile, _r): LOCAL_CMD, _spec(), _opts(sandbox="sandbox-exec", sandbox_network="off"), workdir="/sw", ) - mock_profile.assert_called_once_with(True, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (True, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -463,7 +474,8 @@ def test_recipe_field_overrides_global_default(self, mock_profile, _r): _opts(sandbox="sandbox-exec", sandbox_network="off"), workdir="/sw", ) - mock_profile.assert_called_once_with(False, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (False, "/sw")) @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") @patch("bits_helpers.sandbox.make_sbpl_profile", return_value="/tmp/p.sb") @@ -472,7 +484,8 @@ def test_global_default_on_blocks_when_recipe_silent(self, mock_profile, _r): LOCAL_CMD, _spec(), _opts(sandbox="sandbox-exec", sandbox_network="on"), workdir="/sw", ) - mock_profile.assert_called_once_with(False, "/sw") + mock_profile.assert_called_once() + self.assertEqual(mock_profile.call_args[0], (False, "/sw")) # --------------------------------------------------------------------------- @@ -538,5 +551,130 @@ def test_standard_char_devices_writable(self): os.unlink(path) +# --------------------------------------------------------------------------- +# The macOS sandbox: one package's own files, nesting, the default +# --------------------------------------------------------------------------- + +ENV = {"PKGHASH": "ab12cd", "PKGNAME": "zlib", "EFFECTIVE_ARCHITECTURE": "osx_arm64", + "PKGVERSION": "v1.3", "PKGREVISION": "2", "COMMIT_HASH": "51b7f2a", + "PKGFAMILY": "", "DEVEL_PREFIX": "", "BUILD_FAMILY": ""} + + +class PackageWritePathsTests(unittest.TestCase): + + def test_only_the_packages_own_files(self): + with patch.dict(os.environ, {"CCACHE_DIR": "/cc"}): + sub, lit, dirs = sandbox.package_write_paths(ENV, "/sw", "/bw", ["/extra"]) + self.assertEqual(sub[:7], [ + "/bw/BUILD/ab12cd", "/sw/INSTALLROOT/ab12cd", "/sw/TMP/ab12cd", + "/sw/TARS/osx_arm64/store/ab/ab12cd", "/sw/TARS/osx_arm64/zlib", + "/sw/osx_arm64/zlib/v1.3-2", "/sw/SOURCES/zlib/v1.3/51b7f2a"]) + self.assertIn("/cc", sub) + self.assertIn("/extra", sub) + self.assertIn("/sw/MODULES", sub) + self.assertEqual(lit, ["/bw/BUILD/zlib-latest", "/sw/osx_arm64/zlib/latest"]) + for d in ("/sw", "/bw", "/sw/TARS/osx_arm64/store/ab", "/sw/osx_arm64/zlib", + "/sw/SOURCES/zlib/v1.3", "/sw/SPECS"): + self.assertIn(d, dirs) + self.assertNotIn("/sw/osx_arm64", sub) # not every package's tree + + def test_family_devel_prefix_and_build_family(self): + env = dict(ENV, PKGFAMILY="externals", PKGREVISION="", DEVEL_PREFIX="dev", + BUILD_FAMILY="gcc") + sub, lit, _dirs = sandbox.package_write_paths(env, "/sw", "/sw") + self.assertIn("/sw/osx_arm64/externals/zlib/v1.3", sub) + self.assertIn("/sw/BUILD/zlib-latest-dev", lit) + self.assertIn("/sw/osx_arm64/externals/zlib/latest-gcc", lit) + + def test_profile_kept_beside_build_sh_and_scoped(self): + import tempfile + d = tempfile.mkdtemp() + write = sandbox.package_write_paths(ENV, "/sw", "/sw") + path = make_sbpl_profile(False, "/sw", write=write, path=os.path.join(d, "sandbox.sb"), + replay="sandbox-exec -f sandbox.sb /bin/bash -e -x build.sh") + content = Path(path).read_text() + self.assertTrue(content.startswith("; Written by bits")) + self.assertIn('(allow file-write* (subpath "/sw/INSTALLROOT/ab12cd"))', content) + self.assertNotIn('(subpath "/sw")', content) + self.assertIn('file-write-create', content) + self.assertIn('(subpath "/private/tmp")', content) + + def test_a_latest_link_is_not_resolved_to_its_target(self): + import tempfile + d = os.path.realpath(tempfile.mkdtemp()) + os.makedirs(os.path.join(d, "v1")) + os.symlink("v1", os.path.join(d, "latest")) + self.assertEqual(sandbox._both(os.path.join(d, "latest"), literal=True), + [os.path.join(d, "latest")]) + self.assertEqual(sandbox._both(os.path.join(d, "latest")), + [os.path.join(d, "latest"), os.path.join(d, "v1")]) + + def test_a_quote_in_a_path_is_refused(self): + with self.assertRaises(ValueError): + make_sbpl_profile(False, "/sw", write=(['/a"b'], [], [])) + + @patch("bits_helpers.sandbox.resolve_sandbox_mode", return_value="sandbox-exec") + def test_wrap_writes_the_scoped_profile(self, _r): + import tempfile + d = tempfile.mkdtemp() + opts = _opts(sandbox="sandbox-exec") + opts.sandboxAllowWrite = ["/extra"] + cmd = wrap_build_command(LOCAL_CMD, _spec(), opts, workdir="/sw", pkg_env=ENV, + build_workdir="/sw", script_dir=d) + self.assertIn(os.path.join(d, "sandbox.sb"), cmd) + self.assertIn('(subpath "/extra")', Path(os.path.join(d, "sandbox.sb")).read_text()) + self.assertIn("grep 'Operation not permitted'", sandbox.failure_hint(cmd)) + self.assertEqual(sandbox.failure_hint(LOCAL_CMD), "") + + +class NestingTests(unittest.TestCase): + + def setUp(self): + sandbox._REFUSAL.clear() + self.addCleanup(sandbox._REFUSAL.clear) + + @patch("bits_helpers.sandbox.subprocess.run", return_value=MagicMock(returncode=1)) + def test_already_in_a_sandbox(self, _run): + self.assertIn("do not nest", sandbox.sandbox_exec_refusal()) + + @patch("bits_helpers.sandbox.subprocess.run", return_value=MagicMock(returncode=0)) + def test_usable_and_tried_once(self, run): + self.assertIsNone(sandbox.sandbox_exec_refusal()) + self.assertIsNone(sandbox.sandbox_exec_refusal()) + run.assert_called_once() + + @patch("sys.platform", "darwin") + @patch("bits_helpers.sandbox.sandbox_exec_available", return_value=True) + @patch("bits_helpers.sandbox.sandbox_exec_refusal", return_value="nested") + def test_explicit_refused_auto_off(self, _r, _a): + with self.assertRaises(ValueError): + resolve_sandbox_mode("sandbox-exec", False) + self.assertEqual(resolve_sandbox_mode("auto", False), "off") + + +class DefaultModeTests(unittest.TestCase): + + @patch("sys.platform", "darwin") + @patch("bits_helpers.sandbox.sandbox_exec_available", return_value=True) + @patch("bits_helpers.sandbox.sandbox_exec_refusal", return_value=None) + def test_env_or_defaults_turn_it_on_for_a_local_mac_build(self, _r, _a): + self.assertEqual(sandbox.default_mode("1", None, False), "sandbox-exec") + self.assertEqual(sandbox.default_mode(None, True, False), "sandbox-exec") + self.assertEqual(sandbox.default_mode("", "on", False), "sandbox-exec") + self.assertEqual(sandbox.default_mode("0", True, False), "off") # env wins + self.assertEqual(sandbox.default_mode("1", None, True), "off") # --docker + self.assertEqual(sandbox.default_mode(None, None, False), "off") + + @patch("sys.platform", "darwin") + @patch("bits_helpers.sandbox.sandbox_exec_available", return_value=True) + @patch("bits_helpers.sandbox.sandbox_exec_refusal", return_value="nested") + def test_by_default_where_it_cannot_run_it_is_off(self, _r, _a): + self.assertEqual(sandbox.default_mode("1", None, False), "off") + + @patch("sys.platform", "linux") + def test_never_elsewhere(self): + self.assertEqual(sandbox.default_mode("1", True, False), "off") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 578bf8d6..0b222492 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -79,6 +79,55 @@ def test_purl_and_download(self): self.assertEqual(sbom._download(cmake), "https://cmake.org/files/cmake-3.30.6.tar.gz") + def test_a_system_package_with_a_version_is_identified(self): + m = _manifest() + m["system_package_info"] = {"zlib": {"version": "1.2.11", "references": [ + "cpe:2.3:a:zlib:zlib:1.2.11:*:*:*:*:*:*:*", "cpe:2.3:a:gnu:zlib:1.2.11:*:*:*:*:*:*:*"]}} + zlib = {c["name"]: c for c in sbom.to_cyclonedx(m)["components"]}["zlib"] + self.assertEqual((zlib["version"], zlib["purl"], zlib["cpe"]), + ("1.2.11", "pkg:generic/zlib@1.2.11", "cpe:2.3:a:zlib:zlib:1.2.11:*:*:*:*:*:*:*")) + self.assertIn({"name": "syft:cpe23", "value": "cpe:2.3:a:gnu:zlib:1.2.11:*:*:*:*:*:*:*"}, + zlib["properties"]) + spdx = {p["name"]: p for p in sbom.to_spdx(m)["packages"]}["zlib"] + self.assertEqual(spdx["versionInfo"], "1.2.11") + plain = {c["name"]: c for c in sbom.to_cyclonedx(_manifest())["components"]}["zlib"] + self.assertNotIn("version", plain) # its check said nothing + + def test_a_host_tool_is_marked(self): + m = _manifest() + m["system_package_info"] = {"zlib": {"version": "", "references": [], "sbom": "tool"}} + m["packages"][3]["sbom"] = "tool" + comps = {c["name"]: c for c in sbom.to_cyclonedx(m)["components"]} + for name in ("zlib", "ROOT"): + self.assertIn({"name": "bits:sbom", "value": "tool"}, comps[name]["properties"], name) + self.assertNotIn("bits:sbom", [p["name"] for p in comps["CMake"]["properties"]]) + + def test_a_requested_system_package_is_a_root(self): + m = _manifest() + m["requested_packages"] = ["kkmcee", "zlib"] + doc = sbom.to_cyclonedx(m) + root = doc["metadata"]["component"]["bom-ref"] + (deps,) = [d["dependsOn"] for d in doc["dependencies"] if d["ref"] == root] + self.assertIn("zlib@system", deps) + + def test_a_pinned_version_says_who_pinned_it(self): + m = _manifest() + m["packages"][3]["pinned_by"] = "defaults:lhcb" + root = {c["name"]: c for c in sbom.to_cyclonedx(m)["components"]}["ROOT"] + self.assertIn({"name": "bits:pinned_by", "value": "defaults:lhcb"}, root["properties"]) + + def test_a_system_package_found_by_the_package_manager_goes_by_its_name(self): + m = _manifest() + purl = "pkg:deb/ubuntu/zlib1g@1:1.3.dfsg-3.1?arch=amd64&distro=ubuntu-24.04&upstream=zlib" + m["system_package_info"] = {"zlib": {"version": "1:1.3.dfsg-3.1", "references": [purl]}} + (zlib,) = [c for c in sbom.to_cyclonedx(m)["components"] if c["bom-ref"].startswith("zlib@")] + self.assertEqual((zlib["name"], zlib["version"], zlib["purl"]), ("zlib1g", "1:1.3.dfsg-3.1", purl)) + self.assertIn({"name": "bits:package", "value": "zlib"}, zlib["properties"]) + self.assertNotIn("cpe", zlib) + spdx = {p["name"]: p for p in sbom.to_spdx(m)["packages"]}["zlib1g"] + self.assertEqual(spdx["externalRefs"], [{"referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", "referenceLocator": purl}]) + def test_missing_dependencies_that_are_not_system_are_dropped(self): m = _manifest() m["system_packages"] = [] # e.g. a failed build: zlib never recorded @@ -169,7 +218,8 @@ def test_document(self): self.assertEqual(bom["metadata"]["timestamp"], "2026-09-30T07:18:18Z") comps = {c["name"]: c for c in bom["components"]} root = comps["ROOT"] - self.assertEqual(root["version"], "v6-40-02-5") + self.assertEqual(root["version"], "v6-40-02") # upstream, as scanners compare + self.assertIn({"name": "bits:revision", "value": "5"}, root["properties"]) self.assertEqual(root["hashes"], [{"alg": "SHA-256", "content": "3" * 64}]) self.assertEqual(root["licenses"], [{"expression": "LGPL-2.1-or-later"}]) props = {(p["name"], p["value"]) for p in root["properties"]} @@ -194,6 +244,49 @@ def test_deterministic(self): json.loads(sbom.render(m, "cyclonedx", "rel-1"))["serialNumber"]) +def _with_references(): + m = _manifest() + m["packages"][3]["references"] = ["cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*", + "cpe:2.3:a:root-project:root:6.40.02:*:*:*:*:*:*:*"] + m["packages"][2]["references"] = ["pkg:pypi/cmake@3.30.6"] + m["packages"][4]["references"] = ["cpe:2.3:a:kkmcee:kkmcee:5.01.00\\+cpu:*:*:*:*:*:*:*"] + return m + + +class TestScannerIdentity(unittest.TestCase): + """A recipe's references (CPEs, purls) name the package for vulnerability + scanners; without them a package keeps its pkg:github / pkg:generic purl.""" + + def _refs(self, refs): + m = _manifest() + m["packages"][3]["references"] = refs # ROOT + return m + + def test_cyclonedx_carries_the_first_cpe_and_the_recipes_purl(self): + m = self._refs(["cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*", "pkg:generic/root@6.40.02", + "cpe:2.3:a:root-project:root:6.40.02:*:*:*:*:*:*:*"]) + root = {c["name"]: c for c in sbom.to_cyclonedx(m)["components"]}["ROOT"] + self.assertEqual(root["cpe"], "cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*") + self.assertEqual(root["purl"], "pkg:generic/root@6.40.02") + self.assertIn({"name": "syft:cpe23", "value": "cpe:2.3:a:root-project:root:6.40.02:*:*:*:*:*:*:*"}, + root["properties"]) + + def test_without_references_nothing_changes(self): + root = {c["name"]: c for c in sbom.to_cyclonedx(_manifest())["components"]}["ROOT"] + self.assertNotIn("cpe", root) + self.assertTrue(root["purl"].startswith("pkg:github/root-project/root@")) + + def test_spdx_lists_every_cpe_as_a_security_reference(self): + m = self._refs(["cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*", + "cpe:2.3:a:root-project:root:6.40.02:*:*:*:*:*:*:*"]) + pk = {p["name"]: p for p in sbom.to_spdx(m)["packages"]} + self.assertEqual([r["referenceLocator"] for r in pk["ROOT"]["externalRefs"] + if r["referenceType"] == "cpe23Type"], + ["cpe:2.3:a:cern:root:6.40.02:*:*:*:*:*:*:*", + "cpe:2.3:a:root-project:root:6.40.02:*:*:*:*:*:*:*"]) + self.assertEqual(pk["ROOT"]["versionInfo"], "v6-40-02") + + class TestSPDX(unittest.TestCase): def test_document(self): doc = sbom.to_spdx(_manifest(), "rel-1") @@ -272,7 +365,7 @@ def test_cyclonedx_schema(self): from cyclonedx.validation.json import JsonStrictValidator except ImportError: self.skipTest("cyclonedx-python-lib not installed") - for m in (_manifest(), _odd_manifest()): + for m in (_manifest(), _odd_manifest(), _with_references()): err = JsonStrictValidator(SchemaVersion.V1_6).validate_str( sbom.render(m, "cyclonedx", "rel-1")) self.assertIsNone(err, err) @@ -285,7 +378,7 @@ def test_spdx_validation(self): self.skipTest("spdx-tools not installed") d = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, d, True) path = os.path.join(d, "sbom.spdx.json") - for m in (_manifest(), _odd_manifest()): + for m in (_manifest(), _odd_manifest(), _with_references()): with open(path, "w") as fh: fh.write(sbom.render(m, "spdx", "rel-1")) self.assertEqual(validate_full_spdx_document(parse_file(path)), []) diff --git a/tests/test_service_signer.py b/tests/test_service_signer.py index 462345f7..d20eb9d8 100644 --- a/tests/test_service_signer.py +++ b/tests/test_service_signer.py @@ -4,6 +4,7 @@ the build's pre-approval), no key in CI. HTTP is mocked; the returned envelope is verified over our bytes against the trust anchor before it is written.""" +from pathlib import Path import io import json import os @@ -46,7 +47,7 @@ def fake(req, timeout=None, context=None): s = certify._ServiceSigner("https://bits.cern.ch", "42", "tok") out = s.sign_manifest(self.mf, self.sig) self.assertEqual(out, self.sig) - self.assertEqual(json.load(open(self.sig)), env) + self.assertEqual(json.loads(Path(self.sig).read_text()), env) self.assertIn("/sign/preapproved?build_id=42", calls[0]) def test_sign_passes_certifier(self): diff --git a/tests/test_sign_console.py b/tests/test_sign_console.py index f803ff51..e2c72e19 100644 --- a/tests/test_sign_console.py +++ b/tests/test_sign_console.py @@ -2,7 +2,9 @@ # SPDX-License-Identifier: GPL-3.0-or-later """Tests for the cross-device console signing client (bits_helpers.sign_console).""" +from pathlib import Path import hashlib +import io import json import os import shutil @@ -19,7 +21,7 @@ def setUp(self): self.mp = os.path.join(self.dir, "m.json") with open(self.mp, "wb") as fh: fh.write(b'{"packages":[{"package":"A","group":"lcg"}]}') - self.digest = hashlib.sha256(open(self.mp, "rb").read()).hexdigest() + self.digest = hashlib.sha256(Path(self.mp).read_bytes()).hexdigest() def tearDown(self): shutil.rmtree(self.dir, ignore_errors=True) @@ -39,7 +41,7 @@ def test_flow_verifies_and_writes_sig(self): patch.object(sign_console.trust, "verify_bytes", lambda b, e, t: "kid1234"), \ patch.object(sign_console.time, "sleep", lambda s: None): out = sign_console.sign_via_console("http://x", self.mp, timeout=10) - self.assertEqual(json.load(open(out))["key_id"], "k") + self.assertEqual(json.loads(Path(out).read_text())["key_id"], "k") def test_builds_approve_url_from_console(self): captured = {} @@ -138,5 +140,15 @@ def test_refuses_plain_http_console(self): sign_console._check_console_url("http://localhost:8080", False) # local is fine sign_console._check_console_url("http://tb:8080", True) # explicit override + +class TestPrintQr(unittest.TestCase): + def test_says_why_there_is_no_qr_code(self): + with patch.dict("sys.modules", {"qrcode": None}), \ + patch("sys.stdout", new_callable=io.StringIO) as out: + sign_console._print_qr("https://bits.example/approve?approve=r1") + self.assertIn("module qrcode is not installed", out.getvalue()) + self.assertIn("Approve on your phone: https://bits.example/approve?approve=r1", out.getvalue()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_site.py b/tests/test_site.py new file mode 100644 index 00000000..08c82417 --- /dev/null +++ b/tests/test_site.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""The site configuration: NN-.yaml files, merged in name order.""" + +import os +import shutil +import tempfile +import unittest + +from bits_helpers import site + + +class SiteTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.a, self.b = os.path.join(self.tmp, "a"), os.path.join(self.tmp, "b") + os.makedirs(self.a) + os.makedirs(self.b) + saved = site.SEARCH + site.SEARCH = [self.a, self.b] + site.reset() + + def restore(): + site.SEARCH = saved + site.reset() + self.addCleanup(restore) + + def write(self, d, name, text): + with open(os.path.join(d, name), "w") as f: + f.write(text) + site.reset() + + def test_name_order_last_wins_and_mappings_merge(self): + self.write(self.a, "00-bits.yaml", "containers: {registry: r0, tag: latest}\nstore: {read: s0}\n") + self.write(self.b, "10-site.yaml", "containers: {registry: r1}\n") + self.write(self.a, "20-host.yaml", "store: {read: null}\n") + self.write(self.a, "README.md", "not read") + self.write(self.a, "1-bad-name.yaml", "store: {read: nope}\n") + self.assertEqual(site.get("containers.registry"), "r1") + self.assertEqual(site.get("containers.tag"), "latest") + self.assertIsNone(site.get("store.read")) + self.assertEqual(site.get("store.read", "x"), "x") + self.assertEqual(site.origin("containers.registry"), os.path.join(self.b, "10-site.yaml")) + self.assertEqual([os.path.basename(f) for f in site.files()], + ["00-bits.yaml", "10-site.yaml", "20-host.yaml"]) + + def test_same_name_in_a_later_directory_replaces_and_empty_masks(self): + self.write(self.a, "10-CERN.yaml", "store: {operations: cern}\n") + self.write(self.b, "10-CERN.yaml", "") + self.assertIsNone(site.get("store.operations")) + self.assertEqual(site.files(), [os.path.join(self.b, "10-CERN.yaml")]) + + def test_a_broken_file_stops_bits_with_its_name(self): + self.write(self.a, "00-x.yaml", "store: [unclosed\n") + with self.assertRaises(SystemExit) as cm: + site.get("store.read") + self.assertIn("00-x.yaml", str(cm.exception)) + self.write(self.a, "00-x.yaml", "- a list\n") + with self.assertRaises(SystemExit): + site.get("store.read") + + def test_read_store_by_architecture_then_default(self): + self.write(self.a, "00-x.yaml", "store:\n read: plain\n read_by_arch: {slc9_x86-64: alice}\n") + self.assertEqual(site.read_store("slc9_x86-64"), "alice") + self.assertEqual(site.read_store("x86_64-el9"), "plain") + + +class ShippedTest(unittest.TestCase): + def test_shipped_files_keep_the_previous_built_in_values(self): + saved = site.SEARCH + site.SEARCH = [site.SHIPPED] + site.reset() + self.addCleanup(setattr, site, "SEARCH", saved) + self.addCleanup(site.reset) + self.assertEqual([os.path.basename(f) for f in site.files()], ["00-bits.yaml", "10-CERN.yaml"]) + self.assertEqual(site.get("containers.registry"), "gitlab-registry.cern.ch/bits/containers") + self.assertEqual(site.get("containers.tag"), "latest") + self.assertEqual(site.get("store.operations"), "https://s3.cern.ch/lcgapp-bits-testing") + self.assertEqual(site.read_store("slc9_x86-64"), "https://s3.cern.ch/swift/v1/alibuild-repo") + self.assertEqual(site.read_store("x86_64-el9"), "https://s3.cern.ch/lcgapp-bits-testing") + self.assertEqual(site.get("providers.url"), "https://github.com/bitsorg/bits-providers") + self.assertEqual(site.get("sbom.namespace"), "https://bits.cern.ch/sbom") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_stats.py b/tests/test_stats.py index 717c8754..d7a61025 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -3,6 +3,7 @@ """Tests for the `bits stats` resource report (bits_helpers/stats.py).""" +from pathlib import Path import json import os import sys @@ -42,7 +43,7 @@ def tearDown(self): def _trace(self, samples): p = os.path.join(self.d, "t.json") - json.dump(samples, open(p, "w")) + Path(p).write_text(json.dumps(samples)) return p def test_peaks_and_average(self): @@ -65,7 +66,7 @@ def test_mem_per_thread(self): def test_empty_or_bad(self): self.assertIsNone(stats.trace_metrics(self._trace([]))) - bad = os.path.join(self.d, "bad.json"); open(bad, "w").write("not json") + bad = os.path.join(self.d, "bad.json"); Path(bad).write_text("not json") self.assertIsNone(stats.trace_metrics(bad)) @@ -106,18 +107,17 @@ def test_no_flags_when_modest(self): class CollectAndRenderTest(unittest.TestCase): def setUp(self): self.d = tempfile.mkdtemp() - json.dump({ + Path(os.path.join(self.d, "bits_build_stats.json")).write_text(json.dumps({ "resources": {"cpu": 800, "rss": 16 * GB}, "packages": {"build": { "ROOT": {"cpu": 760, "rss": int(6.2 * GB), "time": 1850}, "Gaudi": {"cpu": 110, "rss": GB, "time": 420}, }}, - }, open(os.path.join(self.d, "bits_build_stats.json"), "w")) + })) sd = os.path.join(self.d, "SPECS", "arch", "Gaudi", "v-1") os.makedirs(sd) - json.dump([{"time": t, "cpu": 100, "rss": GB, "num_threads": 3} - for t in range(0, 420, 5)], - open(os.path.join(sd, "Gaudi.json"), "w")) + samples = [{"time": t, "cpu": 100, "rss": GB, "num_threads": 3} for t in range(0, 420, 5)] + Path(os.path.join(sd, "Gaudi.json")).write_text(json.dumps(samples)) def tearDown(self): shutil.rmtree(self.d, ignore_errors=True) diff --git a/tests/test_status.py b/tests/test_status.py index 579a99e6..168e73ba 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -6,10 +6,12 @@ import json import os import re +import shutil import tempfile import unittest from collections import OrderedDict from io import StringIO +from types import SimpleNamespace from unittest.mock import MagicMock, patch from bits_helpers.status import ( @@ -20,6 +22,7 @@ HASH_UNKNOWN, LOCAL_CHECKOUT, LOCAL_CHECKOUT_UNCHANGED, + SYMLINK_FROM_DEPLOYMENT, _classify, _emit_json, _emit_table, @@ -29,6 +32,7 @@ ) from bits_helpers.hashing import storeHashes from bits_helpers.build import add_initdotsh_hash_marker, prepare_hash_inputs +from bits_helpers.build_config import BuildConfig # ── Helpers ──────────────────────────────────────────────────────────────────── @@ -98,6 +102,65 @@ def _write_tars_symlink(work_dir: str, arch: str, pkg: str, version: str, os.symlink(rel_target, symlink_path) +# ── _try_populate_refs ───────────────────────────────────────────────────────── + +class TestStatusSymlinkSelection(unittest.TestCase): + def test_selection_matches_build_policy_and_reports_source(self): + with tempfile.TemporaryDirectory() as tmpdir: + deployment = os.path.join(tmpdir, "deployment") + work = os.path.join(tmpdir, "work") + source = os.path.join(deployment, "arch", "external", "mylib", "1.0-1") + os.makedirs(os.path.join(source, "etc", "profile.d")) + with open(os.path.join(source, "etc", "profile.d", "init.sh"), "w") as fh: + fh.write("# deployed init\n") + with open(os.path.join(source, ".build-hash"), "w") as fh: + fh.write("aabbcc" + "0" * 34 + "\n") + spec = _make_spec() + spec["pkg_family"] = "external" + cfg = BuildConfig.from_args(SimpleNamespace( + symlinkPackagesFrom=deployment, symlinkPolicy="strict")) + self.assertEqual(_classify(spec, work, "arch", symlink_cfg=cfg), + SYMLINK_FROM_DEPLOYMENT) + self.assertEqual(spec["symlink_source"], source) + with patch("sys.stdout", new_callable=StringIO) as stdout: + _emit_json([dict(package="mylib", version="1.0", hash=spec["hash"], + state=SYMLINK_FROM_DEPLOYMENT, source=source)], "arch") + self.assertEqual(json.loads(stdout.getvalue())["packages"][0]["source"], source) + + local = os.path.join(work, "arch", "external", "mylib", "1.0-1") + os.makedirs(os.path.join(local, "etc", "profile.d")) + with open(os.path.join(local, "etc", "profile.d", "init.sh"), "w") as fh: + fh.write("# installed init\n") + with open(os.path.join(local, ".build-hash"), "w") as fh: + fh.write("aabbcc" + "0" * 34 + "\n") + installed_spec = _make_spec() + installed_spec["pkg_family"] = "external" + self.assertEqual(_classify(installed_spec, work, "arch", symlink_cfg=cfg), + ALREADY_INSTALLED) + shutil.rmtree(local) + + with open(os.path.join(source, ".build-hash"), "w") as fh: + fh.write("different\n") + mismatched = _make_spec() + mismatched["pkg_family"] = "external" + self.assertEqual(_classify(mismatched, work, "arch", symlink_cfg=cfg), + BUILD_FROM_SOURCE) + relaxed = BuildConfig.from_args(SimpleNamespace( + symlinkPackagesFrom=deployment, symlinkPolicy="relaxed")) + relaxed_spec = _make_spec() + relaxed_spec["pkg_family"] = "external" + self.assertEqual(_classify(relaxed_spec, work, "arch", symlink_cfg=relaxed), + SYMLINK_FROM_DEPLOYMENT) + self.assertEqual(relaxed_spec["hash"], "different") + excluded = BuildConfig.from_args(SimpleNamespace( + symlinkPackagesFrom=deployment, symlinkPolicy="relaxed", + buildLocal="mylib")) + excluded_spec = _make_spec() + excluded_spec["pkg_family"] = "external" + self.assertEqual(_classify(excluded_spec, work, "arch", symlink_cfg=excluded), + BUILD_FROM_SOURCE) + + # ── _try_populate_refs ───────────────────────────────────────────────────────── class TestTryPopulateRefs(unittest.TestCase): @@ -946,6 +1009,113 @@ def fake_store_hashes(p, specs, considerRelocation): data = json.loads(mock_out.getvalue()) self.assertEqual(data["packages"][0]["state"], ALREADY_INSTALLED) + @patch("bits_helpers.status.getPackageList") + @patch("bits_helpers.status.parseDefaults") + @patch("bits_helpers.status.readDefaults") + @patch("bits_helpers.hashing.storeHashes") + @patch("bits_helpers.build.storeHook") + def test_docker_checks_and_fingerprint_as_in_build(self, mock_hook, mock_store_hashes, + mock_read_defaults, mock_parse_defaults, + mock_get_package_list): + """--docker: system checks run in the container; own_hash folds its fingerprint.""" + rh = "abcdef" + "0" * 34 + mock_parse_defaults.return_value = (None, {}, {}, {}) + mock_read_defaults.return_value = None + spec = _make_spec(pkg="GCC-Toolchain", version="v15", remote_revision_hash=rh, + remote_hashes=[rh]) + spec["own_hash"] = True + ran, seen = [], {} + + class FakeRunner: + def __init__(self, image, *a, **kw): + seen["image"] = image + def __enter__(self): + def run(cmd, cwd=None): + ran.append(cmd) + return (0, "fp123\n") if "container-fingerprint" in cmd else (1, "") + return run + def __exit__(self, *exc): + return False + + def fake_get_pkg_list(*args, **kwargs): + seen["noSystem"] = kwargs["noSystem"] + seen["check"] = kwargs["performPreferCheck"](spec, "rpm -q zlib") + kwargs["specs"]["GCC-Toolchain"] = spec + return ([], ["GCC-Toolchain"], set(), None) + mock_get_package_list.side_effect = fake_get_pkg_list + + def fake_store_hashes(p, specs, considerRelocation): + seen.update(fp=specs[p].get("container_fingerprint"), + arch=specs[p].get("_own_hash_arch")) + specs[p].update(remote_revision_hash=rh, local_revision_hash="l" + rh, + remote_hashes=[rh], local_hashes=["l" + rh], deps_hash="") + mock_store_hashes.side_effect = fake_store_hashes + + import bits_helpers.status as status_mod + args = self._make_args(["GCC-Toolchain"], json_output=True) + args.docker, args.dockerImage, args.docker_extra_args = True, "img:1", [] + with patch("sys.stdout", new_callable=StringIO), \ + patch("bits_helpers.status.DockerRunner", FakeRunner), \ + patch("bits_helpers.status.topological_sort", return_value=["GCC-Toolchain"]), \ + patch("bits_helpers.status.compute_combined_arch", return_value=self.arch), \ + patch("bits_helpers.status.compute_own_hash_arch", return_value="neutral-arch"), \ + patch("bits_helpers.status.prunePaths"): + import argparse + parser = argparse.ArgumentParser() + parser.error = lambda msg: (_ for _ in ()).throw(SystemExit(msg)) + status_mod.doStatus(args, parser) + self.assertEqual(seen["image"], "img:1") + self.assertIsNone(seen["noSystem"]) + self.assertEqual(seen["check"], (1, "")) + self.assertIn("rpm -q zlib", ran) + self.assertEqual((seen["fp"], seen["arch"]), ("fp123", "neutral-arch")) + + @patch("bits_helpers.status.getPackageList") + @patch("bits_helpers.status.parseDefaults") + @patch("bits_helpers.status.readDefaults") + @patch("bits_helpers.hashing.storeHashes") + @patch("bits_helpers.build.storeHook") + def test_tarball_sources_win_over_git_as_in_build(self, mock_hook, mock_store_hashes, + mock_read_defaults, mock_parse_defaults, + mock_get_package_list): + """source: and sources: → tar mode, as bits build: no git source, no tag.""" + rh = "abcdef" + "0" * 34 + mock_parse_defaults.return_value = (None, {}, {}, {}) + mock_read_defaults.return_value = None + spec = _make_spec(pkg="syscalc", version="1.1.7", tag="v1.1.7", + source="https://github.com/syscalc/syscalc", + remote_revision_hash=rh, remote_hashes=[rh]) + spec["sources"] = ["https://example.org/sysCalc-1.1.7.tar.gz"] + + def fake_get_pkg_list(*args, **kwargs): + kwargs["specs"]["syscalc"] = spec + return ([], ["syscalc"], set(), None) + mock_get_package_list.side_effect = fake_get_pkg_list + seen = {} + + def fake_store_hashes(p, specs, considerRelocation): + seen.update(source="source" in specs[p], tag=specs[p].get("tag")) + specs[p].update(remote_revision_hash=rh, local_revision_hash="l" + rh, + remote_hashes=[rh], local_hashes=["l" + rh], deps_hash="") + mock_store_hashes.side_effect = fake_store_hashes + + import bits_helpers.status as status_mod + args = self._make_args(["syscalc"], json_output=True) + with patch("sys.stdout", new_callable=StringIO), \ + patch("bits_helpers.status.topological_sort", return_value=["syscalc"]), \ + patch("bits_helpers.status.compute_combined_arch", return_value=self.arch), \ + patch("bits_helpers.status.prunePaths"), \ + patch("bits_helpers.status._fetch_refs_with_clone") as clone, \ + patch("bits_helpers.status._try_populate_refs") as refs: + import argparse + parser = argparse.ArgumentParser() + parser.error = lambda msg: (_ for _ in ()).throw(SystemExit(msg)) + status_mod.doStatus(args, parser) + self.assertEqual(seen["source"], False) + self.assertNotEqual(seen["tag"], "v1.1.7") + clone.assert_not_called() + refs.assert_not_called() + @patch("bits_helpers.status.getPackageList") @patch("bits_helpers.status.parseDefaults") @patch("bits_helpers.status.readDefaults") @@ -1026,5 +1196,25 @@ def fake_get_pkg_list(*args, **kwargs): self.assertEqual(data["packages"][0]["state"], HASH_UNKNOWN) + +class TestDefaultsVariablesAsBuild(unittest.TestCase): + """status resolves the defaults' variables with build's code: same hashes.""" + + def test_set_and_release(self): + from types import SimpleNamespace + from bits_helpers.build import resolve_defaults_variables + args = SimpleNamespace(flavours={"release": "LCG_110", "foo": "1"}, + architecture="slc9_x86-64", defaults=["release"]) + meta = {"variables": {"release": "main"}, "env": None} + resolve_defaults_variables(meta, args, "") + self.assertEqual(meta["variables"]["release"], "LCG_110") + self.assertEqual(dict(meta["env"]), {"release": "LCG_110", "foo": "1"}) + # Without --set, the recipe branch names the release. + meta = {"variables": {"release": "main"}} + resolve_defaults_variables(meta, SimpleNamespace( + flavours={}, architecture="slc9_x86-64", defaults=["release"]), "LCG_110-patches") + self.assertEqual(meta["variables"]["release"], "LCG_110") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_store_cache.py b/tests/test_store_cache.py new file mode 100644 index 00000000..0aaee877 --- /dev/null +++ b/tests/test_store_cache.py @@ -0,0 +1,180 @@ +"""Tests for the plan's cache of store listings (bits_helpers/store_cache.py).""" +import json +import os +import tempfile +import threading +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from bits_helpers import plan, store_cache + +ARCH = "x86_64-el9-gcc14-opt" + + +class StoreCacheTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def test_found_tarballs_are_kept_between_runs(self): + c = store_cache.StoreCache(self.tmp.name, "b3://store") + c.put(ARCH, "h1", ["z-1-1.tar.gz"]) + c.put(ARCH, "h2", []) # nothing found: not kept + c.save() + again = store_cache.StoreCache(self.tmp.name, "b3://store") + self.assertEqual((again.get(ARCH, "h1"), again.get(ARCH, "h2")), (["z-1-1.tar.gz"], None)) + self.assertIsNone(store_cache.StoreCache(self.tmp.name, "b3://other").get(ARCH, "h1")) + self.assertTrue(again.path.startswith(os.path.join(self.tmp.name, ".store-cache") + os.sep)) + + def test_two_runs_merge_and_old_entries_go(self): + a = store_cache.StoreCache(self.tmp.name, "s") + b = store_cache.StoreCache(self.tmp.name, "s") + a.put(ARCH, "h1", ["a"]) + b.put(ARCH, "h2", ["b"]) + a.save() + b.save() # merged with a's + c = store_cache.StoreCache(self.tmp.name, "s") + self.assertEqual((c.get(ARCH, "h1"), c.get(ARCH, "h2")), (["a"], ["b"])) + with patch.object(store_cache, "_today", lambda: 10 ** 6): + c.put(ARCH, "h3", ["c"]) + c.entries["%s/h2" % ARCH]["used"] = 10 ** 6 + c.save() + with open(c.path) as f: + self.assertEqual(sorted(json.load(f)["entries"]), ["%s/h2" % ARCH, "%s/h3" % ARCH]) + + def test_a_bad_or_unwritable_file_is_only_a_miss(self): + c = store_cache.StoreCache(self.tmp.name, "s") + os.makedirs(os.path.dirname(c.path)) + with open(c.path, "w") as f: + f.write("{not json") + self.assertIsNone(store_cache.StoreCache(self.tmp.name, "s").get(ARCH, "h")) + nowhere = store_cache.StoreCache(os.path.join(self.tmp.name, "file"), "s") + open(os.path.join(self.tmp.name, "file"), "w").close() # a file, not a directory + nowhere.put(ARCH, "h", ["x"]) + nowhere.save() # logged, not raised + self.assertIsNone(store_cache.StoreCache("", "s").path) + + def test_a_tarball_not_there_after_all_is_forgotten(self): + c = store_cache.StoreCache(self.tmp.name, "s") + c.put(ARCH, "h1", ["a"]) + c.save() + d = store_cache.StoreCache(self.tmp.name, "s") + d.forget(ARCH, ["h1"]) + d.save() # not merged back + self.assertIsNone(store_cache.StoreCache(self.tmp.name, "s").get(ARCH, "h1")) + + def test_store_id(self): + multi = SimpleNamespace(readers=[SimpleNamespace(remoteStore="a"), SimpleNamespace(remoteStore="b")]) + self.assertEqual(store_cache.store_id(multi), "a|b") + self.assertEqual(store_cache.store_id(SimpleNamespace()), "") + + +class KeptListingsTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.kept = store_cache.StoreCache(tmp.name, "s") + self.calls = [] + + def helper(self): + calls = self.calls + + class Helper: + def list_store_tarballs(self, arch, h): + calls.append(h) + return ["%s.tar.gz" % h] if h.startswith("in") else [] + return Helper() + + def test_a_certified_tarball_is_not_asked_for_again(self): + trusted = {(ARCH, "in1"): "sha"} + first = plan._Listings(self.helper(), self.kept, trusted) + self.assertEqual(first.prefetch([(ARCH, "in1"), (ARCH, "in2"), (ARCH, "out")]), 3) + self.assertEqual(sorted(self.kept.entries), ["%s/in1" % ARCH]) # certified only + again = plan._Listings(self.helper(), self.kept, trusted) + self.assertEqual(again.prefetch([(ARCH, "in1"), (ARCH, "in2"), (ARCH, "out")]), 2) + self.assertEqual(again.list_store_tarballs(ARCH, "in1"), ["in1.tar.gz"]) + self.assertEqual(self.calls.count("in1"), 1) + decertified = plan._Listings(self.helper(), self.kept, {}) # no longer signed + decertified.list_store_tarballs(ARCH, "in1") + self.assertEqual(self.calls.count("in1"), 2) + unsigned_reuse = plan._Listings(self.helper(), self.kept, None) # signed reuse off + unsigned_reuse.prefetch([(ARCH, "in1")]) + self.assertEqual(self.calls.count("in1"), 3) + + def test_a_failed_listing_is_not_asked_for_again_ahead(self): + class Failing: + def list_store_tarballs(s, arch, h): + self.calls.append(h) + raise OSError("timeout") + lister = plan._Listings(Failing()) + self.assertEqual(lister.prefetch([(ARCH, "x")]), 1) + self.assertEqual(lister.prefetch([(ARCH, "x")]), 0) # next round + with self.assertRaises(OSError): + lister.list_store_tarballs(ARCH, "x") # when needed + self.assertEqual(self.calls, ["x", "x"]) + + +class RoundsTest(unittest.TestCase): + """A package not in the store is built with its local hash: the hashes of what + needs it are listed in a second round, in parallel, not one by one.""" + + def test_changed_hashes_are_listed_in_another_round(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + + def hashes(p, specs, considerRelocation=False): + s = specs[p] + if "remote_revision_hash" in s: + return + dep = specs["zlib"]["hash"] if p == "curl" else "" + s["remote_revision_hash"] = s["remote_hashes"][0] if p == "zlib" else "in-curl-" + dep + s["remote_hashes"] = [s["remote_revision_hash"]] + s["local_revision_hash"] = "local-" + p + s["local_hashes"] = [s["local_revision_hash"]] + specs = {"zlib": {"package": "zlib", "version": "1", "remote_hashes": ["zr"]}, + "curl": {"package": "curl", "version": "1", "remote_hashes": []}} + calls = [] + + class Store: + def list_store_tarballs(self, arch, h): + calls.append((h, threading.current_thread() is threading.main_thread())) + return ["curl-1-1.%s.tar.gz" % arch] if h.startswith("in-curl-") else [] + from bits_helpers.sync import RemoteSync + store = type("S", (Store, RemoteSync), {})() + store.remoteStore = "b3://store" + args = SimpleNamespace(defaults=["release"], architecture=ARCH) + cfg = SimpleNamespace(require_signed_reuse=False, fetch_build_requires=False) + with patch("bits_helpers.build.storeHook"), \ + patch("bits_helpers.hashing.storeHashes", hashes), \ + patch.object(plan, "_overlay_predicate", lambda cfg: None): + states, _, _ = plan.plan(["zlib", "curl"], specs, args, tmp.name, store, "slc9", cfg, None) + self.assertEqual(states, {"zlib": plan.BUILD, "curl": plan.REMOTE}) + self.assertEqual(sorted(calls), [("in-curl-local-zlib", False), ("in-curl-zr", False), + ("zr", False)]) # each once, all ahead in parallel + self.assertEqual(specs["curl"]["hash"], "in-curl-local-zlib") + # Signed: what was found certified is kept, and not asked for in the next run. + trusted = {(ARCH, "in-curl-local-zlib"): "sha"} + for s in specs.values(): + for k in ("remote_revision_hash", "hash", "revision"): + s.pop(k, None) + specs["zlib"]["remote_hashes"] = ["zr"] + cfg.require_signed_reuse = True + for run in (1, 2): + del calls[:] + with patch("bits_helpers.build.storeHook"), \ + patch("bits_helpers.hashing.storeHashes", hashes), \ + patch.object(plan, "_overlay_predicate", lambda cfg: None): + states, _, _ = plan.plan(["zlib", "curl"], specs, args, tmp.name, store, "slc9", cfg, + lambda: trusted) + for s in specs.values(): + for k in ("remote_revision_hash", "hash", "revision"): + s.pop(k, None) + specs["zlib"]["remote_hashes"] = ["zr"] + self.assertEqual(states["curl"], plan.REMOTE) + self.assertEqual(sorted(h for h, _ in calls), + ["in-curl-local-zlib", "in-curl-zr", "zr"] if run == 1 else ["in-curl-zr", "zr"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sync.py b/tests/test_sync.py index 064ebf47..9adfbc38 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -574,5 +574,36 @@ def test_writeStore_disable_propagates_to_writer(self): self.assertEqual(dual.writer.writeStore, "") +class UnreachableStoreTestCase(unittest.TestCase): + """An offline laptop: the store does not answer.""" + + def test_a_failed_listing_is_an_empty_one(self): + with patch("bits_helpers.sync.symlink"), patch("os.makedirs"), \ + patch("os.listdir", return_value=[]): + syncer = sync.HttpRemoteSync("https://store.invalid", ARCHITECTURE, "/sw", False) + with patch.object(syncer, "getRetry", return_value=None): + syncer.fetch_symlinks(GOOD_SPEC) # no TypeError + listing = [{"name": "zlib-v1.3.1-1.%s.tar.gz" % ARCHITECTURE}, "junk"] + with patch.object(syncer, "getRetry", side_effect=[None, listing, None]): + syncer.fetch_symlinks(GOOD_SPEC) # a link that fails to fetch + sync.symlink.assert_not_called() + + def test_unreachable_is_one_quick_try(self): + import requests + syncer = sync.HttpRemoteSync("https://store.invalid", ARCHITECTURE, "/sw", False) + with patch("requests.head", side_effect=requests.ConnectionError("down")) as head, \ + patch("bits_helpers.sync.time.sleep"): + self.assertEqual(sync.store_unreachable(syncer), "ConnectionError") + self.assertEqual(head.call_count, 2) + self.assertEqual(head.call_args.kwargs["timeout"][0], 5) + for answer in (MagicMock(status_code=403), requests.exceptions.SSLError("private CA"), + requests.exceptions.ReadTimeout("slow")): + with patch("requests.head", side_effect=[answer]): + self.assertEqual(sync.store_unreachable(syncer), "") # something answers + self.assertEqual(sync.store_unreachable(sync.NoRemoteSync()), "") + dual = sync.DualRemoteSync(syncer, sync.NoRemoteSync()) + self.assertEqual(dual.probe_url(), "https://store.invalid") + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_trace.py b/tests/test_trace.py new file mode 100644 index 00000000..0cd0263d --- /dev/null +++ b/tests/test_trace.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""bits trace: a CVMFS file's package, and the backend's answer shown.""" + +import json +import os +import shutil +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs, urlsplit + +from bits_helpers import trace + +META = {"package": {"name": "ROOT", "version": "v6.40.02", "revision": "2", "hash": "8687"}, + "effective_architecture": "x86_64-el9-gcc15-opt", "build_id": "release_x", + "bits_version": "0.6", "dist": {"commit": "43badb4296a8"}} + + +class TraceTest(unittest.TestCase): + def test_split_cvmfs(self): + self.assertEqual(trace.split_cvmfs("/cvmfs/bits.cern.ch/a/b/"), ("bits.cern.ch", "a/b")) + self.assertEqual(trace.split_cvmfs("/cvmfs/r"), ("r", "")) + self.assertIsNone(trace.split_cvmfs("/tmp/cvmfs/r/x")) + self.assertIsNone(trace.split_cvmfs("/cvmfs")) + + def test_package_meta_searched_upwards_within_the_repository(self): + top = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, top, True) + pkg = os.path.join(top, "atlas", "ROOT", "v6-2") + os.makedirs(os.path.join(pkg, "lib")) + with open(os.path.join(pkg, ".meta.json"), "w") as fh: + json.dump(META, fh) + f = os.path.join(pkg, "lib", "libCore.so") + open(f, "w").close() + self.assertEqual(trace.package_meta(f, top=top), (pkg, META)) + self.assertEqual(trace.package_meta(os.path.join(top, "atlas"), top=top), (None, None)) + + def test_the_backend_is_asked_with_the_token_and_shown(self): + answer = {"published_path": "atlas/ROOT/v6-2", "indexed": True, + "publishes": [{"published_at": "2026-10-22T22:05:00Z", "replaced": False, + "pipeline": 7, "known": True, "visible": True, + "community": "atlas", "package": "Athena", "version": "25", + "web_url": "https://gl/p/7", "requested_by": "alice", + "approval": {"approver": "bob", "approved_at": "T", "via": "console", + "event": {"seq": 3, "hash": "ab" * 32}}}], + "built_by_pipelines": [{"pipeline": 5, "known": True, "visible": False}]} + seen = {} + + class H(BaseHTTPRequestHandler): + def do_GET(self): + seen["auth"] = self.headers.get("Authorization") + seen["q"] = parse_qs(urlsplit(self.path).query) + body = json.dumps(answer).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + srv = HTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + self.addCleanup(srv.server_close) + self.addCleanup(srv.shutdown) + info = trace.query("http://127.0.0.1:%d/" % srv.server_port, "tok", "r", "a/b", "release_x") + self.assertEqual(seen["auth"], "Bearer tok") + self.assertEqual(seen["q"], {"repo": ["r"], "path": ["a/b"], "build_id": ["release_x"]}) + text = trace.report("/cvmfs/r/a/b", "/cvmfs/r/atlas/ROOT/v6-2", META, info) + self.assertIn("Package ROOT v6.40.02-2 (x86_64-el9-gcc15-opt)", text) + self.assertIn("approved by bob at T (passkey, console); audit event 3 abababab", text) + self.assertIn("Built pipeline 5 (of a community you do not administer)", text) + with self.assertRaises(SystemExit): + trace.query("http://127.0.0.1:1", "tok", "r", "a", "") # nothing listens + + def test_the_token_goes_over_https_and_never_follows_a_redirect(self): + with self.assertRaises(SystemExit) as cm: + trace.query("http://backend.example", "tok", "r", "a", "") + self.assertIn("https only", str(cm.exception)) + hits = [] + + class H(BaseHTTPRequestHandler): + def do_GET(self): + hits.append(self.headers.get("Authorization")) + self.send_response(302) + self.send_header("Location", "http://127.0.0.1:%d/elsewhere" % srv.server_port) + self.end_headers() + + def log_message(self, *a): + pass + srv = HTTPServer(("127.0.0.1", 0), H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + self.addCleanup(srv.server_close) + self.addCleanup(srv.shutdown) + with self.assertRaises(SystemExit) as cm: + trace.query("http://127.0.0.1:%d" % srv.server_port, "tok", "r", "a", "") + self.assertIn("HTTP 302", str(cm.exception)) + self.assertEqual(hits, ["Bearer tok"]) # asked once, not followed + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trust_manifest_derive.py b/tests/test_trust_manifest_derive.py index 52c2bc4e..d89b5fb3 100644 --- a/tests/test_trust_manifest_derive.py +++ b/tests/test_trust_manifest_derive.py @@ -101,5 +101,42 @@ def test_download_failure_falls_back_to_empty(self): dl.downloadUrllib2 = orig + +class TestPreferredPrefixes(unittest.TestCase): + """trust_manifest_prefix may name several prefixes: per architecture, the + first that has a manifest wins; later ones fill in the rest.""" + + def _run(self, listings, arch="x86_64-el9-gcc15-opt"): + from unittest.mock import patch + from bits_helpers import build + root = "https://s3.cern.ch/swift/v1/b" + with patch.object(build, "_list_store_manifest_srcs", + lambda store, prefix, ep, wd: ["%s/%s" % (root, n) + for n in listings.get(prefix, [])]): + return build.trust_manifest_srcs("b3://b", "CANARY/common-manifest, MANIFESTS/common-manifest", + None, "/tmp", arch) + + def test_the_canary_where_it_has_the_arch_production_elsewhere(self): + srcs, guessed = self._run({ + "CANARY/common-manifest": ["CANARY/common-manifest-x86_64-el9-gcc15-opt.json"], + "MANIFESTS/common-manifest": ["MANIFESTS/common-manifest-x86_64-el9-gcc15-opt.json", + "MANIFESTS/common-manifest-x86_64-el10-gcc14-opt.json"]}) + self.assertEqual(sorted(s.split("/b/")[1] for s in srcs), + ["CANARY/common-manifest-x86_64-el9-gcc15-opt.json", + "MANIFESTS/common-manifest-x86_64-el10-gcc14-opt.json"]) + self.assertEqual(guessed, set()) + + def test_an_empty_canary_leaves_production_alone(self): + srcs, guessed = self._run({"MANIFESTS/common-manifest": + ["MANIFESTS/common-manifest-x86_64-el9-gcc15-opt.json"]}) + self.assertEqual([s.split("/b/")[1] for s in srcs], + ["MANIFESTS/common-manifest-x86_64-el9-gcc15-opt.json"]) + self.assertEqual(guessed, set()) + + def test_an_unlistable_store_guesses_every_prefix(self): + srcs, guessed = self._run({}) + self.assertEqual(set(srcs), guessed) + self.assertEqual(len(srcs), 4) # own arch + shared, for each prefix + if __name__ == "__main__": unittest.main() diff --git a/tests/test_trust_sequence.py b/tests/test_trust_sequence.py new file mode 100644 index 00000000..59c75322 --- /dev/null +++ b/tests/test_trust_sequence.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later +"""A manifest with a sequence: a node never goes back to an older one.""" + +import json +import os +import shutil +import tempfile +import unittest +from unittest import mock + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from bits_helpers import trust + + +class SequenceTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.priv = Ed25519PrivateKey.generate() + keys = os.path.join(self.tmp, "keys") + os.makedirs(keys) + with open(os.path.join(keys, "pub.pem"), "wb") as fh: + fh.write(self.priv.public_key().public_bytes( + serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo)) + with open(os.path.join(keys, "key-policy.json"), "w") as fh: + fh.write('{"default": ["*"]}\n') + self.state = os.path.join(self.tmp, "state", "trust-sequence.json") + env = mock.patch.dict(os.environ, {"BITS_TRUST_KEYS": keys, + "BITS_TRUST_SEQUENCE_STATE": self.state}) + env.start() + self.addCleanup(env.stop) + + def manifest(self, name, seq, arch="el9", hashes=("h1",), **extra): + doc = dict({"kind": "common-manifest", "architecture": arch, + "packages": [{"hash": h, "tarball_sha256": "sha256:" + h, + "effective_architecture": arch} for h in hashes]}, **extra) + if seq is not None: + doc["sequence"] = seq + path = os.path.join(self.tmp, name) + raw = json.dumps(doc).encode() + with open(path, "wb") as fh: + fh.write(raw) + with open(path + ".sig", "w") as fh: + json.dump(trust.sign_bytes(raw, self.priv), fh) + return path + + def trusted(self, path): + return trust.trusted_index(path)[0] is not None + + def test_a_node_never_goes_back(self): + self.assertTrue(self.trusted(self.manifest("m1", 1))) + self.assertTrue(self.trusted(self.manifest("m3", 3, hashes=("h1", "h3")))) + with self.assertLogs("bits", "WARNING") as logs: + self.assertFalse(self.trusted(self.manifest("m2", 2))) # put back + self.assertIn("older than the 3", logs.output[0]) + self.assertTrue(self.trusted(os.path.join(self.tmp, "m3"))) # the same one again + + def test_two_manifests_under_one_number(self): + self.assertTrue(self.trusted(self.manifest("a", 5))) + with self.assertLogs("bits", "WARNING"): + self.assertFalse(self.trusted(self.manifest("b", 5, hashes=("h2",)))) + + def test_each_architecture_counts_on_its_own(self): + self.assertTrue(self.trusted(self.manifest("x", 9, arch="el9"))) + self.assertTrue(self.trusted(self.manifest("y", 1, arch="el10"))) + + def test_after_numbered_ones_an_unnumbered_one_is_refused(self): + self.assertTrue(self.trusted(self.manifest("ci", None))) # before the switch + self.assertTrue(self.trusted(self.manifest("w", 1))) + with self.assertLogs("bits", "WARNING"): + self.assertFalse(self.trusted(self.manifest("ci2", None))) + self.assertTrue(self.trusted(self.manifest("ci10", None, arch="el10"))) + + def test_each_issuer_numbers_on_its_own(self): + self.assertTrue(self.trusted(self.manifest("p", 40, issuer="bits-orchestrator/prod"))) + self.assertTrue(self.trusted(self.manifest("t", 3, issuer="bits-orchestrator/test"))) + with self.assertLogs("bits", "WARNING"): + self.assertFalse(self.trusted(self.manifest("p2", 39, + issuer="bits-orchestrator/prod"))) + + def test_a_damaged_entry_starts_again(self): + self.assertTrue(self.trusted(self.manifest("a", 2))) + with open(self.state) as fh: + state = json.load(fh) + for k in state: + if not k.startswith("numbered-by"): + state[k] = {"sequence": "two"} + with open(self.state, "w") as fh: + json.dump(state, fh) + self.assertTrue(self.trusted(self.manifest("b", 1))) + + def test_manifests_without_a_sequence_are_left_to_their_expiry(self): + self.assertTrue(self.trusted(self.manifest("old", None))) + self.assertFalse(os.path.exists(self.state)) + self.assertFalse(self.trusted(self.manifest("exp", None, expires="2020-01-01T00:00:00Z"))) + + def test_a_malformed_sequence_is_refused(self): + with self.assertLogs("bits", "WARNING"): + self.assertFalse(self.trusted(self.manifest("bad", "7"))) + + def test_an_unwritable_memory_does_not_stop_the_build(self): + os.makedirs(os.path.dirname(self.state)) + os.chmod(os.path.dirname(self.state), 0o500) + self.addCleanup(os.chmod, os.path.dirname(self.state), 0o700) + if os.access(os.path.dirname(self.state), os.W_OK): + self.skipTest("running as root") + with self.assertLogs("bits", "WARNING"): + self.assertTrue(self.trusted(self.manifest("m", 1))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_upgrade_cmd.py b/tests/test_upgrade_cmd.py new file mode 100644 index 00000000..ef97e603 --- /dev/null +++ b/tests/test_upgrade_cmd.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""bits upgrade: version edits, plans, and the recipes edited in place.""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + + +from bits_helpers import upgrade_cmd as up +from bits_helpers.upgrade_cmd import NeedsHand + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +GIT = ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-c", "init.defaultBranch=main"] + + +class EditTest(unittest.TestCase): + def test_tag_rules(self): + self.assertEqual(up.new_tag("v1.2.13", "1.2.13", "1.3.1"), "v1.3.1") + self.assertEqual(up.new_tag("curl-8_18_0", "8.18.0", "8.21.0"), "curl-8_21_0") + self.assertEqual(up.new_tag("1-2-3", "1.2.3", "1.2.4"), "1-2-4") + self.assertIsNone(up.new_tag("v%(version)s", "1.0", "2.0")) + with self.assertRaises(NeedsHand): + up.new_tag("release-candidate", "1.0", "2.0") + + def test_recipe_version_and_tag(self): + text = 'package: curl\nversion: "8.18.0" # pinned\ntag: curl-8_18_0\n---\nversion: 8.18.0\n' + out, edits = up.edit_recipe(text, "8.18.0", "8.21.0") + self.assertEqual(out, 'package: curl\nversion: "8.21.0" # pinned\ntag: curl-8_21_0\n' + '---\nversion: 8.18.0\n') # the body is never touched + self.assertEqual(edits, [("version", "8.18.0", "8.21.0"), ("tag", "curl-8_18_0", "curl-8_21_0")]) + + def test_recipe_derived_tag_and_refusals(self): + out, edits = up.edit_recipe("package: im\nversion: 7.1\ntag: v%(version)s\n---\n", "7.1", "7.2") + self.assertIn("tag: v%(version)s", out) + self.assertEqual(edits, [("version", "7.1", "7.2")]) + with self.assertRaises(NeedsHand): # not the version it says it is + up.edit_recipe("package: a\nversion: 1.1\n---\n", "1.0", "2.0") + with self.assertRaises(NeedsHand): # a tag that does not spell it + up.edit_recipe("package: a\nversion: 1.0\ntag: master\n---\n", "1.0", "2.0") + + def test_override_with_matchers(self): + text = ("package: defaults-dev4\nversion: v1\noverrides:\n" + " ROOT:\n version: \"6.30\"\n" + " ROOT:osx.*:\n version: \"6.30\"\n tag: v6-30\n" + " zlib:\n version: 1.3\n" + "env:\n A: b\n---\n") + out, edits = up.edit_override(text, "root", "6.30", "6.32", active=lambda m: m != "osx.*") + self.assertEqual(edits, [("version", "6.30", "6.32")]) + self.assertIn(' ROOT:\n version: "6.32"\n', out) + self.assertIn(' ROOT:osx.*:\n version: "6.30"\n tag: v6-30\n', out) # not this build's + out, _ = up.edit_override(text, "root", "6.30", "6.32", active=lambda m: True) + self.assertIn("tag: v6-32", out) + with self.assertRaises(NeedsHand): + up.edit_override(text, "gcc", "13", "14") + + def test_override_list_form_and_pin(self): + out, _ = up.edit_override("overrides:\n - zlib = 1.3\n---\n", "zlib", "1.3", "1.3.1") + self.assertEqual(out, "overrides:\n - zlib = 1.3.1\n---\n") + out, edits = up.edit_pin("package: a\nrequires:\n - \"zlib = 1.3\"\n - curl\n---\n", "zlib", "1.3", "1.4") + self.assertIn(' - "zlib = 1.4"\n', out) + self.assertEqual(edits, [("pin", "1.3", "1.4")]) + with self.assertRaises(NeedsHand): + up.edit_pin("package: a\nrequires:\n - zlib\n---\n", "zlib", "1.3", "1.4") + + +class MoreEditTest(unittest.TestCase): + def test_a_version_yaml_would_read_as_a_number_is_quoted(self): + out, _ = up.edit_recipe("package: z\nversion: 1.3\n---\n", "1.3", "1.10") + self.assertIn('version: "1.10"\n', out) + out, _ = up.edit_recipe("package: z\nversion: 1.3\n---\n", "1.3", "1.3.1") + self.assertIn("version: 1.3.1\n", out) + + def test_override_entries_only(self): + text = ("overrides:\n" + " app:\n version: 2\n requires:\n - zlib = 1.3\n" + " variables:\n version: 1.3\n" + " zlib:\n variables:\n version: 1.3\n version: 1.3\n---\n") + out, edits = up.edit_override(text, "zlib", "1.3", "1.4") + self.assertEqual(edits, [("version", "1.3", "1.4")]) + self.assertIn(" - zlib = 1.3\n", out) # app's requires: not an entry + self.assertEqual(out.count("version: 1.3\n"), 2) # the variables: stay + self.assertTrue(out.endswith(" version: \"1.4\"\n---\n")) # quoted: 1.4 reads as a number + + def test_source_checks_resolve_the_new_tag(self): + seen = [] + def tag_exists(source, tag): + seen.append(tag) + return True, "found" + def resolve(spec, text): + return text.replace("%(version)s", spec["version"]) + orig = up._git_tag_exists + up._git_tag_exists = tag_exists + try: + spec = {"package": "a", "version": "1.0", "tag": "v1.0"} + up.check_sources(spec, "package: a\nversion: 2.0\ntag: v%(version)s\nsource: https://x/a\n---\n", + "2.0", resolve) + up.check_sources(spec, "package: a\nversion: 2.0\nsource: https://x/a\n---\n", "2.0", resolve) + finally: + up._git_tag_exists = orig + self.assertEqual(seen, ["v2.0", "2.0"]) + + + + +class WantedTest(unittest.TestCase): + def _plan(self, text): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + path = os.path.join(d, "plan.csv") + with open(path, "w") as f: + f.write(text) + return path + + def _args(self, **kw): + base = dict(to=[], plan=None, waves=[], line=None) + base.update(kw) + return argparse.Namespace(**base) + + def _parser(self): + p = argparse.ArgumentParser() + p.error = lambda msg: (_ for _ in ()).throw(SystemExit(msg)) + return p + + def test_the_consoles_csv_exports(self): + per_line = self._plan("wave,package,from,to,fixes,kev,risk,rebuilds,major,pinned_by\r\n" + "0,go,1.12.5,1.21.3,CVE-1,CVE-1,80,13,,\r\n1,curl,8.1.0,8.6.0,CVE-2,,9,3,,\r\n" + "host,zlib,1.2,1.3,CVE-3,,1,,,\r\nno fix,left,1.0,,CVE-4,,3,,,\r\n") + self.assertEqual(up.wanted(self._args(plan=per_line, waves=[0]), self._parser()), {"go": "1.21.3"}) + self.assertEqual(up.wanted(self._args(plan=per_line, waves=[0, 1]), self._parser()), + {"go": "1.21.3", "curl": "8.6.0"}) + self.assertEqual(up.wanted(self._args(plan=per_line), self._parser()), # every wave + {"go": "1.21.3", "curl": "8.6.0"}) + conv = self._plan("package,version,community,line,platform,from,action,major,pinned_by,own_target,kev,risk\r\n" + "curl,8.7.0,lcg,release_gcc15,x86_64-el9,8.1.0,upgrade,,,8.6.0,,9\r\n" + "java,21,lcg,release_gcc15,x86_64-el9,17,pinned,,defaults:dev4,21,,2\r\n" + "curl,8.7.0,lcg,release_gcc14,x86_64-el9,8.7.0,keep,,,,,0\r\n") + self.assertEqual(up.wanted(self._args(plan=conv, line="release_gcc15"), self._parser()), {"curl": "8.7.0"}) + with self.assertRaises(SystemExit): + up.wanted(self._args(plan=conv), self._parser()) # which line + self.assertEqual(up.wanted(self._args(to=["go=1.26.9"]), self._parser()), {"go": "1.26.9"}) + bom = self._plan("\ufeffwave,package,from,to\r\n0,go,1,2\r\n0,short\r\n") + self.assertEqual(up.wanted(self._args(plan=bom, waves=[0]), self._parser()), {"go": "2"}) + for bad in (self._args(to=["go"]), self._args(plan=self._plan("a,b\r\n1,2\r\n")), + self._args(plan=per_line, waves=[7])): + with self.assertRaises(SystemExit): + up.wanted(bad, self._parser()) + + def test_a_plan_among_the_packages(self): + args = argparse.Namespace(pkgname=["LCG", "plan.csv"], plan=None) + up.split_plan(args, self._parser()) + self.assertEqual((args.pkgname, args.plan), (["LCG"], "plan.csv")) + for bad in (["LCG", "a.csv", "b.csv"], ["plan.csv"]): + with self.assertRaises(SystemExit): + up.split_plan(argparse.Namespace(pkgname=bad, plan=None), self._parser()) + + def test_the_newest_plan_in_the_current_directory(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + cwd = os.getcwd() + os.chdir(d) + self.addCleanup(os.chdir, cwd) + with self.assertRaises(SystemExit): + up.wanted(self._args(), self._parser()) # no plan here + for name, text, when in (("upgrade-plan-LCG_109.csv", "wave,package,from,to\r\n0,go,1,2\r\n", 1), + ("upgrade-plan-LCG_110.csv", "wave,package,from,to\r\n1,curl,1,3\r\n", 2)): + with open(name, "w") as f: + f.write(text) + os.utime(name, (when, when)) + self.assertEqual(up.wanted(self._args(), self._parser()), {"curl": "3"}) + self.assertEqual(up.wanted(self._args(to=["go=4"]), self._parser()), {"go": "4"}) # --to alone + + +class GitInsteadTest(unittest.TestCase): + RECIPE = ('package: fmt\nversion: "12.1.0"\ntag: "%(version)s"\nsources:\n' + ' - https://lcgpackages.example/fmt-%(version)s.tar.gz\npatches:\n - fmt-fix.patch\n' + 'requires:\n - CMake\n---\n#!/bin/bash\n') + + def test_tag_spellings(self): + tags = {"v2.0.1", "curl-8_18_0", "boost-1.86.0", "1.0"} + self.assertEqual(up.find_tag(tags, "x", "2.0.1"), "v2.0.1") + self.assertEqual(up.find_tag(tags, "curl", "8.18.0"), "curl-8_18_0") + self.assertEqual(up.find_tag(tags, "Boost", "1.86.0"), "boost-1.86.0") + self.assertIsNone(up.find_tag(tags, "x", "3.0")) + self.assertEqual(up.find_tag({"R_2_9_0"}, "expat", "2.9.0", "R_2_9_0"), "R_2_9_0") # the recipe's + + def test_sources_and_patches_commented_out(self): + text = up.to_git(self.RECIPE, "https://github.com/fmtlib/fmt", "v12.2.0", "12.2.0") + self.assertEqual(text.split("---")[0], + 'package: fmt\nversion: "12.1.0"\nsource: https://github.com/fmtlib/fmt\n' + 'tag: "v%(version)s"\n# bits upgrade: no tarball of 12.2.0, built from the git tag\n' + '# sources:\n# - https://lcgpackages.example/fmt-%(version)s.tar.gz\n' + '# patches:\n# - fmt-fix.patch\nrequires:\n - CMake\n') + import yaml + meta = yaml.safe_load(text.split("---")[0]) + self.assertNotIn("sources", meta) + self.assertEqual(meta["tag"], "v%(version)s") + gaps = up.to_git("package: a\nversion: 1\nsources:\n - a\n\n# b too\n - b\n\n# deps\nrequires:\n" + " - c\n---\n", "https://x/a", "2", "2") + self.assertEqual(yaml.safe_load(gaps.split("---")[0]), + {"package": "a", "version": 1, "source": "https://x/a", "tag": "%(version)s", + "requires": ["c"]}) + self.assertIn("\n# deps\nrequires:", gaps) + own = up.to_git(self.RECIPE.replace("tag: \"%(version)s\"\n", "source: https://x/old\n"), + "https://x/old", "fmt_12_2_0", "12.2.0") + self.assertIn('source: https://x/old\ntag: "fmt_12_2_0"\n', own) + + def test_configure_generated_for_autotools(self): + gen = up.generate_configure + head = "package: a\nversion: 1\n---\n#!/bin/bash -e\n. $(bits-include AutoToolsRecipe)\n" + own = head + "function Configure() {\n if x; then\n y\n fi\n CFLAGS=-O2 \\\n ./configure\n}\n" + text, made = gen(own) + self.assertEqual(made, "made") + self.assertIn("function Configure() {\n [ -e configure ] || [ ! -e configure.ac ] || autoreconf -fi", text) + self.assertEqual(gen(text), (text, None)) # once only + brace = head + "Configure()\n{\n ./configure\n}\n" + self.assertIn("Configure()\n{\n [ -e configure ]", gen(brace)[0]) + text, made = gen(head) # the default Configure + self.assertEqual(made, "made") + self.assertTrue(text.endswith(' [ -f autogen.sh ] && ./autogen.sh\n ./configure --prefix="$INSTALLROOT"\n}\n')) + two = head + "if a; then\n Configure() {\n ./configure\n }\nelse\n function Configure {\n ./configure\n }\nfi\n" + self.assertEqual(gen(two)[0].count("autoreconf"), 2) + cm = "package: a\nversion: 1\n---\n. $(bits-include CMakeRecipe)\ncp $SOURCEDIR/configure.h.in x\n" + self.assertEqual(gen(cm), (cm, None)) + for odd in (head + "Configure() { ./configure; }\n", # one line + head + "function Configure { ./configure --x; }\n", + "package: a\nversion: 1\n---\ncd b\n../configure --prefix=$INSTALLROOT\nmake\n"): + self.assertEqual(gen(odd), (odd, "check")) + for none in ("package: a\nversion: 1\n---\n. $(bits-include CMakeRecipe)\n# not ./configure\n", + "package: a\nversion: 1\n"): + self.assertEqual(gen(none), (none, None)) + for t in (gen(own)[0], gen(head)[0], gen(brace)[0]): + r = subprocess.run(["bash", "-n"], input=t.split("---\n", 1)[1], text=True, capture_output=True) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_the_first_repository_with_the_tag(self): + tried = [] + def repos(meta, package): + if meta.get("source"): + yield meta["source"], False + yield "https://github.com/a/fmt.git", True + yield "https://github.com/a/fmt", True # the same one again + yield "https://github.com/fmtlib/fmt", True + tags = {"https://github.com/fmtlib/fmt": {"12.2.0"}} + orig = up._repos, up._tags, up._release_assets + up._repos = repos + up._tags = lambda r: tried.append(r) or tags.get(r, set()) + up._release_assets = lambda repo, tag: [] # no release files + try: + got = up.upstream_instead({"package": "fmt"}, + self.RECIPE.replace("sources:", "source: https://x/fmt\nsources:"), "12.2.0") + self.assertEqual((got[0],) + got[2:], ("git", "https://github.com/fmtlib/fmt", "12.2.0", True)) + self.assertEqual(tried, ["https://x/fmt", "https://github.com/a/fmt.git", + "https://github.com/fmtlib/fmt"]) + self.assertIsNone(up.upstream_instead({"package": "fmt"}, self.RECIPE, "9.9")) + finally: + up._repos, up._tags, up._release_assets = orig + + def test_an_upstream_release_tarball_first(self): + rel = "https://github.com/libexpat/libexpat/releases/download/R_2_9_0/" + assets = [rel + "expat-2.9.0.tar.xz.asc", rel + "expat-2.9.0.tar.xz", rel + "expat-2.9.0.tar.gz", + rel + "expat-win32bin-2.9.0.zip"] + orig = up._repos, up._tags, up._release_assets, up._url_exists + up._repos = lambda meta, package: iter([("https://github.com/libexpat/libexpat", False)]) + up._tags = lambda r: {"R_2_9_0"} + up._release_assets = lambda repo, tag: assets if tag == "R_2_9_0" else [] + up._url_exists = lambda url: (True, "HTTP 200") + try: + recipe = ('package: expat\nversion: "2.9.0"\ntag: "R_2_9_0"\nsources:\n' + ' - https://lcgpackages.example/%(name)s-%(version)s.tar.gz\npatches:\n - x.patch\n---\n') + kind, text, repo, url, guessed = up.upstream_instead({"package": "expat"}, recipe, "2.9.0") + self.assertEqual((kind, url), ("tarball", rel + "expat-2.9.0.tar.gz")) + self.assertIn("# bits upgrade: no tarball of 2.9.0 in the usual place, upstream's instead\nsources:\n" + " - https://github.com/libexpat/libexpat/releases/download/R_2_9_0/expat-%(version)s.tar.gz\n" + "patches:\n - x.patch\n", text) + two = recipe.replace("sources:\n", "sources:\n - https://other/data.tgz\n") + self.assertIsNone(up.to_tarball(two, url, "2.9.0")) # left to git + self.assertEqual(up.upstream_instead({"package": "expat"}, two, "2.9.0")[0], "git") + cm = "https://github.com/Kitware/CMake/releases/download/v3.30.0/" + assets[:] = [cm + "cmake-3.30.0-linux-aarch64.tar.gz", cm + "cmake-3.30.0-docs.tar.gz", + cm + "cmake-3.30.0.tar.gz", cm + "cmake-3.30.0.1.tar.gz"] + self.assertEqual(up.release_tarball("https://github.com/x/y", "R_2_9_0", "3.30.0", "CMake"), + cm + "cmake-3.30.0.tar.gz") # not a binary nor docs + lv = "https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.0/" + assets[:] = [lv + "lld-18.1.0.src.tar.xz", lv + "llvm-18.1.0.src.tar.xz", lv + "llvm-project-18.1.0.src.tar.xz"] + self.assertEqual(up.release_tarball("https://github.com/llvm/llvm-project", "R_2_9_0", "18.1.0", "llvm"), + lv + "llvm-18.1.0.src.tar.xz") + ev = "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/" + self.assertEqual(up._templated(ev + "libevent-2.1.12-stable.tar.gz", "2.1.12"), + ev.replace("2.1.12", "%(version)s") + "libevent-%(version)s-stable.tar.gz") + self.assertEqual(up._templated("https://github.com/foo/lib1/releases/download/v1/lib1-1.tar.gz", "1"), + "https://github.com/foo/lib1/releases/download/v%(version)s/lib1-%(version)s.tar.gz") + self.assertEqual(up.release_tarball("https://gitlab.gnome.org/GNOME/glib", None, "2.88.1"), + "https://download.gnome.org/sources/glib/2.88/glib-2.88.1.tar.xz") + finally: + up._repos, up._tags, up._release_assets, up._url_exists = orig + + +RECIPES = { + "defaults-release.sh": 'package: defaults-release\nversion: v1\noverrides:\n zlib:\n' + ' version: "1.3.1"\n---\n', + "zlib.sh": 'package: zlib\nversion: "1.2.13"\ntag: v1.2.13\nsource: https://example.invalid/zlib\n' + '---\necho zlib\n', + "app.sh": 'package: app\nversion: "1.0"\nrequires:\n - curl\n---\necho app\n', + "other.sh": 'package: other\nversion: "2.0"\n---\necho other\n', +} +PROVIDED = { # in a repository provider's cached checkout + "curl.sh": "package: curl\nversion: 8.18.0\ntag: curl-8_18_0\nsource: https://example.invalid/curl\n" + "patches:\n - curl-fix.patch\n - curl-osx.patch:osx.* && strip=0\nrequires:\n - zlib\n" + "---\necho curl\n", + "patches/curl-fix.patch": "--- a\n+++ b\n", + "patches/curl-osx.patch": "--- a\n+++ b\n", + "checksums/curl.checksum": "sources: {}\n", + ".bits_provider_ok": "abc\n", +} + + +class InPlaceTest(unittest.TestCase): + def setUp(self): + self.tmp = t = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, t, True) + self.cfg, self.sw = os.path.join(t, "recipes"), os.path.join(t, "sw") + self.cache = os.path.join(self.sw, "REPOS", "lcg.bits", "abc") + for d, files in ((self.cfg, RECIPES), (self.cache, PROVIDED)): + for name, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(d, name)), exist_ok=True) + with open(os.path.join(d, name), "w") as f: + f.write(text) + self._git(d, "init", "-q") + self._git(d, "add", ".") + self._git(d, "commit", "-qm", "init") + site = os.path.join(t, "site") + os.makedirs(site) + with open(os.path.join(site, "99-test.yaml"), "w") as f: + f.write("providers:\n url: null\n") # no network + self.env = dict(os.environ, BITS_SITE_DIR=site, XDG_CONFIG_HOME=os.path.join(t, "xdg"), + BITS_PATH=self.cache) + for k in ("BITS_PROVIDERS", "BITS_REPO_DIR"): + self.env.pop(k, None) + + def _git(self, cwd, *argv): + subprocess.run(GIT + list(argv), cwd=cwd, check=True, capture_output=True) + + def _bits(self, *argv, code=0): + r = subprocess.run([sys.executable, os.path.join(ROOT, "bitsBuild")] + list(argv), + cwd=self.tmp, env=self.env, capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, code, r.stdout + r.stderr) + return r.stdout + r.stderr + + def _upgrade(self, *more, code=0): + return self._bits("upgrade", "app", "other", "-c", self.cfg, "-w", self.sw, "-a", "slc9_x86-64", + "--no-check", *more, code=code) + + def _versions(self): + out = self._bits("status", "app", "other", "-c", self.cfg, "-w", self.sw, "-a", "slc9_x86-64", "--json") + return {p["package"]: p["version"] for p in json.loads(out[out.index("{"):out.rindex("}") + 1])["packages"]} + + def test_edited_in_place_or_copied_from_a_provider(self): + self._bits("-n", "upgrade", "app", "other", "-c", self.cfg, "-w", self.sw, "-a", "slc9_x86-64", + "--no-check", "--to", "curl=8.21.0") + self.assertFalse(os.path.exists(os.path.join(self.cfg, "curl.sh"))) # a dry run writes nothing + out = self._upgrade("--to", "curl=8.21.0", "--to", "zlib=1.3.2", "--to", "other=2.0") + self.assertIn("copied curl.sh, patches/curl-fix.patch, patches/curl-osx.patch," + " checksums/curl.checksum from", out) + with open(os.path.join(self.cfg, "curl.sh")) as f: + self.assertIn("version: 8.21.0\ntag: curl-8_21_0\n", f.read()) + self.assertTrue(os.path.isfile(os.path.join(self.cfg, "patches", "curl-fix.patch"))) + with open(os.path.join(self.cache, "curl.sh")) as f: + self.assertEqual(f.read(), PROVIDED["curl.sh"]) # the cache untouched + with open(os.path.join(self.cfg, "defaults-release.sh")) as f: + self.assertIn(' zlib:\n version: "1.3.2"\n', f.read()) # the override pins it + self.assertEqual(self._versions(), {"defaults-release": "v1", "zlib": "1.3.2", "curl": "8.21.0", + "app": "1.0", "other": "2.0"}) + + def test_a_plan_csv_after_the_packages(self): + with open(os.path.join(self.tmp, "plan.csv"), "w") as f: + f.write("wave,package,from,to\r\n0,curl,8.18.0,8.21.0\r\n2,other,2.0,3.0\r\n") + out = self._upgrade("--wave", "0", "plan.csv") # after the options too + self.assertIn("Plan: plan.csv", out) + self.assertEqual(self._versions()["curl"], "8.21.0") + self.assertEqual(self._versions()["other"], "2.0") # wave 2 not taken + + def test_two_edits_of_one_copied_recipe(self): + with open(os.path.join(self.cache, "curl.sh")) as f: + text = f.read().replace(" - zlib\n", " - zlib = 1.3.1\n") # curl pins zlib + with open(os.path.join(self.cache, "curl.sh"), "w") as f: + f.write(text) + for dry in (True, False): + out = self._bits(*(["-n"] if dry else []), "upgrade", "app", "other", "-c", self.cfg, "-w", self.sw, + "-a", "slc9_x86-64", "--no-check", "--to", "curl=8.21.0", "--to", "zlib=1.3.2") + self.assertIn("in the copy made above", out) + with open(os.path.join(self.cfg, "curl.sh")) as f: + text = f.read() + self.assertIn("version: 8.21.0", text) + self.assertIn(" - zlib = 1.3.2\n", text) + + def test_a_copy_never_replaces_a_file(self): + with open(os.path.join(self.cfg, "curl.sh"), "w") as f: + f.write("mine\n") + with self.assertRaises(NeedsHand): + up._bring_in(os.path.join(self.cache, "curl.sh"), self.cfg, "curl", dry=False) + with open(os.path.join(self.cfg, "curl.sh")) as f: + self.assertEqual(f.read(), "mine\n") + self.assertFalse(os.path.exists(os.path.join(self.cfg, "patches"))) + + def test_left_for_a_hand_edit(self): + out = self._upgrade("--to", "nope=1", "--to", "zlib=1.3.2", code=1) + self.assertIn("not in the build", out) + with open(os.path.join(self.cfg, "app.sh"), "w") as f: + f.write('package: app\nversion: "1.0"\ntag: master\nrequires:\n - curl\n---\n') + self.assertIn("edit by hand", self._upgrade("--to", "app=2.0", code=1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 1f19238c..a9c55deb 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -16,7 +16,7 @@ from bits_helpers.utilities import apply_version_from from bits_helpers.utilities import topological_sort from bits_helpers.matchers import _parse_req_matcher, _collect_version_pins -from bits_helpers.defaults import asDict, merge_dicts +from bits_helpers.defaults import asDict, merge_dicts, parseDefaults from bits_helpers.matchers import _version_compare, _parse_patch_entry, filterPatches, _matcher_active from collections import OrderedDict import bits_helpers @@ -382,6 +382,21 @@ def test_resolve_variables(self) -> None: self.assertEqual(["herwig3"], list(filterByArchitectureDefaults( linux, ["dev4"], ["herwig3:(?heavygen)"], rv))) + def test_variables_not_exported(self) -> None: + """`export: false`: a --set of the variable sets it, but stays out of the env.""" + from bits_helpers.matchers import exported_flavours + linux = "ubuntu2510_x86-64-gcc15" + block = {"release": "main", "athena_release": {"value": "main", "export": False}} + self.assertEqual("main", resolve_variables(block, {}, linux, ["atlas"])["athena_release"]) + cli = {"release": "LCG_110", "athena_release": "release/25.0.75"} + self.assertEqual("release/25.0.75", + resolve_variables(block, cli, linux, ["atlas"])["athena_release"]) + self.assertEqual({"release": "LCG_110"}, dict(exported_flavours(block, cli))) + self.assertEqual(cli, dict(exported_flavours({"release": "main"}, cli))) + with patch("bits_helpers.matchers.dieOnError", side_effect=SystemExit): + with self.assertRaises(SystemExit): + exported_flavours({"x": {"value": 1, "export": "no"}}, {}) + def test_parse_req_matcher(self) -> None: """_parse_req_matcher should correctly parse all requirement syntax variants.""" # Plain name @@ -633,6 +648,41 @@ def test_empty_is_empty(self): self.assertEqual(asDict([]), {}) self.assertEqual(asDict(None), {}) +class OverrideMatcherCaseTest(unittest.TestCase): + """parseDefaults lowercases an override's package regex, not its matcher: + variable names, versions and profile names in the matcher keep their case.""" + + ARCH = "slc9_x86-64" + + def _keys(self, *keys): + meta = {"package": "defaults-release", "version": "v1", + "overrides": OrderedDict((k, {"tag": "t"}) for k in keys)} + err, overrides, _, _ = parseDefaults([], lambda: (meta, ""), print) + self.assertIsNone(err) + return [k for k in overrides if k != "defaults-release"] + + def _active(self, key, version): + matcher = key.partition(":")[2] + return _matcher_active(matcher, self.ARCH, ["release", "O2"], + {"ENABLE_CUDA": True}, version) + + def test_package_lowercased_matcher_kept(self): + self.assertEqual(self._keys("ROOT:(?ENABLE_CUDA)", "Boost@dist", "ROOT"), + ["root:(?ENABLE_CUDA)", "boost", "root"]) + + def test_variable_matcher_still_active(self): + key, = self._keys("ROOT:(?ENABLE_CUDA)") + self.assertTrue(self._active(key, "v6-32-01")) + + def test_version_matcher_still_active(self): + key, = self._keys("ROOT:version>=v6-30-01-ALICE") + self.assertTrue(self._active(key, "v6-32-01-ALICE")) + self.assertFalse(self._active(key, "v6-28-01-ALICE")) + + def test_defaults_matcher_still_active(self): + key, = self._keys("ROOT:defaults=O2") + self.assertTrue(self._active(key, None)) + class DefaultsChainMergeTest(unittest.TestCase): """Chained defaults (a::b::c) deep-merge overrides: union of entries, last diff --git a/tests/test_workarea.py b/tests/test_workarea.py index b4cb6152..4393bc49 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -321,6 +321,16 @@ def test_patch_strip_uses_declared_level(self, mock_run): self.assertEqual(self._patch_cmd(mock_run.call_args), ["patch", "-p0", "--batch", "--input", expected]) + def test_patchset_fingerprint_follows_the_strip_level(self): + """A tree patched at another strip level is not reused: a declared level + changes the fingerprint, the -p1 default (implicit or explicit) does not.""" + from bits_helpers.workarea import _patchset_fingerprint + fp = lambda strip: _patchset_fingerprint(dict(self._spec(patches=["fix-a.patch"]), + patch_strip=strip), self.source_dir) + self.assertEqual(fp({}), fp({"fix-a.patch": 1})) + self.assertNotEqual(fp({}), fp({"fix-a.patch": 0})) + self.assertEqual(fp({"fix-b.patch": 0}), fp({})) # another patch's level + @patch("subprocess.run") def test_patch_strip_absent_defaults_to_p1(self, mock_run): """Without a declared strip, application stays at the -p1 default.""" diff --git a/tox.ini b/tox.ini index 56d51829..3a8fa856 100644 --- a/tox.ini +++ b/tox.ini @@ -54,6 +54,8 @@ setenv = # Keep coverage info for later upload, if needed. Files in {envtmpdir} are # deleted after each run. COVERAGE_FILE = {toxworkdir}/coverage.{envname} + # bits' own site configuration only, not the host's /etc/bits/site.d. + BITS_SITE_ISOLATED = 1 ARCHITECTURE = slc7_x86-64 darwin: ARCHITECTURE = osx_arm64