From 4756b0af6a9c0ddf9cf09641cd54e2f4d0257613 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 14:57:23 +1000 Subject: [PATCH 01/10] Windows: no Defender real-time scan in the build jobs Installing the build and host environments and deleting a package's work directory afterwards took 3-4x as long as compiling in lyrical's win-64 jobs (e.g. 377 s of installs and 186 s of clean-up against 127 s of builds, the clean-up ending in "Access is denied" retries): Defender scanning every file. The runners are thrown away after each job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 3 +++ .scripts/build_win.bat | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index bcdfe794b..82cea6769 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -177,6 +177,9 @@ jobs: if: runner.os == 'Windows' shell: pwsh run: | + # Defender's real-time scan slows down installing and deleting the environments' + # files several times over; the runner is thrown away after the job + Set-MpPreference -DisableRealtimeMonitoring $true "CONDA_BLD_PATH=C:\bld\" | Out-File -FilePath $env:GITHUB_ENV -Append New-Item -ItemType Directory -Force -Path C:\bld | Out-Null New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index a6c29e123..c5e66d6f8 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -1,6 +1,10 @@ setlocal EnableExtensions EnableDelayedExpansion +:: Defender's real-time scan slows down installing and deleting the environments' +:: files several times over; the runner is thrown away after the job +powershell -NoProfile -Command "Set-MpPreference -DisableRealtimeMonitoring $true" + set CONDA_BLD_PATH=C:\bld echo "PATH is %PATH%" From a8fd40b5bad1cf1d75f09365f711ed7f90248323 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 15:03:52 +1000 Subject: [PATCH 02/10] macOS: Spotlight off in the build jobs Spotlight's indexer works through every file the environments and builds write; other projects saw it take one or two of a GitHub macOS runner's three or four cores (PennyLaneAI/catalyst: "xcode builds very slow if we don't turn off spotlight"). The runners are thrown away after each job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 6 ++++++ .scripts/build_unix.sh | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index 82cea6769..c221df5e6 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -186,6 +186,12 @@ jobs: Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + - name: Turn off Spotlight + if: runner.os == 'macOS' + run: sudo mdutil -a -i off || true + - name: Clean up PATH if: runner.os == 'Windows' uses: egor-tensin/cleanup-path@v5 diff --git a/.scripts/build_unix.sh b/.scripts/build_unix.sh index 39d855bd4..2ef476aa8 100755 --- a/.scripts/build_unix.sh +++ b/.scripts/build_unix.sh @@ -24,6 +24,9 @@ export "CONDA_BLD_PATH=$HOME/conda-bld/" if [[ "$target" == *"osx"* ]]; then echo "osx" + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + sudo mdutil -a -i off || true export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':') fi From f50115bea71ac8c406f27b09038085044fe4ff83 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 15:40:04 +1000 Subject: [PATCH 03/10] PR builds: build independent recipes in parallel (rattler-build 0.77) rattler-build 0.77's --parallel-builds builds outputs whose dependencies are ready side by side; on Linux and macOS they share a make/ninja jobserver with the runner's cores, so they don't oversubscribe it. Windows has no jobserver: two builds at a time. A PR's builds run in one rattler-build call per platform, so most of their per-recipe time (environment installs, configure, packaging) can overlap. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 3 ++ pixi.lock | 68 +++++++++++++++++------------------ pixi.toml | 2 +- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index c221df5e6..50b9a2df5 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -251,6 +251,9 @@ jobs: extra=() # shorter work dirs for MAX_PATH on Windows [ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id) + # independent recipes side by side; on Linux and macOS a shared jobserver keeps + # them at the runner's cores in total (Windows has none: two at a time there) + if [ "$PLATFORM" == "win-64" ]; then extra+=(--parallel-builds 2); else extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)"); fi pixi run rs "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}" 2>&1 | tee rattler-build.log status=${PIPESTATUS[0]} echo "exit-code=${status}" >> "$GITHUB_OUTPUT" diff --git a/pixi.lock b/pixi.lock index 27f58885f..6bada540b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -70,7 +70,7 @@ environments: - conda: https://prefix.dev/conda-forge/linux-64/patchelf-0.19.2-hee9eb32_0.conda - conda: https://prefix.dev/conda-forge/linux-64/python-3.14.7-hcd007b5_106_cp314.conda - conda: https://prefix.dev/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda - - conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.76.1-hf01adef_0.conda + - conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda - conda: https://prefix.dev/conda-forge/linux-64/rattler-index-0.31.5-h10a7753_0.conda - conda: https://prefix.dev/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda - conda: https://prefix.dev/conda-forge/linux-64/rhash-1.4.6-hb03c661_2.conda @@ -150,7 +150,7 @@ environments: - conda: https://prefix.dev/conda-forge/linux-aarch64/patchelf-0.19.2-hfb11796_0.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/python-3.14.7-hbec3b18_106_cp314.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/pyyaml-6.0.3-py314h807365f_1.conda - - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.76.1-hc342849_0.conda + - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.77.0-hc342849_1.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-index-0.31.5-hf75afee_0.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/readline-8.3-ha7194a6_1.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/rhash-1.4.6-he30d5cf_2.conda @@ -236,7 +236,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/openssl-3.6.5-h332eb6d_0.conda - conda: https://prefix.dev/conda-forge/osx-64/python-3.14.7-hafa0b4b_106_cp314.conda - conda: https://prefix.dev/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda - - conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.76.1-h1311731_0.conda + - conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda - conda: https://prefix.dev/conda-forge/osx-64/rattler-index-0.31.5-hca43645_0.conda - conda: https://prefix.dev/conda-forge/osx-64/readline-8.3-h000c2f7_1.conda - conda: https://prefix.dev/conda-forge/osx-64/rhash-1.4.6-ha1e9b39_2.conda @@ -309,7 +309,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/openssl-3.6.5-h55eecbc_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/python-3.14.7-h0ae1c2c_106_cp314.conda - conda: https://prefix.dev/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda - - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.76.1-h74d5219_0.conda + - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-index-0.31.5-h7b803a5_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda - conda: https://prefix.dev/conda-forge/osx-arm64/rhash-1.4.6-h84a0fba_2.conda @@ -380,7 +380,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/openssl-3.6.5-hf411b9b_0.conda - conda: https://prefix.dev/conda-forge/win-64/python-3.14.7-h53f6dd8_106_cp314.conda - conda: https://prefix.dev/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda - - conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.76.1-he94b42d_0.conda + - conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda - conda: https://prefix.dev/conda-forge/win-64/rattler-index-0.31.5-h91801bb_0.conda - conda: https://prefix.dev/conda-forge/win-64/tk-8.6.13-h967ab96_4.conda - conda: https://prefix.dev/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda @@ -935,23 +935,23 @@ packages: run_exports: {} size: 202391 timestamp: 1770223462836 -- conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.76.1-hf01adef_0.conda - sha256: d9476ee6237db525d4f2bb49e67f4e68719644a6adb754994118192e8d3cc920 - md5: a721700e8181fb5e4c1b371987f7c0eb +- conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda + sha256: 7bcadeeda659a37783003d43a157252c23826d6e81e1df9c0777cb61f9996f39 + md5: 1b82a8513a170c17d1eaea225ea79420 depends: - patchelf - libstdcxx >=15 - libgcc >=15 - __glibc >=2.17,<3.0.a0 - - openssl >=3.5.8,<4.0a0 + - openssl >=3.5.9,<4.0a0 constrains: - __glibc >=2.17 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19561706 - timestamp: 1789389044876 + size: 21828116 + timestamp: 1792604777652 - conda: https://prefix.dev/conda-forge/linux-64/rattler-index-0.31.5-h10a7753_0.conda sha256: 09cc437c855086dacc5eb079e0ef4e803ad1481a01e2d1888a61aee0997acc4a md5: a7095c222125caa80758999f633e1fca @@ -1529,22 +1529,22 @@ packages: run_exports: {} size: 195678 timestamp: 1770223441816 -- conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.76.1-hc342849_0.conda - sha256: 45a7837ba1b33d2b8ab4f66cf396a985b9d49e25a8f221bdc5e4452a0059abf8 - md5: 2b315913f52f71b0b0cafe30c1c5c2df +- conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.77.0-hc342849_1.conda + sha256: b106b6e73c4694acb3db3c3829991baadd48f871b0617b4e0e04027aad54bc6a + md5: ebeae39e3de9508dda7997ec0abe98ec depends: - patchelf - - libstdcxx >=15 - libgcc >=15 - - openssl >=3.5.8,<4.0a0 + - libstdcxx >=15 + - openssl >=3.5.9,<4.0a0 constrains: - __glibc >=2.17 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19054533 - timestamp: 1789389033349 + size: 21544350 + timestamp: 1792683029542 - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-index-0.31.5-hf75afee_0.conda sha256: 332321ce14ae5c2d471db5f4b3a13f3c70360dbcee294a398512de058c79f54d md5: bb4f8d531aaefb38859eeb5277aac2a3 @@ -2222,20 +2222,20 @@ packages: run_exports: {} size: 191947 timestamp: 1770226344240 -- conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.76.1-h1311731_0.conda - sha256: 1e1c90835f13bc2225502e1d18bb124b73fcbf80d8c339932496a47e9acb75df - md5: 5a5b03d4e292a0dd303f190dd9fc909b +- conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda + sha256: 73e2612701c4d2ef4d77cfde2c322e45511a0b6039d68bcb256ccd1f21891c34 + md5: 754bcf356018f9f1b16a96376cfc5336 depends: - - libcxx >=21 - __osx >=11.0 + - libcxx >=21 constrains: - __osx >=11.0 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19075633 - timestamp: 1789389091418 + size: 21461377 + timestamp: 1792604950742 - conda: https://prefix.dev/conda-forge/osx-64/rattler-index-0.31.5-hca43645_0.conda sha256: ee6542a6bd763f58cc1e6d3b8b70816b1aa20f80d8f7d4d75bd195ff94c76f40 md5: 34482f1edda229ad12ea95e261c26caf @@ -2709,9 +2709,9 @@ packages: run_exports: {} size: 189475 timestamp: 1770223788648 -- conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.76.1-h74d5219_0.conda - sha256: 7da95b859c8f04edcd64077719f0fbe891244cd688e1e4176e205d318d09535c - md5: db2e01dca41265c66b5a3c2d65849172 +- conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda + sha256: 1905e563b8ea0ed5d750a1aa879c42def85b7a4ce4bfa0d131e9dbb1688b8029 + md5: f83ca3594af92932486aa219ff2d1635 depends: - libcxx >=21 - __osx >=11.0 @@ -2721,8 +2721,8 @@ packages: license_family: BSD purls: [] run_exports: {} - size: 17540271 - timestamp: 1789389037150 + size: 19719448 + timestamp: 1792604807546 - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-index-0.31.5-h7b803a5_0.conda sha256: 090e1707a06de52be1e61083ef0495aa3bc9c335cf65c80af0f67d4ca2be5477 md5: d1305520d9d027244137eb688f67a45d @@ -3160,9 +3160,9 @@ packages: run_exports: {} size: 181257 timestamp: 1770223460931 -- conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.76.1-he94b42d_0.conda - sha256: 8825b67ae50f887334350f0c81a0818b4cf551e1aca00a947d1fd74105732ff6 - md5: 2b0a2e175aa4e3239c3c91c86f32d70c +- conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda + sha256: 5e62743cf0648a451a604b8097862e41791fa10da90ef4ed46198ef2559ff4b3 + md5: a33b0a9d59da14acd79d2264b9f67f49 depends: - vc >=14.3,<15 - vc14_runtime >=14.44.35208 @@ -3171,8 +3171,8 @@ packages: license_family: BSD purls: [] run_exports: {} - size: 20119294 - timestamp: 1789389029287 + size: 22565961 + timestamp: 1792604776870 - conda: https://prefix.dev/conda-forge/win-64/rattler-index-0.31.5-h91801bb_0.conda sha256: c3ff1eca8e17fab8aae4cb7161a8a0c9ea46cfa6af508a92b6a5e3d2b66f4ea3 md5: 33e5d80562641bba6d5b4aec4c0d2209 diff --git a/pixi.toml b/pixi.toml index 8db3dde99..daffb3981 100644 --- a/pixi.toml +++ b/pixi.toml @@ -17,7 +17,7 @@ PYTHONIOENCODING = "utf-8" [dependencies] python = ">=3.14.0,<3.15" -rattler-build = ">=0.76.1,<0.77" +rattler-build = ">=0.77.0,<0.78" cmake = "<4.0" setuptools = "<82.0" rattler-index = ">=0.27.16" From 5855c6abf7d909ab4b492919381925a011fbd559 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 15:42:12 +1000 Subject: [PATCH 04/10] PR build estimate: from the measured build times ci_estimate.py counted every package to build at one flat per-platform average from jazzy's October rebuild (19 s on linux-64, 69 s on win-64), so rebuilding rtabmap was estimated like a message package. It now adds up each recipe's measured time (build_timings.timings(), the same times vinca packs the build jobs by); the flat averages stay for platforms without times and for the packages restored from the cache. Co-Authored-By: Claude Opus 5.5 --- tools/build_timings.py | 20 ++++++++++++++------ tools/ci_estimate.py | 16 ++++++++++++---- tools/tests/test_build_timings.py | 23 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/tools/build_timings.py b/tools/build_timings.py index c39c70c04..3018fcacf 100644 --- a/tools/build_timings.py +++ b/tools/build_timings.py @@ -87,18 +87,26 @@ def dumps(recipes: dict) -> str: return "{\n" + ",\n".join(lines) + "\n}\n" -def vinca_timings(distro: str, platform: str, path: Path) -> bool: - """Write vinca-gha's --timings file for the platform: each recipe's average over the - distributions, and ALIAS_SECONDS for the distribution's compatibility recipes - (ros--, built right after ros2-). False when there are no times yet.""" +def timings(distro: str, platform: str) -> dict | None: + """{"recipes": {recipe: seconds}, "default": seconds} for the platform: each recipe's + average over the distributions, and ALIAS_SECONDS for the distribution's compatibility + recipes (ros--, built right after ros2-). None without times yet.""" source = TIMINGS / f"{platform}.json" if not source.is_file(): - return False + return None recipes = {r: round(statistics.fmean(by.values()), 1) for r, by in json.loads(source.read_text()).items()} # unmeasured recipes: the median of the packages (ros2-*) default = statistics.median([s for r, s in recipes.items() if r.startswith("ros2-")] or list(recipes.values())) recipes.update({f"ros-{distro}-{r[len('ros2-'):]}": ALIAS_SECONDS for r in list(recipes) if r.startswith("ros2-")}) - path.write_text(json.dumps({"recipes": recipes, "default": default}) + "\n") + return {"recipes": recipes, "default": default} + + +def vinca_timings(distro: str, platform: str, path: Path) -> bool: + """Write vinca-gha's --timings file for the platform (timings()); False when there are + no times yet.""" + if (data := timings(distro, platform)) is None: + return False + path.write_text(json.dumps(data) + "\n") return True diff --git a/tools/ci_estimate.py b/tools/ci_estimate.py index 1257abfb4..218280688 100644 --- a/tools/ci_estimate.py +++ b/tools/ci_estimate.py @@ -3,9 +3,10 @@ python tools/ci_estimate.py --recipes distros//work/recipes --cache-dir --platform

Recipes whose package is in the restored cache are skipped by rattler-build; the rest -are built. The seconds per package below are rough averages from the jazzy full -rebuild in October 2026 (wall clock of the build step, including environment setup): -building one package, and skipping one that is already in the cache. +are built, each taking its measured time (shared/build_timings, see build_timings.py). +The seconds per package below are rough averages from the jazzy full rebuild in October +2026 (wall clock of the build step, including environment setup): building one package, +for platforms without measured times, and skipping one that is already in the cache. """ import argparse @@ -14,6 +15,8 @@ import sys from pathlib import Path +import build_timings + SECONDS = { # platform: (per built package, per skipped package) "linux-64": (19, 2), "linux-aarch64": (21, 3), @@ -45,7 +48,12 @@ def main() -> int: to_build = sorted(recipes - cached) skipped = len(recipes & cached) build, skip = SECONDS.get(args.platform, max(SECONDS.values())) - seconds = len(to_build) * build + skipped * skip + distro = args.recipes.resolve().parent.parent.name # distros//work/recipes + measured = build_timings.timings(distro, args.platform) + if measured: + seconds = sum(measured["recipes"].get(r, measured["default"]) for r in to_build) + skipped * skip + else: + seconds = len(to_build) * build + skipped * skip hours = seconds / 3600 rounds = int(hours // JOB_LIMIT_HOURS) + 1 done = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=seconds) diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index aed3bb47f..313536213 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -97,3 +97,26 @@ def fake_run_tee(cmd, cwd): assert calls[-1][calls[-1].index("--timings") + 1] == "build_timings.json" assert json.loads((work / "build_timings.json").read_text()) == { "recipes": {"ros2-a": 3.0, "ros-jazzy-a": build_timings.ALIAS_SECONDS}, "default": 3.0} + + +def test_ci_estimate_uses_the_measured_times(tmp_path, monkeypatch, capsys): + import ci_estimate + + timings = tmp_path / "timings" + timings.mkdir() + (timings / "linux-64.json").write_text(build_timings.dumps( + {"ros2-a": {"jazzy": 100.0, "lyrical": 200.0}, "ros2-b": {"jazzy": 10.0}})) + monkeypatch.setattr(build_timings, "TIMINGS", timings) + recipes = tmp_path / "distros" / "jazzy" / "work" / "recipes" + for r in ("ros2-a", "ros-jazzy-a", "ros2-b", "ros2-new"): + (recipes / r).mkdir(parents=True) + (recipes / r / "recipe.yaml").write_text("") + cache = tmp_path / "cache" + cache.mkdir() + (cache / "ros2-b-1.0-h0_1.conda").write_text("") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", "") + monkeypatch.setattr("sys.argv", ["ci_estimate.py", "--recipes", str(recipes), "--cache-dir", str(cache), + "--platform", "linux-64"]) + assert ci_estimate.main() == 0 + # ros2-a 150 s (average), its alias 0.3 s, ros2-new the median 80 s, ros2-b cached 2 s + assert capsys.readouterr().out.startswith("3 packages to build, 1 restored from the cache: roughly 4 min") From a80a26f94c04c2c24fc0dc9cf0da45c5706de7dd Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 15:55:34 +1000 Subject: [PATCH 05/10] Windows: builds, package cache and temporary files on D: GitHub's Windows runners have a faster second drive (D:, about 14k IOPS against C:'s 4.5k, chadgolden1/gh-runner-benchmarks); uv uses it for the same reason. The package cache moves with the builds so that rattler keeps hardlinking packages into the environments. Prints both drives' free space. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 10 +++++++--- .scripts/build_win.bat | 10 +++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index 50b9a2df5..1ea48cab1 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -156,7 +156,7 @@ jobs: # while uploads go to a test channel, packages already there aren't built again ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) - CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'C:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} + CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'D:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} steps: - uses: actions/checkout@v6 with: @@ -180,8 +180,12 @@ jobs: # Defender's real-time scan slows down installing and deleting the environments' # files several times over; the runner is thrown away after the job Set-MpPreference -DisableRealtimeMonitoring $true - "CONDA_BLD_PATH=C:\bld\" | Out-File -FilePath $env:GITHUB_ENV -Append - New-Item -ItemType Directory -Force -Path C:\bld | Out-Null + # the builds, rattler's package cache (on the same drive, for hardlinks) and the + # temporary files on D:, about three times as fast as C: on GitHub's runners + "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | + Out-File -FilePath $env:GITHUB_ENV -Append + New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null + Get-PSDrive C, D | Format-Table Name, @{n="free GB"; e={[math]::Round($_.Free / 1GB)}} New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index c5e66d6f8..4bd69e4bd 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -5,7 +5,15 @@ setlocal EnableExtensions EnableDelayedExpansion :: files several times over; the runner is thrown away after the job powershell -NoProfile -Command "Set-MpPreference -DisableRealtimeMonitoring $true" -set CONDA_BLD_PATH=C:\bld +:: The builds, rattler's package cache (on the same drive, for hardlinks) and the +:: temporary files on D:, about three times as fast as C: on GitHub's runners +set CONDA_BLD_PATH=D:\bld +set VINCA_CUSTOM_CMAKE_BUILD_DIR=D:\x\ +set RATTLER_CACHE_DIR=D:\rattler +set TMP=D:\tmp +set TEMP=D:\tmp +mkdir D:\tmp +powershell -NoProfile -Command "Get-PSDrive C, D | Format-Table Name, @{n='free GB'; e={[math]::Round($_.Free / 1GB)}}" echo "PATH is %PATH%" rmdir /Q/S C:\Strawberry\ From 34188d6a3395b88fff0b6b1077137f0fa033b0ae Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 17:25:53 +1000 Subject: [PATCH 06/10] Build branches: each job's recipes in one rattler-build run, in parallel build_unix.sh / build_win.bat call build-ci once with all of the job's recipes instead of once per recipe. build-ci then adds --parallel-builds (3 builds sharing a jobserver with the runner's cores on Linux and macOS, 2 on Windows) and --log-style json: parallel builds interleave their output, so it prints each line as "HH:MM:SS [recipe] message", and records each recipe's time from its first to its last line in its build span for packing the jobs. Locally (osx-arm64), the 20 test recipes of #154 build in 169 s. Co-Authored-By: Claude Opus 5.5 --- .scripts/build_unix.sh | 10 +++--- .scripts/build_win.bat | 16 ++++----- tools/robostack.py | 60 +++++++++++++++++++++++++++---- tools/tests/test_build_timings.py | 32 +++++++++++++++++ 4 files changed, 96 insertions(+), 22 deletions(-) diff --git a/.scripts/build_unix.sh b/.scripts/build_unix.sh index 2ef476aa8..12c548dac 100755 --- a/.scripts/build_unix.sh +++ b/.scripts/build_unix.sh @@ -30,13 +30,13 @@ if [[ "$target" == *"osx"* ]]; then export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':') fi +# All of the job's recipes in one build-ci run (tools/robostack.py), which builds +# independent ones in parallel and adds the variant config and the distro's channels. +recipe_args=() for recipe in ${CURRENT_RECIPES[@]}; do - # build-ci (tools/robostack.py) adds the variant config and the distro's channels. - pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci \ - --recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \ - --output-dir $CONDA_BLD_PATH - + recipe_args+=(--recipe "${FEEDSTOCK_ROOT}/recipes/${recipe}") done +pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci "${recipe_args[@]}" --output-dir $CONDA_BLD_PATH # Check if it build something, this is a hotfix for the skips inside additional_recipes shopt -s nullglob diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index 4bd69e4bd..97b29739d 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -30,16 +30,12 @@ reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t :: ... and in git, for sources that vendor packages clone during the build git config --global core.longpaths true -for %%X in (%CURRENT_RECIPES%) do ( - echo "BUILDING RECIPE %%X" - cd %FEEDSTOCK_ROOT%\recipes\%%X\ - rem build-ci (tools/robostack.py) adds the variant config and the distro's channels. - pixi run -v rs %ROBOSTACK_DISTRO% build-ci --recipe %FEEDSTOCK_ROOT%\recipes\%%X\ ^ - --output-dir %CONDA_BLD_PATH% - - if errorlevel 1 exit 1 - rem -m %FEEDSTOCK_ROOT%\.ci_support\conda_forge_pinnings.yaml -) +:: All of the job's recipes in one build-ci run (tools/robostack.py), which builds +:: independent ones in parallel and adds the variant config and the distro's channels. +set RECIPE_ARGS= +for %%X in (%CURRENT_RECIPES%) do set "RECIPE_ARGS=!RECIPE_ARGS! --recipe %FEEDSTOCK_ROOT%\recipes\%%X\" +pixi run -v rs %ROBOSTACK_DISTRO% build-ci !RECIPE_ARGS! --output-dir %CONDA_BLD_PATH% +if errorlevel 1 exit 1 :: Check if .conda files exist in the win-64 directory if exist "%CONDA_BLD_PATH%\win-64\*.conda" ( diff --git a/tools/robostack.py b/tools/robostack.py index 2f79bf2ac..938a596c8 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -112,19 +112,63 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: NETWORK_WAIT = 60 # doubled on each retry: 1, 2, 4, 8 minutes -def run_build(cmd: list[str], cwd: Path) -> int: +# a build job's recipes side by side; on Linux and macOS a shared jobserver keeps them at +# the runner's cores in total (Windows has none) +PARALLEL_BUILDS = "2" if sys.platform == "win32" else f"3:{os.cpu_count()}" + + +def build_ci_parallel(distro: str, recipes: list[str], args: list[str], cwd: Path) -> int: + """build-ci for all of a build job's recipes in one rattler-build run, building + independent ones in parallel. Its JSON log tells which recipe each line belongs to: + printed as "HH:MM:SS [recipe] message", and each recipe's time (first to last line) + is recorded for packing the build jobs.""" + span: dict[str, list[str]] = {} # recipe -> [first, last] timestamp + + def show(line: bytes) -> None: + try: + entry = json.loads(line) + spans = entry.get("spans", []) + recipe = next((s["recipe"] for s in spans if "recipe" in s), "") + name = recipe.rsplit("-", 2)[0] # the output's name-version-build + # the build itself: rattler-build renders every recipe up front + if name and any(s.get("name") == "Running build for" for s in spans): + span.setdefault(name, [entry["timestamp"]] * 2)[1] = entry["timestamp"] + text = f"{entry['timestamp'][11:19]} [{name or '-'}] {entry['fields'].get('message', '')}\n" + sys.stdout.write(text) + except (ValueError, KeyError, AttributeError, TypeError): + sys.stdout.buffer.write(line) + sys.stdout.flush() + + args = [*args, "--parallel-builds", PARALLEL_BUILDS, "--log-style", "json"] + rc = run_build(rattler_build(distro, args, skip_existing=False), cwd, show) + if rc == 0 and os.environ.get("GITHUB_ACTIONS"): + import build_timings # for packing the build jobs by time + from datetime import datetime + + for recipe in recipes: + if recipe in span: + first, last = (datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[recipe]) + build_timings.note(distro, recipe, (last - first).total_seconds()) + return rc + + +def run_build(cmd: list[str], cwd: Path, show=None) -> int: """run() for builds: retry when the solver misses a package, waiting for the channel's index to catch up with the previous stage's uploads (at most INDEX_LAG_RETRIES times), or when the runner's network failed (at most - NETWORK_RETRIES times, waiting longer each time).""" + NETWORK_RETRIES times, waiting longer each time). show(line) prints a line instead of + writing it as is.""" index_retries = network_retries = 0 while True: print("+", " ".join(cmd), f" (in {cwd.relative_to(ROOT)})", flush=True) missing = network = False with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: for line in proc.stdout: - sys.stdout.buffer.write(line) - sys.stdout.buffer.flush() + if show: + show(line) + else: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() missing = missing or any(m in line.lower() for m in INDEX_LAG_MESSAGES) network = network or any(m in line for m in NETWORK_MESSAGES) if proc.returncode == 0: @@ -427,13 +471,15 @@ def task(distro: str, name: str, args: list[str]) -> int: if name == "build": return run_build(rattler_build(distro, ["--recipe-dir", "./recipes", *args]), w) if name == "build-ci": # used by .scripts/build_unix.sh / build_win.bat + recipes = [Path(args[i + 1].rstrip("\\/")).name for i, a in enumerate(args[:-1]) if a == "--recipe"] + if len(recipes) > 1: + return build_ci_parallel(distro, recipes, args, w) started = time.monotonic() rc = run_build(rattler_build(distro, args, skip_existing=False), w) - if rc == 0 and "--recipe" in args[:-1] and os.environ.get("GITHUB_ACTIONS"): + if rc == 0 and recipes and os.environ.get("GITHUB_ACTIONS"): import build_timings # for packing the build jobs by time - recipe = Path(args[args.index("--recipe") + 1].rstrip("\\/")).name - build_timings.note(distro, recipe, time.monotonic() - started) + build_timings.note(distro, recipes[0], time.monotonic() - started) return rc if name == "build-one": pkg = args[0] if args else "ros2-ros-workspace" diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index 313536213..6843a6c85 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -120,3 +120,35 @@ def test_ci_estimate_uses_the_measured_times(tmp_path, monkeypatch, capsys): assert ci_estimate.main() == 0 # ros2-a 150 s (average), its alias 0.3 s, ros2-new the median 80 s, ros2-b cached 2 s assert capsys.readouterr().out.startswith("3 packages to build, 1 restored from the cache: roughly 4 min") + + +def test_build_ci_parallel_times_each_build(tmp_path, monkeypatch, capsys): + import robostack + + def line(t, recipe, message, span="Running build for"): + spans = [{"recipe": recipe, "name": span}] if recipe else [] + return (json.dumps({"timestamp": f"2026-10-24T07:00:{t:02d}.0Z", "fields": {"message": message}, + "spans": spans}) + "\n").encode() + + log = [line(0, "", "rattler-build 0.77.0"), line(1, "ros2-a-1.0-h0_1", "Fetching source"), + line(5, "ros2-b-1.0-h0_1", "Fetching source"), line(9, "ros2-a-1.0-h0_1", "Artifact"), + line(20, "ros2-b-1.0-h0_1", "Artifact"), b"not json\n", + line(30, "ros2-a-1.0-h0_1", "summary", span="Build summary for")] + shown = [] + + def run_build(cmd, cwd, show=None): + assert "--parallel-builds" in cmd and "json" in cmd + for entry in log: + show(entry) + return 0 + + noted = [] + monkeypatch.setattr(robostack, "run_build", run_build) + monkeypatch.setattr(robostack, "rattler_build", lambda distro, args, skip_existing=True: ["rattler-build", *args]) + monkeypatch.setattr(build_timings, "note", lambda distro, recipe, seconds: noted.append((recipe, seconds))) + monkeypatch.setenv("GITHUB_ACTIONS", "true") + assert robostack.build_ci_parallel("jazzy", ["ros2-a", "ros2-b"], ["--recipe", "a", "--recipe", "b"], tmp_path) == 0 + # from each recipe's first to its last line in the build, not the summary at the end + assert noted == [("ros2-a", 8.0), ("ros2-b", 15.0)] + out = capsys.readouterr().out + assert "07:00:01 [ros2-a] Fetching source" in out and "not json" in out From 2c49c6415affffc970e554f515a427ac41a8433e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 17:29:29 +1000 Subject: [PATCH 07/10] build-ci: parallel builds' times scaled to add up to their wall time A recipe's time measured while others build alongside is longer than its share of the job. Scaling all of them by wall time / their sum gives each recipe its share, which is what packing the jobs needs (and what the serial times were). Co-Authored-By: Claude Opus 5.5 --- tools/robostack.py | 15 +++++++++------ tools/tests/test_build_timings.py | 5 +++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/robostack.py b/tools/robostack.py index 938a596c8..0254fe31c 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -120,8 +120,9 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: def build_ci_parallel(distro: str, recipes: list[str], args: list[str], cwd: Path) -> int: """build-ci for all of a build job's recipes in one rattler-build run, building independent ones in parallel. Its JSON log tells which recipe each line belongs to: - printed as "HH:MM:SS [recipe] message", and each recipe's time (first to last line) - is recorded for packing the build jobs.""" + printed as "HH:MM:SS [recipe] message", and each recipe's time (first to last line of + its build) is recorded for packing the build jobs, scaled so that the times add up to + the builds' wall time: overlapping builds share it.""" span: dict[str, list[str]] = {} # recipe -> [first, last] timestamp def show(line: bytes) -> None: @@ -145,10 +146,12 @@ def show(line: bytes) -> None: import build_timings # for packing the build jobs by time from datetime import datetime - for recipe in recipes: - if recipe in span: - first, last = (datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[recipe]) - build_timings.note(distro, recipe, (last - first).total_seconds()) + times = {r: [datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[r]] for r in recipes if r in span} + measured = {r: (last - first).total_seconds() for r, (first, last) in times.items()} + if times and sum(measured.values()): + wall = (max(last for _, last in times.values()) - min(first for first, _ in times.values())).total_seconds() + for recipe, seconds in measured.items(): + build_timings.note(distro, recipe, seconds * wall / sum(measured.values())) return rc diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index 6843a6c85..f7070cf58 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -148,7 +148,8 @@ def run_build(cmd, cwd, show=None): monkeypatch.setattr(build_timings, "note", lambda distro, recipe, seconds: noted.append((recipe, seconds))) monkeypatch.setenv("GITHUB_ACTIONS", "true") assert robostack.build_ci_parallel("jazzy", ["ros2-a", "ros2-b"], ["--recipe", "a", "--recipe", "b"], tmp_path) == 0 - # from each recipe's first to its last line in the build, not the summary at the end - assert noted == [("ros2-a", 8.0), ("ros2-b", 15.0)] + # from each recipe's first to its last line in the build (8 s, 15 s), not the summary at + # the end, scaled to add up to the builds' wall time (19 s): overlapping builds share it + assert [(r, round(s, 2)) for r, s in noted] == [("ros2-a", 6.61), ("ros2-b", 12.39)] out = capsys.readouterr().out assert "07:00:01 [ros2-a] Fetching source" in out and "not json" in out From 807a3ac96af63138c6c4e292f28d5bff2be47987 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 17:31:00 +1000 Subject: [PATCH 08/10] build-ci: no times for the compatibility recipes of parallel builds They only repackage but, building alongside, mostly wait for the others: their measured times inflated the sum that the packages' times are scaled by. The timing files never keep them (ALIAS_SECONDS), so leave them out. Co-Authored-By: Claude Opus 5.5 --- tools/robostack.py | 7 +++++-- tools/tests/test_build_timings.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/robostack.py b/tools/robostack.py index 0254fe31c..ee769ddb2 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -122,7 +122,9 @@ def build_ci_parallel(distro: str, recipes: list[str], args: list[str], cwd: Pat independent ones in parallel. Its JSON log tells which recipe each line belongs to: printed as "HH:MM:SS [recipe] message", and each recipe's time (first to last line of its build) is recorded for packing the build jobs, scaled so that the times add up to - the builds' wall time: overlapping builds share it.""" + the builds' wall time: overlapping builds share it. The compatibility recipes + (ros--) only repackage, but wait for others while building alongside: + they are left out (build_timings gives them ALIAS_SECONDS).""" span: dict[str, list[str]] = {} # recipe -> [first, last] timestamp def show(line: bytes) -> None: @@ -146,7 +148,8 @@ def show(line: bytes) -> None: import build_timings # for packing the build jobs by time from datetime import datetime - times = {r: [datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[r]] for r in recipes if r in span} + times = {r: [datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[r]] + for r in recipes if r in span and not r.startswith(f"ros-{distro}-")} measured = {r: (last - first).total_seconds() for r, (first, last) in times.items()} if times and sum(measured.values()): wall = (max(last for _, last in times.values()) - min(first for first, _ in times.values())).total_seconds() diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index f7070cf58..9ae50042a 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -132,7 +132,8 @@ def line(t, recipe, message, span="Running build for"): log = [line(0, "", "rattler-build 0.77.0"), line(1, "ros2-a-1.0-h0_1", "Fetching source"), line(5, "ros2-b-1.0-h0_1", "Fetching source"), line(9, "ros2-a-1.0-h0_1", "Artifact"), - line(20, "ros2-b-1.0-h0_1", "Artifact"), b"not json\n", + line(20, "ros2-b-1.0-h0_1", "Artifact"), line(21, "ros-jazzy-a-1.0-h0_1", "Fetching source"), + line(40, "ros-jazzy-a-1.0-h0_1", "Artifact"), b"not json\n", line(30, "ros2-a-1.0-h0_1", "summary", span="Build summary for")] shown = [] @@ -147,9 +148,11 @@ def run_build(cmd, cwd, show=None): monkeypatch.setattr(robostack, "rattler_build", lambda distro, args, skip_existing=True: ["rattler-build", *args]) monkeypatch.setattr(build_timings, "note", lambda distro, recipe, seconds: noted.append((recipe, seconds))) monkeypatch.setenv("GITHUB_ACTIONS", "true") - assert robostack.build_ci_parallel("jazzy", ["ros2-a", "ros2-b"], ["--recipe", "a", "--recipe", "b"], tmp_path) == 0 + assert robostack.build_ci_parallel("jazzy", ["ros2-a", "ros2-b", "ros-jazzy-a"], + ["--recipe", "a", "--recipe", "b", "--recipe", "c"], tmp_path) == 0 # from each recipe's first to its last line in the build (8 s, 15 s), not the summary at - # the end, scaled to add up to the builds' wall time (19 s): overlapping builds share it + # the end, scaled to add up to the builds' wall time (19 s): overlapping builds share it; + # not the compatibility recipe assert [(r, round(s, 2)) for r, s in noted] == [("ros2-a", 6.61), ("ros2-b", 12.39)] out = capsys.readouterr().out assert "07:00:01 [ros2-a] Fetching source" in out and "not json" in out From aca5a187552067bdabb44188be88bf0c9335d674 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 18:11:55 +1000 Subject: [PATCH 09/10] Windows: one build at a time Two parallel builds were no faster on Windows (#154: 305 s against 282 s for one at a time, with the builds on D:), and there is no jobserver to keep them from oversubscribing the cores. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 6 +++--- tools/robostack.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index 1ea48cab1..1db94a66f 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -255,9 +255,9 @@ jobs: extra=() # shorter work dirs for MAX_PATH on Windows [ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id) - # independent recipes side by side; on Linux and macOS a shared jobserver keeps - # them at the runner's cores in total (Windows has none: two at a time there) - if [ "$PLATFORM" == "win-64" ]; then extra+=(--parallel-builds 2); else extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)"); fi + # independent recipes side by side, with a shared jobserver keeping them at the + # runner's cores in total; Windows has no jobserver, and was no faster with two + [ "$PLATFORM" != "win-64" ] && extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)") pixi run rs "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}" 2>&1 | tee rattler-build.log status=${PIPESTATUS[0]} echo "exit-code=${status}" >> "$GITHUB_OUTPUT" diff --git a/tools/robostack.py b/tools/robostack.py index ee769ddb2..c3886768b 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -112,9 +112,9 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: NETWORK_WAIT = 60 # doubled on each retry: 1, 2, 4, 8 minutes -# a build job's recipes side by side; on Linux and macOS a shared jobserver keeps them at -# the runner's cores in total (Windows has none) -PARALLEL_BUILDS = "2" if sys.platform == "win32" else f"3:{os.cpu_count()}" +# a build job's recipes side by side, with a shared jobserver keeping them at the runner's +# cores in total; Windows has no jobserver, and was no faster with two (#154) +PARALLEL_BUILDS = "1" if sys.platform == "win32" else f"3:{os.cpu_count()}" def build_ci_parallel(distro: str, recipes: list[str], args: list[str], cwd: Path) -> int: From 94cd474ae29f0e381d062317be4b50810d16f30d Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 24 Oct 2026 20:32:13 +1000 Subject: [PATCH 10/10] Windows PR builds: pixi's package cache on D: too Co-Authored-By: Claude Opus 5.5 --- .github/workflows/testpr.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index 1db94a66f..e95c096f8 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -167,6 +167,11 @@ jobs: # (e.g. vinca's), which breaks every later run restoring that cache - name: Disable git auto-maintenance run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true + # pixi's package cache on D:, with the builds' (see the Windows step below) + - name: "pixi's cache on D:" + if: runner.os == 'Windows' + shell: pwsh + run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' - uses: prefix-dev/setup-pixi@v0.11.0 with: locked: true # the lock file must match pixi.toml