diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index bcdfe794b..e95c096f8 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -156,7 +156,7 @@ jobs: # while uploads go to a test channel, packages already there aren't built again ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} # rattler-build writes to distros//work/output, or to CONDA_BLD_PATH (win-64) - CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'C:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} + CACHE_DIR: ${{ matrix.target.platform == 'win-64' && 'D:/bld/win-64' || format('distros/{0}/work/output/{1}', matrix.distro, matrix.target.platform) }} steps: - uses: actions/checkout@v6 with: @@ -167,6 +167,11 @@ jobs: # (e.g. vinca's), which breaks every later run restoring that cache - name: Disable git auto-maintenance run: git config --global maintenance.auto false && git config --global gc.auto 0 && git config --global core.longpaths true + # pixi's package cache on D:, with the builds' (see the Windows step below) + - name: "pixi's cache on D:" + if: runner.os == 'Windows' + shell: pwsh + run: '"PIXI_CACHE_DIR=D:\pixi-cache" | Out-File -FilePath $env:GITHUB_ENV -Append' - uses: prefix-dev/setup-pixi@v0.11.0 with: locked: true # the lock file must match pixi.toml @@ -177,12 +182,25 @@ jobs: if: runner.os == 'Windows' shell: pwsh run: | - "CONDA_BLD_PATH=C:\bld\" | Out-File -FilePath $env:GITHUB_ENV -Append - New-Item -ItemType Directory -Force -Path C:\bld | Out-Null + # Defender's real-time scan slows down installing and deleting the environments' + # files several times over; the runner is thrown away after the job + Set-MpPreference -DisableRealtimeMonitoring $true + # the builds, rattler's package cache (on the same drive, for hardlinks) and the + # temporary files on D:, about three times as fast as C: on GitHub's runners + "CONDA_BLD_PATH=D:\bld\", "RATTLER_CACHE_DIR=D:\rattler", "TMP=D:\tmp", "TEMP=D:\tmp" | + Out-File -FilePath $env:GITHUB_ENV -Append + New-Item -ItemType Directory -Force -Path D:\bld, D:\rattler, D:\tmp | Out-Null + Get-PSDrive C, D | Format-Table Name, @{n="free GB"; e={[math]::Round($_.Free / 1GB)}} New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force Remove-Item -Recurse -Force C:\Strawberry -ErrorAction SilentlyContinue Remove-Item -Recurse -Force "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0" -ErrorAction SilentlyContinue + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + - name: Turn off Spotlight + if: runner.os == 'macOS' + run: sudo mdutil -a -i off || true + - name: Clean up PATH if: runner.os == 'Windows' uses: egor-tensin/cleanup-path@v5 @@ -242,6 +260,9 @@ jobs: extra=() # shorter work dirs for MAX_PATH on Windows [ "$PLATFORM" == "win-64" ] && extra+=(--no-build-id) + # independent recipes side by side, with a shared jobserver keeping them at the + # runner's cores in total; Windows has no jobserver, and was no faster with two + [ "$PLATFORM" != "win-64" ] && extra+=(--parallel-builds "3:$(getconf _NPROCESSORS_ONLN)") pixi run rs "$DISTRO" build --target-platform "$PLATFORM" "${extra[@]}" 2>&1 | tee rattler-build.log status=${PIPESTATUS[0]} echo "exit-code=${status}" >> "$GITHUB_OUTPUT" diff --git a/.scripts/build_unix.sh b/.scripts/build_unix.sh index 39d855bd4..12c548dac 100755 --- a/.scripts/build_unix.sh +++ b/.scripts/build_unix.sh @@ -24,16 +24,19 @@ export "CONDA_BLD_PATH=$HOME/conda-bld/" if [[ "$target" == *"osx"* ]]; then echo "osx" + # Spotlight indexes every file the environments and builds write, using up to a + # core or two of the runner's three or four; the runner is thrown away after the job + sudo mdutil -a -i off || true export PATH=$(echo $PATH | tr ":" "\n" | grep -v 'homebrew' | xargs | tr ' ' ':') fi +# All of the job's recipes in one build-ci run (tools/robostack.py), which builds +# independent ones in parallel and adds the variant config and the distro's channels. +recipe_args=() for recipe in ${CURRENT_RECIPES[@]}; do - # build-ci (tools/robostack.py) adds the variant config and the distro's channels. - pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci \ - --recipe ${FEEDSTOCK_ROOT}/recipes/${recipe} \ - --output-dir $CONDA_BLD_PATH - + recipe_args+=(--recipe "${FEEDSTOCK_ROOT}/recipes/${recipe}") done +pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci "${recipe_args[@]}" --output-dir $CONDA_BLD_PATH # Check if it build something, this is a hotfix for the skips inside additional_recipes shopt -s nullglob diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index a6c29e123..97b29739d 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -1,7 +1,19 @@ setlocal EnableExtensions EnableDelayedExpansion -set CONDA_BLD_PATH=C:\bld +:: Defender's real-time scan slows down installing and deleting the environments' +:: files several times over; the runner is thrown away after the job +powershell -NoProfile -Command "Set-MpPreference -DisableRealtimeMonitoring $true" + +:: The builds, rattler's package cache (on the same drive, for hardlinks) and the +:: temporary files on D:, about three times as fast as C: on GitHub's runners +set CONDA_BLD_PATH=D:\bld +set VINCA_CUSTOM_CMAKE_BUILD_DIR=D:\x\ +set RATTLER_CACHE_DIR=D:\rattler +set TMP=D:\tmp +set TEMP=D:\tmp +mkdir D:\tmp +powershell -NoProfile -Command "Get-PSDrive C, D | Format-Table Name, @{n='free GB'; e={[math]::Round($_.Free / 1GB)}}" echo "PATH is %PATH%" rmdir /Q/S C:\Strawberry\ @@ -18,16 +30,12 @@ reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t :: ... and in git, for sources that vendor packages clone during the build git config --global core.longpaths true -for %%X in (%CURRENT_RECIPES%) do ( - echo "BUILDING RECIPE %%X" - cd %FEEDSTOCK_ROOT%\recipes\%%X\ - rem build-ci (tools/robostack.py) adds the variant config and the distro's channels. - pixi run -v rs %ROBOSTACK_DISTRO% build-ci --recipe %FEEDSTOCK_ROOT%\recipes\%%X\ ^ - --output-dir %CONDA_BLD_PATH% - - if errorlevel 1 exit 1 - rem -m %FEEDSTOCK_ROOT%\.ci_support\conda_forge_pinnings.yaml -) +:: All of the job's recipes in one build-ci run (tools/robostack.py), which builds +:: independent ones in parallel and adds the variant config and the distro's channels. +set RECIPE_ARGS= +for %%X in (%CURRENT_RECIPES%) do set "RECIPE_ARGS=!RECIPE_ARGS! --recipe %FEEDSTOCK_ROOT%\recipes\%%X\" +pixi run -v rs %ROBOSTACK_DISTRO% build-ci !RECIPE_ARGS! --output-dir %CONDA_BLD_PATH% +if errorlevel 1 exit 1 :: Check if .conda files exist in the win-64 directory if exist "%CONDA_BLD_PATH%\win-64\*.conda" ( diff --git a/pixi.lock b/pixi.lock index 27f58885f..6bada540b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -70,7 +70,7 @@ environments: - conda: https://prefix.dev/conda-forge/linux-64/patchelf-0.19.2-hee9eb32_0.conda - conda: https://prefix.dev/conda-forge/linux-64/python-3.14.7-hcd007b5_106_cp314.conda - conda: https://prefix.dev/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda - - conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.76.1-hf01adef_0.conda + - conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda - conda: https://prefix.dev/conda-forge/linux-64/rattler-index-0.31.5-h10a7753_0.conda - conda: https://prefix.dev/conda-forge/linux-64/readline-8.3-hd6e31c0_1.conda - conda: https://prefix.dev/conda-forge/linux-64/rhash-1.4.6-hb03c661_2.conda @@ -150,7 +150,7 @@ environments: - conda: https://prefix.dev/conda-forge/linux-aarch64/patchelf-0.19.2-hfb11796_0.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/python-3.14.7-hbec3b18_106_cp314.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/pyyaml-6.0.3-py314h807365f_1.conda - - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.76.1-hc342849_0.conda + - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.77.0-hc342849_1.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-index-0.31.5-hf75afee_0.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/readline-8.3-ha7194a6_1.conda - conda: https://prefix.dev/conda-forge/linux-aarch64/rhash-1.4.6-he30d5cf_2.conda @@ -236,7 +236,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/openssl-3.6.5-h332eb6d_0.conda - conda: https://prefix.dev/conda-forge/osx-64/python-3.14.7-hafa0b4b_106_cp314.conda - conda: https://prefix.dev/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda - - conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.76.1-h1311731_0.conda + - conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda - conda: https://prefix.dev/conda-forge/osx-64/rattler-index-0.31.5-hca43645_0.conda - conda: https://prefix.dev/conda-forge/osx-64/readline-8.3-h000c2f7_1.conda - conda: https://prefix.dev/conda-forge/osx-64/rhash-1.4.6-ha1e9b39_2.conda @@ -309,7 +309,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/openssl-3.6.5-h55eecbc_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/python-3.14.7-h0ae1c2c_106_cp314.conda - conda: https://prefix.dev/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda - - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.76.1-h74d5219_0.conda + - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-index-0.31.5-h7b803a5_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/readline-8.3-h8b90a29_1.conda - conda: https://prefix.dev/conda-forge/osx-arm64/rhash-1.4.6-h84a0fba_2.conda @@ -380,7 +380,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/openssl-3.6.5-hf411b9b_0.conda - conda: https://prefix.dev/conda-forge/win-64/python-3.14.7-h53f6dd8_106_cp314.conda - conda: https://prefix.dev/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda - - conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.76.1-he94b42d_0.conda + - conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda - conda: https://prefix.dev/conda-forge/win-64/rattler-index-0.31.5-h91801bb_0.conda - conda: https://prefix.dev/conda-forge/win-64/tk-8.6.13-h967ab96_4.conda - conda: https://prefix.dev/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda @@ -935,23 +935,23 @@ packages: run_exports: {} size: 202391 timestamp: 1770223462836 -- conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.76.1-hf01adef_0.conda - sha256: d9476ee6237db525d4f2bb49e67f4e68719644a6adb754994118192e8d3cc920 - md5: a721700e8181fb5e4c1b371987f7c0eb +- conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda + sha256: 7bcadeeda659a37783003d43a157252c23826d6e81e1df9c0777cb61f9996f39 + md5: 1b82a8513a170c17d1eaea225ea79420 depends: - patchelf - libstdcxx >=15 - libgcc >=15 - __glibc >=2.17,<3.0.a0 - - openssl >=3.5.8,<4.0a0 + - openssl >=3.5.9,<4.0a0 constrains: - __glibc >=2.17 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19561706 - timestamp: 1789389044876 + size: 21828116 + timestamp: 1792604777652 - conda: https://prefix.dev/conda-forge/linux-64/rattler-index-0.31.5-h10a7753_0.conda sha256: 09cc437c855086dacc5eb079e0ef4e803ad1481a01e2d1888a61aee0997acc4a md5: a7095c222125caa80758999f633e1fca @@ -1529,22 +1529,22 @@ packages: run_exports: {} size: 195678 timestamp: 1770223441816 -- conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.76.1-hc342849_0.conda - sha256: 45a7837ba1b33d2b8ab4f66cf396a985b9d49e25a8f221bdc5e4452a0059abf8 - md5: 2b315913f52f71b0b0cafe30c1c5c2df +- conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-build-0.77.0-hc342849_1.conda + sha256: b106b6e73c4694acb3db3c3829991baadd48f871b0617b4e0e04027aad54bc6a + md5: ebeae39e3de9508dda7997ec0abe98ec depends: - patchelf - - libstdcxx >=15 - libgcc >=15 - - openssl >=3.5.8,<4.0a0 + - libstdcxx >=15 + - openssl >=3.5.9,<4.0a0 constrains: - __glibc >=2.17 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19054533 - timestamp: 1789389033349 + size: 21544350 + timestamp: 1792683029542 - conda: https://prefix.dev/conda-forge/linux-aarch64/rattler-index-0.31.5-hf75afee_0.conda sha256: 332321ce14ae5c2d471db5f4b3a13f3c70360dbcee294a398512de058c79f54d md5: bb4f8d531aaefb38859eeb5277aac2a3 @@ -2222,20 +2222,20 @@ packages: run_exports: {} size: 191947 timestamp: 1770226344240 -- conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.76.1-h1311731_0.conda - sha256: 1e1c90835f13bc2225502e1d18bb124b73fcbf80d8c339932496a47e9acb75df - md5: 5a5b03d4e292a0dd303f190dd9fc909b +- conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda + sha256: 73e2612701c4d2ef4d77cfde2c322e45511a0b6039d68bcb256ccd1f21891c34 + md5: 754bcf356018f9f1b16a96376cfc5336 depends: - - libcxx >=21 - __osx >=11.0 + - libcxx >=21 constrains: - __osx >=11.0 license: BSD-3-Clause license_family: BSD purls: [] run_exports: {} - size: 19075633 - timestamp: 1789389091418 + size: 21461377 + timestamp: 1792604950742 - conda: https://prefix.dev/conda-forge/osx-64/rattler-index-0.31.5-hca43645_0.conda sha256: ee6542a6bd763f58cc1e6d3b8b70816b1aa20f80d8f7d4d75bd195ff94c76f40 md5: 34482f1edda229ad12ea95e261c26caf @@ -2709,9 +2709,9 @@ packages: run_exports: {} size: 189475 timestamp: 1770223788648 -- conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.76.1-h74d5219_0.conda - sha256: 7da95b859c8f04edcd64077719f0fbe891244cd688e1e4176e205d318d09535c - md5: db2e01dca41265c66b5a3c2d65849172 +- conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda + sha256: 1905e563b8ea0ed5d750a1aa879c42def85b7a4ce4bfa0d131e9dbb1688b8029 + md5: f83ca3594af92932486aa219ff2d1635 depends: - libcxx >=21 - __osx >=11.0 @@ -2721,8 +2721,8 @@ packages: license_family: BSD purls: [] run_exports: {} - size: 17540271 - timestamp: 1789389037150 + size: 19719448 + timestamp: 1792604807546 - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-index-0.31.5-h7b803a5_0.conda sha256: 090e1707a06de52be1e61083ef0495aa3bc9c335cf65c80af0f67d4ca2be5477 md5: d1305520d9d027244137eb688f67a45d @@ -3160,9 +3160,9 @@ packages: run_exports: {} size: 181257 timestamp: 1770223460931 -- conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.76.1-he94b42d_0.conda - sha256: 8825b67ae50f887334350f0c81a0818b4cf551e1aca00a947d1fd74105732ff6 - md5: 2b0a2e175aa4e3239c3c91c86f32d70c +- conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda + sha256: 5e62743cf0648a451a604b8097862e41791fa10da90ef4ed46198ef2559ff4b3 + md5: a33b0a9d59da14acd79d2264b9f67f49 depends: - vc >=14.3,<15 - vc14_runtime >=14.44.35208 @@ -3171,8 +3171,8 @@ packages: license_family: BSD purls: [] run_exports: {} - size: 20119294 - timestamp: 1789389029287 + size: 22565961 + timestamp: 1792604776870 - conda: https://prefix.dev/conda-forge/win-64/rattler-index-0.31.5-h91801bb_0.conda sha256: c3ff1eca8e17fab8aae4cb7161a8a0c9ea46cfa6af508a92b6a5e3d2b66f4ea3 md5: 33e5d80562641bba6d5b4aec4c0d2209 diff --git a/pixi.toml b/pixi.toml index 8db3dde99..daffb3981 100644 --- a/pixi.toml +++ b/pixi.toml @@ -17,7 +17,7 @@ PYTHONIOENCODING = "utf-8" [dependencies] python = ">=3.14.0,<3.15" -rattler-build = ">=0.76.1,<0.77" +rattler-build = ">=0.77.0,<0.78" cmake = "<4.0" setuptools = "<82.0" rattler-index = ">=0.27.16" diff --git a/tools/build_timings.py b/tools/build_timings.py index c39c70c04..3018fcacf 100644 --- a/tools/build_timings.py +++ b/tools/build_timings.py @@ -87,18 +87,26 @@ def dumps(recipes: dict) -> str: return "{\n" + ",\n".join(lines) + "\n}\n" -def vinca_timings(distro: str, platform: str, path: Path) -> bool: - """Write vinca-gha's --timings file for the platform: each recipe's average over the - distributions, and ALIAS_SECONDS for the distribution's compatibility recipes - (ros--, built right after ros2-). False when there are no times yet.""" +def timings(distro: str, platform: str) -> dict | None: + """{"recipes": {recipe: seconds}, "default": seconds} for the platform: each recipe's + average over the distributions, and ALIAS_SECONDS for the distribution's compatibility + recipes (ros--, built right after ros2-). None without times yet.""" source = TIMINGS / f"{platform}.json" if not source.is_file(): - return False + return None recipes = {r: round(statistics.fmean(by.values()), 1) for r, by in json.loads(source.read_text()).items()} # unmeasured recipes: the median of the packages (ros2-*) default = statistics.median([s for r, s in recipes.items() if r.startswith("ros2-")] or list(recipes.values())) recipes.update({f"ros-{distro}-{r[len('ros2-'):]}": ALIAS_SECONDS for r in list(recipes) if r.startswith("ros2-")}) - path.write_text(json.dumps({"recipes": recipes, "default": default}) + "\n") + return {"recipes": recipes, "default": default} + + +def vinca_timings(distro: str, platform: str, path: Path) -> bool: + """Write vinca-gha's --timings file for the platform (timings()); False when there are + no times yet.""" + if (data := timings(distro, platform)) is None: + return False + path.write_text(json.dumps(data) + "\n") return True diff --git a/tools/ci_estimate.py b/tools/ci_estimate.py index 1257abfb4..218280688 100644 --- a/tools/ci_estimate.py +++ b/tools/ci_estimate.py @@ -3,9 +3,10 @@ python tools/ci_estimate.py --recipes distros//work/recipes --cache-dir --platform

Recipes whose package is in the restored cache are skipped by rattler-build; the rest -are built. The seconds per package below are rough averages from the jazzy full -rebuild in October 2026 (wall clock of the build step, including environment setup): -building one package, and skipping one that is already in the cache. +are built, each taking its measured time (shared/build_timings, see build_timings.py). +The seconds per package below are rough averages from the jazzy full rebuild in October +2026 (wall clock of the build step, including environment setup): building one package, +for platforms without measured times, and skipping one that is already in the cache. """ import argparse @@ -14,6 +15,8 @@ import sys from pathlib import Path +import build_timings + SECONDS = { # platform: (per built package, per skipped package) "linux-64": (19, 2), "linux-aarch64": (21, 3), @@ -45,7 +48,12 @@ def main() -> int: to_build = sorted(recipes - cached) skipped = len(recipes & cached) build, skip = SECONDS.get(args.platform, max(SECONDS.values())) - seconds = len(to_build) * build + skipped * skip + distro = args.recipes.resolve().parent.parent.name # distros//work/recipes + measured = build_timings.timings(distro, args.platform) + if measured: + seconds = sum(measured["recipes"].get(r, measured["default"]) for r in to_build) + skipped * skip + else: + seconds = len(to_build) * build + skipped * skip hours = seconds / 3600 rounds = int(hours // JOB_LIMIT_HOURS) + 1 done = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=seconds) diff --git a/tools/robostack.py b/tools/robostack.py index 2f79bf2ac..c3886768b 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -112,19 +112,69 @@ def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: NETWORK_WAIT = 60 # doubled on each retry: 1, 2, 4, 8 minutes -def run_build(cmd: list[str], cwd: Path) -> int: +# a build job's recipes side by side, with a shared jobserver keeping them at the runner's +# cores in total; Windows has no jobserver, and was no faster with two (#154) +PARALLEL_BUILDS = "1" if sys.platform == "win32" else f"3:{os.cpu_count()}" + + +def build_ci_parallel(distro: str, recipes: list[str], args: list[str], cwd: Path) -> int: + """build-ci for all of a build job's recipes in one rattler-build run, building + independent ones in parallel. Its JSON log tells which recipe each line belongs to: + printed as "HH:MM:SS [recipe] message", and each recipe's time (first to last line of + its build) is recorded for packing the build jobs, scaled so that the times add up to + the builds' wall time: overlapping builds share it. The compatibility recipes + (ros--) only repackage, but wait for others while building alongside: + they are left out (build_timings gives them ALIAS_SECONDS).""" + span: dict[str, list[str]] = {} # recipe -> [first, last] timestamp + + def show(line: bytes) -> None: + try: + entry = json.loads(line) + spans = entry.get("spans", []) + recipe = next((s["recipe"] for s in spans if "recipe" in s), "") + name = recipe.rsplit("-", 2)[0] # the output's name-version-build + # the build itself: rattler-build renders every recipe up front + if name and any(s.get("name") == "Running build for" for s in spans): + span.setdefault(name, [entry["timestamp"]] * 2)[1] = entry["timestamp"] + text = f"{entry['timestamp'][11:19]} [{name or '-'}] {entry['fields'].get('message', '')}\n" + sys.stdout.write(text) + except (ValueError, KeyError, AttributeError, TypeError): + sys.stdout.buffer.write(line) + sys.stdout.flush() + + args = [*args, "--parallel-builds", PARALLEL_BUILDS, "--log-style", "json"] + rc = run_build(rattler_build(distro, args, skip_existing=False), cwd, show) + if rc == 0 and os.environ.get("GITHUB_ACTIONS"): + import build_timings # for packing the build jobs by time + from datetime import datetime + + times = {r: [datetime.fromisoformat(t.replace("Z", "+00:00")) for t in span[r]] + for r in recipes if r in span and not r.startswith(f"ros-{distro}-")} + measured = {r: (last - first).total_seconds() for r, (first, last) in times.items()} + if times and sum(measured.values()): + wall = (max(last for _, last in times.values()) - min(first for first, _ in times.values())).total_seconds() + for recipe, seconds in measured.items(): + build_timings.note(distro, recipe, seconds * wall / sum(measured.values())) + return rc + + +def run_build(cmd: list[str], cwd: Path, show=None) -> int: """run() for builds: retry when the solver misses a package, waiting for the channel's index to catch up with the previous stage's uploads (at most INDEX_LAG_RETRIES times), or when the runner's network failed (at most - NETWORK_RETRIES times, waiting longer each time).""" + NETWORK_RETRIES times, waiting longer each time). show(line) prints a line instead of + writing it as is.""" index_retries = network_retries = 0 while True: print("+", " ".join(cmd), f" (in {cwd.relative_to(ROOT)})", flush=True) missing = network = False with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: for line in proc.stdout: - sys.stdout.buffer.write(line) - sys.stdout.buffer.flush() + if show: + show(line) + else: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() missing = missing or any(m in line.lower() for m in INDEX_LAG_MESSAGES) network = network or any(m in line for m in NETWORK_MESSAGES) if proc.returncode == 0: @@ -427,13 +477,15 @@ def task(distro: str, name: str, args: list[str]) -> int: if name == "build": return run_build(rattler_build(distro, ["--recipe-dir", "./recipes", *args]), w) if name == "build-ci": # used by .scripts/build_unix.sh / build_win.bat + recipes = [Path(args[i + 1].rstrip("\\/")).name for i, a in enumerate(args[:-1]) if a == "--recipe"] + if len(recipes) > 1: + return build_ci_parallel(distro, recipes, args, w) started = time.monotonic() rc = run_build(rattler_build(distro, args, skip_existing=False), w) - if rc == 0 and "--recipe" in args[:-1] and os.environ.get("GITHUB_ACTIONS"): + if rc == 0 and recipes and os.environ.get("GITHUB_ACTIONS"): import build_timings # for packing the build jobs by time - recipe = Path(args[args.index("--recipe") + 1].rstrip("\\/")).name - build_timings.note(distro, recipe, time.monotonic() - started) + build_timings.note(distro, recipes[0], time.monotonic() - started) return rc if name == "build-one": pkg = args[0] if args else "ros2-ros-workspace" diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index aed3bb47f..9ae50042a 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -97,3 +97,62 @@ def fake_run_tee(cmd, cwd): assert calls[-1][calls[-1].index("--timings") + 1] == "build_timings.json" assert json.loads((work / "build_timings.json").read_text()) == { "recipes": {"ros2-a": 3.0, "ros-jazzy-a": build_timings.ALIAS_SECONDS}, "default": 3.0} + + +def test_ci_estimate_uses_the_measured_times(tmp_path, monkeypatch, capsys): + import ci_estimate + + timings = tmp_path / "timings" + timings.mkdir() + (timings / "linux-64.json").write_text(build_timings.dumps( + {"ros2-a": {"jazzy": 100.0, "lyrical": 200.0}, "ros2-b": {"jazzy": 10.0}})) + monkeypatch.setattr(build_timings, "TIMINGS", timings) + recipes = tmp_path / "distros" / "jazzy" / "work" / "recipes" + for r in ("ros2-a", "ros-jazzy-a", "ros2-b", "ros2-new"): + (recipes / r).mkdir(parents=True) + (recipes / r / "recipe.yaml").write_text("") + cache = tmp_path / "cache" + cache.mkdir() + (cache / "ros2-b-1.0-h0_1.conda").write_text("") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", "") + monkeypatch.setattr("sys.argv", ["ci_estimate.py", "--recipes", str(recipes), "--cache-dir", str(cache), + "--platform", "linux-64"]) + assert ci_estimate.main() == 0 + # ros2-a 150 s (average), its alias 0.3 s, ros2-new the median 80 s, ros2-b cached 2 s + assert capsys.readouterr().out.startswith("3 packages to build, 1 restored from the cache: roughly 4 min") + + +def test_build_ci_parallel_times_each_build(tmp_path, monkeypatch, capsys): + import robostack + + def line(t, recipe, message, span="Running build for"): + spans = [{"recipe": recipe, "name": span}] if recipe else [] + return (json.dumps({"timestamp": f"2026-10-24T07:00:{t:02d}.0Z", "fields": {"message": message}, + "spans": spans}) + "\n").encode() + + log = [line(0, "", "rattler-build 0.77.0"), line(1, "ros2-a-1.0-h0_1", "Fetching source"), + line(5, "ros2-b-1.0-h0_1", "Fetching source"), line(9, "ros2-a-1.0-h0_1", "Artifact"), + line(20, "ros2-b-1.0-h0_1", "Artifact"), line(21, "ros-jazzy-a-1.0-h0_1", "Fetching source"), + line(40, "ros-jazzy-a-1.0-h0_1", "Artifact"), b"not json\n", + line(30, "ros2-a-1.0-h0_1", "summary", span="Build summary for")] + shown = [] + + def run_build(cmd, cwd, show=None): + assert "--parallel-builds" in cmd and "json" in cmd + for entry in log: + show(entry) + return 0 + + noted = [] + monkeypatch.setattr(robostack, "run_build", run_build) + monkeypatch.setattr(robostack, "rattler_build", lambda distro, args, skip_existing=True: ["rattler-build", *args]) + monkeypatch.setattr(build_timings, "note", lambda distro, recipe, seconds: noted.append((recipe, seconds))) + monkeypatch.setenv("GITHUB_ACTIONS", "true") + assert robostack.build_ci_parallel("jazzy", ["ros2-a", "ros2-b", "ros-jazzy-a"], + ["--recipe", "a", "--recipe", "b", "--recipe", "c"], tmp_path) == 0 + # from each recipe's first to its last line in the build (8 s, 15 s), not the summary at + # the end, scaled to add up to the builds' wall time (19 s): overlapping builds share it; + # not the compatibility recipe + assert [(r, round(s, 2)) for r, s in noted] == [("ros2-a", 6.61), ("ros2-b", 12.39)] + out = capsys.readouterr().out + assert "07:00:01 [ros2-a] Fetching source" in out and "not json" in out