diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml deleted file mode 100644 index c950bf7e0..000000000 --- a/.github/actions/setup-sccache/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Set up sccache -description: Install sccache and enable the GitHub Actions cache backend for Rust builds. - -runs: - using: composite - steps: - - name: sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - - name: Configure sccache for Rust - shell: bash - run: | - echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - # Write the cache only on main. Other refs read it, since their - # entries serve only that ref and evict the ones main shares. - if [ "$GITHUB_REF" != "refs/heads/main" ]; then - echo "SCCACHE_GHA_RW_MODE=READ_ONLY" >> "$GITHUB_ENV" - fi diff --git a/.github/workflows/CargoAudit.yml b/.github/workflows/CargoAudit.yml index 9150ad99a..e2b2ab4a2 100644 --- a/.github/workflows/CargoAudit.yml +++ b/.github/workflows/CargoAudit.yml @@ -20,6 +20,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: toolchain: "1.94" + cache: false - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 with: diff --git a/.github/workflows/Coverage.yml b/.github/workflows/Coverage.yml index 39a77f110..96690328d 100644 --- a/.github/workflows/Coverage.yml +++ b/.github/workflows/Coverage.yml @@ -44,18 +44,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - shared-key: "${{ runner.os }}-debug" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Set up sccache - uses: ./.github/actions/setup-sccache - - name: Build guest binaries run: just guests diff --git a/.github/workflows/PrimeCaches.yml b/.github/workflows/PrimeCaches.yml index 46ac63ff2..c4762ba5e 100644 --- a/.github/workflows/PrimeCaches.yml +++ b/.github/workflows/PrimeCaches.yml @@ -1,13 +1,16 @@ # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json -# Builds on main to populate the GitHub Actions cache for all Swatinem/rust-cache -# shared-keys used by PR workflows. Caches saved on `refs/heads/main` are -# restorable from every PR via the base-branch fallback. PR workflows are -# configured with `save-if: github.ref == 'refs/heads/main'`, so they only ever -# read from this cache. +# The only workflow that writes the Rust caches pull requests restore. Entries +# saved on main are restorable from every pull request. Every other workflow +# only reads them. # -# No tests are run here. The goal is just to populate dependency build artifacts -# so that PR jobs start warm. +# Each job looks up the entry for its current lockfiles and toolchain. If it +# is missing, the job builds from a cold target and saves. rust-cache keeps +# every dependency version it restores, so a build on a restored entry would +# grow it with each lockfile change. Tests build but do not run. +# +# Keys hash lockfiles and toolchains, not build commands. Bump `prefix-key` in +# every rust-cache step after changing what a job here builds. name: Prime Caches @@ -16,6 +19,12 @@ on: branches: [main] workflow_dispatch: +# A new run replaces a pending one and never cancels the running one, so the +# newest commit always saves. +concurrency: + group: prime-caches-${{ github.ref }} + cancel-in-progress: false + env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full @@ -40,17 +49,31 @@ jobs: with: arch: ${{ matrix.arch }} config: ${{ matrix.config }} + prime_cache: true - # Populate v0-rust-code-checks-{linux,windows} caches. + # Populate v1-rust-code-checks-{linux,windows} caches. code-checks: uses: ./.github/workflows/dep_code_checks.yml secrets: inherit + with: + prime_cache: true + + # Populate the fuzz cache (key fuzz-{arch}) with every fuzz target. + fuzz: + uses: ./.github/workflows/dep_fuzzing.yml + secrets: inherit + with: + prime_cache: true + arch: X64 + target: all + max_total_time: 0 + timeout_minutes: 30 # Populate the host caches (key {arch}-{os}-{hypervisor}-{config}) used by # dep_build_test.yml. Linux is primed once per hypervisor because the kvm # pools run Ubuntu and the mshv3 pools run Azure Linux 3, and we want each - # distro to restore a cache produced under its own libc. Only the host - # dependency graph is built here, no tests are run. + # distro to restore a cache produced under its own libc. The host, its tests + # and hyperlight-ci are built here. No tests run. host: strategy: fail-fast: false @@ -102,17 +125,44 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Keep the steps before this one in sync with dep_build_test.yml. The key + # hashes the installed toolchains and the Rust and C build env vars. - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + id: rust-cache + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "${{ matrix.arch }}-${{ runner.os }}-${{ matrix.hypervisor }}-${{ matrix.config }}" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} + lookup-only: true - - name: Set up sccache - uses: ./.github/actions/setup-sccache + - name: Build host and tests + if: steps.rust-cache.outputs.cache-hit != 'true' + run: just prime-caches ${{ matrix.config }} - - name: Build host - run: just build ${{ matrix.config }} + # Rust keys end in a toolchain hash and a lockfile hash. Sysroot keys end in + # one hash. Each change leaves the old entry behind. Keep the newest entry + # per key and delete the rest to stay under the repository cache quota. + cleanup: + needs: [guests, code-checks, fuzz, host] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + actions: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Delete superseded caches + run: | + # Filter by key prefix. The listing caps at 1000 entries. + for prefix in v1-rust- sysroot-; do + gh cache list --ref "$GITHUB_REF" --key "$prefix" --limit 1000 --json id,key,createdAt + done | jq -s -r ' + add + | group_by(.key | if startswith("sysroot-") then sub("-[^-]*$"; "") else sub("-[^-]*-[^-]*$"; "") end) + | map(sort_by(.createdAt) | reverse | .[1:][]) + | .[] | "\(.id) \(.key)"' | + while read -r id key; do + echo "Deleting $key" + gh cache delete "$id" || echo "::warning::Failed to delete $key" + done diff --git a/.github/workflows/RustNightly.yml b/.github/workflows/RustNightly.yml index 2cb61c365..031cbb4fc 100644 --- a/.github/workflows/RustNightly.yml +++ b/.github/workflows/RustNightly.yml @@ -50,29 +50,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # cargo-hyperlight builds a custom sysroot for x86_64-hyperlight-none target. - # rust-cache cleans "anything not a dependency" from target dirs, removing the sysroot. - # We cache sysroot separately to avoid rebuilding it (~10s) on every run. - - name: Sysroot cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - src/tests/rust_guests/target/sysroot - src/tests/rust_guests/target-non-pie/sysroot - key: sysroot-linux-${{ matrix.config }}-${{ hashFiles('rust-toolchain.toml') }} - - - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - shared-key: "nightly-${{ matrix.config }}" - cache-on-failure: "true" - workspaces: | - . -> target - src/tests/rust_guests -> target - - - name: Set up sccache - uses: ./.github/actions/setup-sccache - - name: Install cargo-hyperlight run: just ensure-cargo-hyperlight diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index e9bf6f3a8..8b8bbb056 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -27,6 +27,8 @@ jobs: with: toolchain: nightly components: rustfmt + # The agent runs on branches, where saved caches serve only that branch and evict those of main. + cache: false - uses: hyperlight-dev/ci-setup-workflow@2f4142ba17cf573af44fc1e1f1ffc743daded5b3 # v1.10.0 diff --git a/.github/workflows/dep_benchmarks.yml b/.github/workflows/dep_benchmarks.yml index a40ff51ef..7a3fd70ca 100644 --- a/.github/workflows/dep_benchmarks.yml +++ b/.github/workflows/dep_benchmarks.yml @@ -82,14 +82,12 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-release" save-if: "false" - - name: Set up sccache - uses: ./.github/actions/setup-sccache - - name: Download Rust guests uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: @@ -116,6 +114,13 @@ jobs: name: c-guests-${{ inputs.arch }}-release-baseline path: ${{ runner.temp }}/baseline-guests/c/ + # The baseline builds in its own target directory. Cargo fingerprints + # path dependencies by mtime and workspace path, so two checkouts cannot + # share one. A copy of the restored cache gives it the dependencies. + - name: Seed baseline target directory + if: inputs.baseline_ref != '' + run: if [ -d target ]; then cp -a target "$RUNNER_TEMP/baseline-target"; fi + # Both commits are measured on this machine, so the comparison reflects # them rather than the difference between two runners. The branch point # is read off the merge this run validates, so on anything else there is @@ -130,7 +135,7 @@ jobs: echo "::error::HEAD is not a merge, so ${{ inputs.baseline_ref }} is not the branch point" exit 1 } - just bench-against ${{ inputs.baseline_ref }} "$RUNNER_TEMP/baseline-guests" + just bench-against ${{ inputs.baseline_ref }} "$RUNNER_TEMP/baseline-guests" "" "$RUNNER_TEMP/baseline-target" else just bench-ci main fi diff --git a/.github/workflows/dep_build_guests.yml b/.github/workflows/dep_build_guests.yml index d2d6c9976..f6f845a86 100644 --- a/.github/workflows/dep_build_guests.yml +++ b/.github/workflows/dep_build_guests.yml @@ -30,6 +30,13 @@ on: required: false type: string default: "" + prime_cache: + description: >- + Build the Rust guests from a cold target and save the caches. Skips + the build when they are up to date, and skips C guests and uploads. + required: false + type: boolean + default: false env: CARGO_TERM_COLOR: always @@ -87,45 +94,51 @@ jobs: id: sysroot-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - src/tests/rust_guests/target/sysroot - src/tests/rust_guests/target-non-pie/sysroot + path: src/tests/rust_guests/target/sysroot key: sysroot-linux-${{ inputs.arch }}-${{ inputs.config }}-${{ hashFiles('rust-toolchain.toml') }} + # Priming builds cold, so the entry holds only what this commit needs. - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + id: rust-cache + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "guests-${{ inputs.arch }}-${{ inputs.config }}" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} + lookup-only: ${{ inputs.prime_cache }} + save-if: ${{ inputs.prime_cache }} workspaces: | - . -> target src/tests/rust_guests -> target + src/tests/rust_guests -> target-non-pie/build - - name: Set up sccache - uses: ./.github/actions/setup-sccache + - name: Check whether the caches are up to date + id: prime + if: inputs.prime_cache + run: echo "skip=${{ steps.rust-cache.outputs.cache-hit == 'true' && steps.sysroot-cache.outputs.cache-hit == 'true' }}" >> "$GITHUB_OUTPUT" - name: Install cargo-hyperlight + if: steps.prime.outputs.skip != 'true' run: just ensure-cargo-hyperlight - name: Build Rust guests + if: steps.prime.outputs.skip != 'true' run: | just build-rust-guests ${{ inputs.config }} just move-rust-guests ${{ inputs.config }} - name: Build non-PIE Rust guests + if: steps.prime.outputs.skip != 'true' run: | just build-rust-guests-non-pie ${{ inputs.config }} just move-rust-guests-non-pie ${{ inputs.config }} - name: Build C guests + if: ${{ !inputs.prime_cache }} run: | just build-c-guests ${{ inputs.config }} just move-c-guests ${{ inputs.config }} - name: Upload Rust guests + if: ${{ !inputs.prime_cache }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: rust-guests-${{ inputs.arch }}-${{ inputs.config }}${{ inputs.artifact_suffix }} @@ -135,6 +148,7 @@ jobs: if-no-files-found: error - name: Upload C guests + if: ${{ !inputs.prime_cache }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: c-guests-${{ inputs.arch }}-${{ inputs.config }}${{ inputs.artifact_suffix }} @@ -143,13 +157,10 @@ jobs: if-no-files-found: error # A job step, not a post-job step, so the save runs before rust-cache's - # post-job cleanup deletes the sysroot. Only main saves, as caches are not - # shared across branches. + # post-job cleanup deletes the sysroot. - name: Save sysroot cache - if: ${{ github.ref == 'refs/heads/main' && steps.sysroot-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.prime_cache && steps.sysroot-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - src/tests/rust_guests/target/sysroot - src/tests/rust_guests/target-non-pie/sysroot + path: src/tests/rust_guests/target/sysroot key: ${{ steps.sysroot-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/dep_build_test.yml b/.github/workflows/dep_build_test.yml index 12f75fb87..f54c00737 100644 --- a/.github/workflows/dep_build_test.yml +++ b/.github/workflows/dep_build_test.yml @@ -106,18 +106,15 @@ jobs: $llvmPath >> $env:GITHUB_PATH - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust # Concurrent macOS runners share ~/.cargo/bin, so cleanup can remove Cargo from another job. cache-bin: ${{ runner.os != 'macOS' }} shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Set up sccache - uses: ./.github/actions/setup-sccache + # PrimeCaches writes this entry. Keep the steps before this one in + # sync with its host job. + save-if: "false" - name: Download Rust guests uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 diff --git a/.github/workflows/dep_code_checks.yml b/.github/workflows/dep_code_checks.yml index 28d239339..7de48b8f0 100644 --- a/.github/workflows/dep_code_checks.yml +++ b/.github/workflows/dep_code_checks.yml @@ -10,6 +10,13 @@ on: required: false type: string default: "false" + prime_cache: + description: >- + Run the checks from a cold target and save the caches. Skips them + when the caches are up to date. + required: false + type: boolean + default: false env: CARGO_TERM_COLOR: always @@ -52,57 +59,68 @@ jobs: src/tests/rust_guests/target/sysroot key: sysroot-linux-${{ hashFiles('rust-toolchain.toml') }} + # Priming builds cold, so the entry holds only what this commit needs. - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + id: rust-cache + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "code-checks-linux" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} + lookup-only: ${{ inputs.prime_cache }} + save-if: ${{ inputs.prime_cache }} workspaces: | . -> target src/tests/rust_guests -> target - - name: Set up sccache - uses: ./.github/actions/setup-sccache + - name: Check whether the caches are up to date + id: prime + if: inputs.prime_cache + run: echo "skip=${{ steps.rust-cache.outputs.cache-hit == 'true' && steps.sysroot-cache.outputs.cache-hit == 'true' }}" >> "$GITHUB_OUTPUT" - name: Ensure up-to-date Cargo.lock + if: steps.prime.outputs.skip != 'true' run: | cargo fetch --locked cargo fetch --manifest-path src/tests/rust_guests/Cargo.toml --locked - name: fmt + if: steps.prime.outputs.skip != 'true' run: just fmt-check - name: Install cargo-hyperlight + if: steps.prime.outputs.skip != 'true' run: just ensure-cargo-hyperlight - name: clippy exhaustive check (debug) + if: steps.prime.outputs.skip != 'true' run: just clippy-exhaustive debug - name: clippy exhaustive check (release) + if: steps.prime.outputs.skip != 'true' run: just clippy-exhaustive release - name: Verify MSRV + if: steps.prime.outputs.skip != 'true' run: ./dev/verify-msrv.sh hyperlight-host hyperlight-guest hyperlight-guest-bin hyperlight-common - name: Check cargo features compile + if: steps.prime.outputs.skip != 'true' run: just check - name: Check compilation with no default features + if: steps.prime.outputs.skip != 'true' run: | just test-compilation-no-default-features debug just test-compilation-no-default-features release - name: Run loom concurrency tests + if: steps.prime.outputs.skip != 'true' run: just test-loom # A job step, not a post-job step, so the save runs before rust-cache's - # post-job cleanup deletes the sysroot. Only main saves, as caches are not - # shared across branches. + # post-job cleanup deletes the sysroot. - name: Save sysroot cache - if: ${{ github.ref == 'refs/heads/main' && steps.sysroot-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.prime_cache && steps.sysroot-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | @@ -138,58 +156,68 @@ jobs: src/tests/rust_guests/target/sysroot key: sysroot-windows-${{ hashFiles('rust-toolchain.toml') }} + # Priming builds cold, so the entry holds only what this commit needs. - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + id: rust-cache + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "code-checks-windows" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache - save-if: ${{ github.ref == 'refs/heads/main' }} + lookup-only: ${{ inputs.prime_cache }} + save-if: ${{ inputs.prime_cache }} workspaces: | . -> target src/tests/rust_guests -> target - - name: Set up sccache - uses: ./.github/actions/setup-sccache + - name: Check whether the caches are up to date + id: prime + if: inputs.prime_cache + run: echo "skip=${{ steps.rust-cache.outputs.cache-hit == 'true' && steps.sysroot-cache.outputs.cache-hit == 'true' }}" >> "$GITHUB_OUTPUT" - name: Ensure up-to-date Cargo.lock + if: steps.prime.outputs.skip != 'true' run: | cargo fetch --locked cargo fetch --manifest-path src/tests/rust_guests/Cargo.toml --locked - name: fmt + if: steps.prime.outputs.skip != 'true' run: just fmt-check - name: Install cargo-hyperlight + if: steps.prime.outputs.skip != 'true' run: just ensure-cargo-hyperlight - name: clippy (debug) + if: steps.prime.outputs.skip != 'true' run: | just clippy debug just clippy-guests debug - name: clippy (release) + if: steps.prime.outputs.skip != 'true' run: | just clippy release just clippy-guests release - name: Verify MSRV + if: steps.prime.outputs.skip != 'true' run: ./dev/verify-msrv.sh hyperlight-host hyperlight-guest hyperlight-guest-bin hyperlight-common - name: Check cargo features compile + if: steps.prime.outputs.skip != 'true' run: just check - name: Check compilation with no default features + if: steps.prime.outputs.skip != 'true' run: | just test-compilation-no-default-features debug just test-compilation-no-default-features release # A job step, not a post-job step, so the save runs before rust-cache's - # post-job cleanup deletes the sysroot. Only main saves, as caches are not - # shared across branches. + # post-job cleanup deletes the sysroot. - name: Save sysroot cache - if: ${{ github.ref == 'refs/heads/main' && steps.sysroot-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.prime_cache && steps.sysroot-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | diff --git a/.github/workflows/dep_fuzzing.yml b/.github/workflows/dep_fuzzing.yml index fda29a1cd..cb361d37a 100644 --- a/.github/workflows/dep_fuzzing.yml +++ b/.github/workflows/dep_fuzzing.yml @@ -29,6 +29,13 @@ on: required: false type: string default: "false" + prime_cache: + description: >- + Build every fuzz target from a cold target and save the cache, without + fuzzing. Skips the build when the cache is up to date. + required: false + type: boolean + default: false permissions: contents: read @@ -57,18 +64,42 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Set up sccache - uses: ./.github/actions/setup-sccache + # rust-cache hashes installed toolchains, so the pinned nightly must be + # installed before the key is computed. + - name: Install fuzzing toolchain + run: rustup toolchain install "$(just --evaluate nightly-toolchain)" --profile minimal + + # One entry serves every target. Priming builds cold, so the entry holds + # only what this commit needs. + - name: Rust cache + id: rust-cache + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + prefix-key: v1-rust + shared-key: "fuzz-${{ inputs.arch }}" + lookup-only: ${{ inputs.prime_cache }} + save-if: ${{ inputs.prime_cache }} + + - name: Check whether the cache is up to date + id: prime + if: inputs.prime_cache + run: echo "skip=${{ steps.rust-cache.outputs.cache-hit == 'true' }}" >> "$GITHUB_OUTPUT" - name: Download Rust guests + if: ${{ !inputs.prime_cache }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: rust-guests-${{ inputs.arch }}-release path: src/tests/rust_guests/bin/release/ - name: Install cargo-fuzz + if: steps.prime.outputs.skip != 'true' run: command -v cargo-fuzz >/dev/null 2>&1 || cargo install cargo-fuzz + - name: Build fuzz targets + if: inputs.prime_cache && steps.prime.outputs.skip != 'true' + run: just build-fuzz-targets + # cargo-fuzz resolves the fuzz project by walking up from the working # directory, so the corpus lives at /fuzz/corpus/. # Runners are single-use, so without this each run starts from an empty @@ -83,6 +114,7 @@ jobs: fuzz-corpus-${{ inputs.target }}- - name: Run Fuzzing + if: ${{ !inputs.prime_cache }} run: just fuzz-timed ${{ inputs.target }} ${{ inputs.max_total_time }} > /dev/null working-directory: src/hyperlight_host diff --git a/.github/workflows/dep_miri.yml b/.github/workflows/dep_miri.yml index 858e5b6ed..8eebe6cae 100644 --- a/.github/workflows/dep_miri.yml +++ b/.github/workflows/dep_miri.yml @@ -59,14 +59,13 @@ jobs: tool: just,cargo-nextest - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust shared-key: "${{ inputs.arch }}-Linux-miri" - cache-on-failure: "true" - # Only save on main as caches are not shared across branches. - save-if: ${{ github.ref == 'refs/heads/main' }} + save-if: "false" # Miri reads no guest binaries: it covers hyperlight-common and the # shared memory tests, neither of which loads one. - name: Run Miri tests - run: env -u RUSTC_WRAPPER just miri-tests + run: just miri-tests diff --git a/.github/workflows/dep_run_examples.yml b/.github/workflows/dep_run_examples.yml index 03ea8b2ee..2e427ea9c 100644 --- a/.github/workflows/dep_run_examples.yml +++ b/.github/workflows/dep_run_examples.yml @@ -69,16 +69,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Rust cache - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: + prefix-key: v1-rust # Concurrent macOS runners share ~/.cargo/bin, so cleanup can remove Cargo from another job. cache-bin: ${{ runner.os != 'macOS' }} shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}" save-if: "false" - - name: Set up sccache - uses: ./.github/actions/setup-sccache - - name: Download Rust guests uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: diff --git a/Justfile b/Justfile index 0ccfb4c2f..fbb2b83ce 100644 --- a/Justfile +++ b/Justfile @@ -76,12 +76,10 @@ build-and-move-rust-guests: (build-rust-guests "debug") (move-rust-guests "debug build-and-move-c-guests: (build-c-guests "debug") (move-c-guests "debug") (build-c-guests "release") (move-c-guests "release") # Build non-PIE variants of rust guests for testing ELF VA mapping. -# Phase 1 builds the sysroot without RUSTFLAGS (avoids RUSTFLAGS leaking -# into the sysroot wrapper build in cargo-hyperlight). -# Phase 2 uses plain cargo with --sysroot and non-PIE link flags. -build-rust-guests-non-pie target=default-target: (ensure-cargo-hyperlight) - cd src/tests/rust_guests/simpleguest && cargo hyperlight build --target-dir ../target-non-pie --profile={{ if target == "debug" { "dev" } else { target } }} - {{ if os() == "windows" { "$env:RUSTC_BOOTSTRAP=1; $env:RUSTFLAGS='--sysroot=' + (Resolve-Path src/tests/rust_guests/target-non-pie/sysroot).Path + ' -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint';" } else { "" } }} cd src/tests/rust_guests/simpleguest && {{ if os() == "windows" { "" } else { "RUSTC_BOOTSTRAP=1 RUSTFLAGS=\"--sysroot=$(cd .. && pwd)/target-non-pie/sysroot -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint\"" } }} cargo build --target {{ hyperlight-target }} --target-dir ../target-non-pie/build --profile={{ if target == "debug" { "dev" } else { target } }} +# Uses plain cargo with non-PIE link flags and the sysroot of the PIE build. +# cargo-hyperlight would leak these RUSTFLAGS into its sysroot wrapper build. +build-rust-guests-non-pie target=default-target: (build-rust-guests target) + {{ if os() == "windows" { "$env:RUSTC_BOOTSTRAP=1; $env:RUSTFLAGS='--sysroot=' + (Resolve-Path src/tests/rust_guests/target/sysroot).Path + ' -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint';" } else { "" } }} cd src/tests/rust_guests/simpleguest && {{ if os() == "windows" { "" } else { "RUSTC_BOOTSTRAP=1 RUSTFLAGS=\"--sysroot=$(cd .. && pwd)/target/sysroot -C relocation-model=static -C link-args=--no-pie -C link-args=--image-base=0x1000000 --cfg=hyperlight --check-cfg=cfg(hyperlight) -Clink-args=-eentrypoint\"" } }} cargo build --target {{ hyperlight-target }} --target-dir ../target-non-pie/build --profile={{ if target == "debug" { "dev" } else { target } }} non_pie_guests_target := "src/tests/rust_guests/target-non-pie/build/" + hyperlight-target @@ -248,6 +246,20 @@ like-ci config=default-target hypervisor="kvm": # runs all tests test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration target features) (test-doc target features) (test-tools target) +# Cargo launches test binaries through the runner in this variable, for example +# CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER. +no-run-runner-var := "CARGO_TARGET_" + uppercase(arch()) + (if os() == "windows" { "_PC_WINDOWS_MSVC" } else if os() == "macos" { "_APPLE_DARWIN" } else { "_UNKNOWN_LINUX_GNU" }) + "_RUNNER" +# Cargo runs ` `. Both forms exit 0 and ignore the rest. +no-run-runner := if os() == "windows" { "cmd /c rem" } else { "true" } + +# Builds what `just test` and `cargo ci` compile, to fill the CI cache. +# Tests build but do not run. Doctests are skipped because rustdoc compiles +# them when they run. +prime-caches target=default-target: + just build {{ target }} + cargo build -p hyperlight-ci + {{ set-env-command }}{{ no-run-runner-var }}='{{ no-run-runner }}'; just test-unit {{ target }} && just test-isolated {{ target }} && just test-integration {{ target }} && just test-tools {{ target }} + # runs the tests of the tools this repository builds for itself, which sit # outside the default members the other recipes select test-tools target=default-target: @@ -467,8 +479,8 @@ bench-ci baseline features="": cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }} # Measure a commit and the current tree on this machine, and compare them -bench-against commit guests features="": - cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --baseline-ref {{ commit }} --baseline-guests {{ guests }} --verbose +bench-against commit guests features="" baseline_target_dir="": + cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --baseline-ref {{ commit }} --baseline-guests {{ guests }} {{ if baseline_target_dir == "" {''} else { "--baseline-target-dir " + baseline_target_dir } }} --verbose bench features="": cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose @@ -536,6 +548,10 @@ fuzz-trace-cmin fuzz-target="fuzz_guest_trace": build-trace-fuzzers: cargo +{{nightly-toolchain}} fuzz build fuzz_guest_trace --features trace +# Builds every fuzz target with the flags `just fuzz` uses +build-fuzz-targets: + targets="$(cargo +{{nightly-toolchain}} fuzz list)" && [ -n "$targets" ] && for t in $targets; do case "$t" in *trace*) f="--features trace" ;; *) f="" ;; esac; cargo +{{nightly-toolchain}} fuzz build "$t" --release $f || exit 1; done + #################### ### COVERAGE ####### #################### diff --git a/src/hyperlight_ci/src/ballast.rs b/src/hyperlight_ci/src/ballast.rs index 32113fca7..2a5f113a6 100644 --- a/src/hyperlight_ci/src/ballast.rs +++ b/src/hyperlight_ci/src/ballast.rs @@ -3,7 +3,7 @@ //! A resident VM held for the duration of a benchmark run. use std::io::{BufRead, BufReader}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; use anyhow::{Context, bail}; @@ -26,9 +26,10 @@ pub(crate) struct Ballast { } impl Ballast { - /// Build the helper, start it, and wait until its sandbox is live. - pub(crate) fn start() -> anyhow::Result { - let exe = build().context("Failed to build the ballast helper")?; + /// Build the helper in `target_dir`, start it, and wait until its sandbox + /// is live. + pub(crate) fn start(target_dir: Option<&Path>) -> anyhow::Result { + let exe = build(target_dir).context("Failed to build the ballast helper")?; let mut child = Command::new(&exe) .stdin(Stdio::piped()) @@ -62,21 +63,23 @@ impl Drop for Ballast { } /// Build the ballast example and return the executable cargo produced. -fn build() -> anyhow::Result { +fn build(target_dir: Option<&Path>) -> anyhow::Result { let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); - let output = Command::new(cargo) - .args([ - "build", - "--release", - "--package", - "hyperlight-host", - "--example", - "ballast", - "--message-format=json-render-diagnostics", - ]) - .stderr(Stdio::inherit()) - .output()?; + let mut cmd = Command::new(cargo); + cmd.args([ + "build", + "--release", + "--package", + "hyperlight-host", + "--example", + "ballast", + "--message-format=json-render-diagnostics", + ]); + if let Some(dir) = target_dir { + cmd.arg("--target-dir").arg(dir); + } + let output = cmd.stderr(Stdio::inherit()).output()?; if !output.status.success() { bail!("cargo build failed for the ballast helper"); diff --git a/src/hyperlight_ci/src/bench.rs b/src/hyperlight_ci/src/bench.rs index 065b2875c..9fe331d6b 100644 --- a/src/hyperlight_ci/src/bench.rs +++ b/src/hyperlight_ci/src/bench.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::io::IsTerminal; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::Context; use criterion_swarm::{CriterionSwarm, OutputMode}; @@ -101,6 +101,10 @@ pub struct BenchArgs { /// `rust` and one `c` directory. #[arg(long, value_name = "DIR")] pub baseline_guests: Option, + /// Cargo target directory the `--baseline-ref` pass builds in. A directory + /// seeded from a build of a nearby commit saves most of the compile. + #[arg(long, value_name = "DIR", requires = "baseline_ref")] + pub baseline_target_dir: Option, /// Name criterion keeps the `--baseline-ref` results under #[arg(long, value_name = "NAME", default_value = "base")] pub baseline_name: String, @@ -113,7 +117,7 @@ pub struct BenchArgs { pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { let Some(spec) = args.baseline_ref.take() else { let bench_args = std::mem::take(&mut args.bench_args); - return measure(&args, bench_args, None).await; + return measure(&args, bench_args, None, None).await; }; let commit = resolve(&spec, args.repo.as_deref())?; @@ -133,6 +137,12 @@ pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { std::fs::create_dir_all(&home) .with_context(|| format!("Failed to create {}", home.display()))?; std::env::set_var("CRITERION_HOME", &home); + let target_dir = args + .baseline_target_dir + .as_deref() + .map(std::path::absolute) + .transpose() + .context("Failed to resolve the baseline target directory")?; // A baseline left by an earlier run would otherwise stand in for any // benchmark this one cannot measure, which is every benchmark the commit @@ -152,7 +162,13 @@ pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { let here = std::env::current_dir().context("Failed to read the working directory")?; std::env::set_current_dir(worktree.path()) .with_context(|| format!("Failed to enter {}", worktree.path().display()))?; - let measured = measure(&args, with(&args, "--save-baseline"), None).await; + let measured = measure( + &args, + with(&args, "--save-baseline"), + None, + target_dir.as_deref(), + ) + .await; std::env::set_current_dir(&here) .with_context(|| format!("Failed to return to {}", here.display()))?; measured.with_context(|| format!("Failed to measure {commit}"))?; @@ -164,6 +180,7 @@ pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { &args, with(&args, "--baseline-lenient"), Some(measured_against), + None, ) .await } @@ -224,10 +241,26 @@ fn with(args: &BenchArgs, flag: &str) -> Vec { bench_args } +/// The cargo arguments that build the benchmarks. +fn cargo_args(features: &[String], target_dir: Option<&Path>) -> anyhow::Result> { + let mut cargo_args = Vec::new(); + if !features.is_empty() { + cargo_args.extend(["--features".to_string(), features.join(",")]); + } + if let Some(dir) = target_dir { + let dir = dir + .to_str() + .with_context(|| format!("{} is not valid UTF-8", dir.display()))?; + cargo_args.push(format!("--target-dir={dir}")); + } + Ok(cargo_args) +} + async fn measure( args: &BenchArgs, bench_args: Vec, baseline: Option, + target_dir: Option<&Path>, ) -> anyhow::Result<()> { let config = args .config_file @@ -241,9 +274,7 @@ async fn measure( swarm = swarm.binaries(args.binary.clone()); } - if !args.features.is_empty() { - swarm = swarm.build_args(["--features".to_string(), args.features.join(",")]); - } + swarm = swarm.build_args(cargo_args(&args.features, target_dir)?); for arg in bench_args { swarm = swarm.bench_arg(arg); @@ -307,7 +338,7 @@ async fn measure( let ballast = if args.no_ballast { None } else { - Some(Ballast::start()?) + Some(Ballast::start(target_dir)?) }; let result = swarm.run().await.context("Failed to run criterion swarm"); @@ -319,7 +350,9 @@ async fn measure( #[cfg(test)] mod tests { - use super::{clear_baseline, resolve}; + use std::path::Path; + + use super::{cargo_args, clear_baseline, resolve}; /// Only the baseline being rebuilt goes, so a run holds on to the /// measurements it is not about to replace. @@ -383,6 +416,23 @@ mod tests { } } + #[test] + fn cargo_args_name_features_and_target_dir() { + assert!(cargo_args(&[], None).unwrap().is_empty()); + assert_eq!( + cargo_args( + &["kvm".to_string(), "mshv3".to_string()], + Some(Path::new("/tmp/baseline-target")) + ) + .unwrap(), + [ + "--features", + "kvm,mshv3", + "--target-dir=/tmp/baseline-target" + ] + ); + } + #[test] fn a_pull_request_without_a_number_is_reported() { let error = resolve("base-of:not-a-number", None)