From cf763073d3048ad1f3daed901d28b92b0bbef649 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 20 Oct 2026 23:42:10 +0530 Subject: [PATCH 1/5] Queue pull request CI runs per author and workflow One author opening or pushing many pull requests could fill the runner queue and delay everyone else. Each PR-triggered workflow now uses a concurrency group keyed on the workflow and the actor, so an author has one run in progress per workflow while the rest wait in order (queue: max) rather than being cancelled. Pushes to main, scheduled runs and manual dispatches fall back to the run ID and are never held back. Signed-off-by: dhruv8sh --- .github/workflows/codeql.yml | 9 +++++++++ .github/workflows/format.yml | 9 +++++++++ .github/workflows/integration-tests.yml | 9 +++++++++ .github/workflows/pbs-example.yml | 9 +++++++++ .github/workflows/test.yml | 9 +++++++++ 5 files changed, 45 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4b963e0153..0a03f49e82 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,6 +19,15 @@ on: schedule: - cron: "42 6 * * 0" +# Each author runs one pull request at a time per workflow, so one person +# pushing many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled. Other events get their +# own group from the run ID and are never held back. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + cancel-in-progress: false + queue: max + jobs: analyze: name: Analyze (${{ matrix.language }}) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index f458572eb7..be6543b5d0 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -8,6 +8,15 @@ on: permissions: contents: read +# Each author runs one pull request at a time per workflow, so one person +# pushing many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled. Other events get their +# own group from the run ID and are never held back. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + cancel-in-progress: false + queue: max + jobs: check-claude-md-symlink: name: CLAUDE.md symlink guard diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 157a3582d3..d5cf97b726 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,6 +10,15 @@ on: types: [opened, synchronize, reopened] workflow_dispatch: +# Each author runs one pull request at a time per workflow, so one person +# pushing many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled. Other events get their +# own group from the run ID and are never held back. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + cancel-in-progress: false + queue: max + env: ORIGIN_PORT: 8888 ARTIFACTS_DIR: /tmp/integration-test-artifacts diff --git a/.github/workflows/pbs-example.yml b/.github/workflows/pbs-example.yml index dced0c7b63..ca3245d0c0 100644 --- a/.github/workflows/pbs-example.yml +++ b/.github/workflows/pbs-example.yml @@ -17,6 +17,15 @@ on: permissions: contents: read +# Each author runs one pull request at a time per workflow, so one person +# pushing many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled. Other events get their +# own group from the run ID and are never held back. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + cancel-in-progress: false + queue: max + jobs: local-checks: name: PBS Terraform and Compose checks diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b27173dd66..777578bb36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,15 @@ on: branches: [main] pull_request: +# Each author runs one pull request at a time per workflow, so one person +# pushing many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled. Other events get their +# own group from the run ID and are never held back. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + cancel-in-progress: false + queue: max + jobs: test-rust: name: cargo test From 68e005b052c96754a5e5fc8f8a94000c56999e58 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 21 Oct 2026 00:47:36 +0530 Subject: [PATCH 2/5] Key the CI queue on the PR author and add job timeouts Group pull request runs by the PR author instead of the triggering actor, so a maintainer pushing to or updating contributors' branches does not pile those runs into the maintainer's queue. Bot-authored PRs fall back to the run ID so a Dependabot batch is not serialized. Give every job in the queued workflows a timeout of roughly three times its longest recent run, so a hung job cannot hold an author's queue for GitHub's six-hour default. Signed-off-by: dhruv8sh --- .github/workflows/codeql.yml | 12 +++++++----- .github/workflows/format.yml | 16 +++++++++++----- .github/workflows/integration-tests.yml | 11 ++++++----- .github/workflows/pbs-example.yml | 11 ++++++----- .github/workflows/test.yml | 17 ++++++++++++----- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0a03f49e82..09f03e0e0a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,12 +19,13 @@ on: schedule: - cron: "42 6 * * 0" -# Each author runs one pull request at a time per workflow, so one person -# pushing many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled. Other events get their -# own group from the run ID and are never held back. +# Each pull request author runs one PR at a time per workflow, so one person +# opening many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled, and every job has a +# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and +# other events get their own group from the run ID and are never held back. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} cancel-in-progress: false queue: max @@ -37,6 +38,7 @@ jobs: # - https://gh.io/using-larger-runners (GitHub.com only) # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: 45 permissions: # required for all workflows security-events: write diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index be6543b5d0..7de102c2de 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -8,12 +8,13 @@ on: permissions: contents: read -# Each author runs one pull request at a time per workflow, so one person -# pushing many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled. Other events get their -# own group from the run ID and are never held back. +# Each pull request author runs one PR at a time per workflow, so one person +# opening many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled, and every job has a +# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and +# other events get their own group from the run ID and are never held back. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} cancel-in-progress: false queue: max @@ -21,6 +22,7 @@ jobs: check-claude-md-symlink: name: CLAUDE.md symlink guard runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v4 @@ -51,6 +53,7 @@ jobs: format-rust: name: cargo fmt runs-on: ubuntu-latest + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -112,6 +115,7 @@ jobs: format-typescript: runs-on: ubuntu-latest + timeout-minutes: 10 defaults: run: working-directory: crates/trusted-server-js/lib @@ -143,6 +147,7 @@ jobs: format-docs: runs-on: ubuntu-latest + timeout-minutes: 10 defaults: run: working-directory: docs @@ -184,6 +189,7 @@ jobs: docs-proposal-scripts: name: docs-proposal scripts runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d5cf97b726..2346f5a525 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,12 +10,13 @@ on: types: [opened, synchronize, reopened] workflow_dispatch: -# Each author runs one pull request at a time per workflow, so one person -# pushing many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled. Other events get their -# own group from the run ID and are never held back. +# Each pull request author runs one PR at a time per workflow, so one person +# opening many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled, and every job has a +# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and +# other events get their own group from the run ID and are never held back. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} cancel-in-progress: false queue: max diff --git a/.github/workflows/pbs-example.yml b/.github/workflows/pbs-example.yml index ca3245d0c0..2c1fb07303 100644 --- a/.github/workflows/pbs-example.yml +++ b/.github/workflows/pbs-example.yml @@ -17,12 +17,13 @@ on: permissions: contents: read -# Each author runs one pull request at a time per workflow, so one person -# pushing many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled. Other events get their -# own group from the run ID and are never held back. +# Each pull request author runs one PR at a time per workflow, so one person +# opening many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled, and every job has a +# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and +# other events get their own group from the run ID and are never held back. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} cancel-in-progress: false queue: max diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 777578bb36..0514afee2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,12 +8,13 @@ on: branches: [main] pull_request: -# Each author runs one pull request at a time per workflow, so one person -# pushing many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled. Other events get their -# own group from the run ID and are never held back. +# Each pull request author runs one PR at a time per workflow, so one person +# opening many PRs cannot fill the runner queue ahead of everyone else. Their +# other runs wait in order instead of being cancelled, and every job has a +# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and +# other events get their own group from the run ID and are never held back. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.actor || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} cancel-in-progress: false queue: max @@ -77,6 +78,7 @@ jobs: test-axum: name: cargo test (axum native) runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -125,6 +127,7 @@ jobs: test-cloudflare: name: cargo check (cloudflare native + wasm32-unknown-unknown) runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -146,6 +149,7 @@ jobs: test-spin: name: cargo check/build/test (spin native + wasm32-wasip1) runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -179,6 +183,7 @@ jobs: test-parity: name: cargo test (cross-adapter parity) runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -204,6 +209,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} + timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -268,6 +274,7 @@ jobs: test-typescript: name: vitest runs-on: ubuntu-latest + timeout-minutes: 10 defaults: run: working-directory: crates/trusted-server-js/lib From 7074ec1c56becff04458cb1a087a84b8cb562111 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 21:46:12 +0530 Subject: [PATCH 3/5] Cancel superseded PR runs and queue each job per author A workflow-level group per pull request now cancels a run when a newer push arrives, so stale runs no longer hold an author's queue or count toward GitHub's 100-pending cap. The per-author queue moves to each job's own concurrency block with queue: max, keyed on the matrix for matrix jobs so their legs do not queue behind each other. Bot-authored PRs and non-PR events still key on the run ID. The shared comment now states the FIFO ordering and the 100-pending limit. Signed-off-by: dhruv8sh --- .github/workflows/codeql.yml | 20 +++++++----- .github/workflows/format.yml | 32 ++++++++++++++----- .github/workflows/integration-tests.yml | 29 ++++++++++++----- .github/workflows/pbs-example.yml | 20 +++++++----- .github/workflows/test.yml | 41 ++++++++++++++++++++----- 5 files changed, 102 insertions(+), 40 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 09f03e0e0a..1d3f825ffe 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,15 +19,16 @@ on: schedule: - cron: "42 6 * * 0" -# Each pull request author runs one PR at a time per workflow, so one person -# opening many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled, and every job has a -# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and -# other events get their own group from the run ID and are never held back. +# A newer push to a pull request cancels that PR's superseded run. Each job +# then joins a per-author queue for that job (its own `concurrency` block), so +# one person opening many PRs runs one copy of each job at a time and cannot +# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub +# keeps up to 100 pending per queue and cancels arrivals past that. Every job +# has a timeout so a hung job cannot hold its queue for hours. Bot-authored +# PRs and non-PR events key on the run ID and never wait on a queue. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} - cancel-in-progress: false - queue: max + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: analyze: @@ -39,6 +40,9 @@ jobs: # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} timeout-minutes: 45 + concurrency: + group: ${{ github.workflow }}-analyze-${{ matrix.language }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max permissions: # required for all workflows security-events: write diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 7de102c2de..1d03f47364 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -8,21 +8,25 @@ on: permissions: contents: read -# Each pull request author runs one PR at a time per workflow, so one person -# opening many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled, and every job has a -# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and -# other events get their own group from the run ID and are never held back. +# A newer push to a pull request cancels that PR's superseded run. Each job +# then joins a per-author queue for that job (its own `concurrency` block), so +# one person opening many PRs runs one copy of each job at a time and cannot +# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub +# keeps up to 100 pending per queue and cancels arrivals past that. Every job +# has a timeout so a hung job cannot hold its queue for hours. Bot-authored +# PRs and non-PR events key on the run ID and never wait on a queue. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} - cancel-in-progress: false - queue: max + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: check-claude-md-symlink: name: CLAUDE.md symlink guard runs-on: ubuntu-latest timeout-minutes: 5 + concurrency: + group: ${{ github.workflow }}-check-claude-md-symlink-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -54,6 +58,9 @@ jobs: name: cargo fmt runs-on: ubuntu-latest timeout-minutes: 25 + concurrency: + group: ${{ github.workflow }}-format-rust-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -116,6 +123,9 @@ jobs: format-typescript: runs-on: ubuntu-latest timeout-minutes: 10 + concurrency: + group: ${{ github.workflow }}-format-typescript-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max defaults: run: working-directory: crates/trusted-server-js/lib @@ -148,6 +158,9 @@ jobs: format-docs: runs-on: ubuntu-latest timeout-minutes: 10 + concurrency: + group: ${{ github.workflow }}-format-docs-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max defaults: run: working-directory: docs @@ -190,6 +203,9 @@ jobs: name: docs-proposal scripts runs-on: ubuntu-latest timeout-minutes: 10 + concurrency: + group: ${{ github.workflow }}-docs-proposal-scripts-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3867d79224..a276d23ecc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,15 +10,16 @@ on: types: [opened, synchronize, reopened] workflow_dispatch: -# Each pull request author runs one PR at a time per workflow, so one person -# opening many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled, and every job has a -# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and -# other events get their own group from the run ID and are never held back. +# A newer push to a pull request cancels that PR's superseded run. Each job +# then joins a per-author queue for that job (its own `concurrency` block), so +# one person opening many PRs runs one copy of each job at a time and cannot +# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub +# keeps up to 100 pending per queue and cancels arrivals past that. Every job +# has a timeout so a hung job cannot hold its queue for hours. Bot-authored +# PRs and non-PR events key on the run ID and never wait on a queue. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} - cancel-in-progress: false - queue: max + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true env: ORIGIN_PORT: 8888 @@ -33,6 +34,9 @@ jobs: name: prepare integration artifacts runs-on: ubuntu-latest timeout-minutes: 30 + concurrency: + group: ${{ github.workflow }}-prepare-artifacts-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -70,6 +74,9 @@ jobs: needs: prepare-artifacts runs-on: ubuntu-latest timeout-minutes: 20 + concurrency: + group: ${{ github.workflow }}-integration-tests-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -129,6 +136,9 @@ jobs: needs: prepare-artifacts runs-on: ubuntu-latest timeout-minutes: 15 + concurrency: + group: ${{ github.workflow }}-integration-tests-fastly-ec-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -173,6 +183,9 @@ jobs: # once the pinned Playwright version supports the newer image. runs-on: ubuntu-24.04 timeout-minutes: 15 + concurrency: + group: ${{ github.workflow }}-browser-tests-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pbs-example.yml b/.github/workflows/pbs-example.yml index 2c1fb07303..d61072867a 100644 --- a/.github/workflows/pbs-example.yml +++ b/.github/workflows/pbs-example.yml @@ -17,21 +17,25 @@ on: permissions: contents: read -# Each pull request author runs one PR at a time per workflow, so one person -# opening many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled, and every job has a -# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and -# other events get their own group from the run ID and are never held back. +# A newer push to a pull request cancels that PR's superseded run. Each job +# then joins a per-author queue for that job (its own `concurrency` block), so +# one person opening many PRs runs one copy of each job at a time and cannot +# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub +# keeps up to 100 pending per queue and cancels arrivals past that. Every job +# has a timeout so a hung job cannot hold its queue for hours. Bot-authored +# PRs and non-PR events key on the run ID and never wait on a queue. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} - cancel-in-progress: false - queue: max + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: local-checks: name: PBS Terraform and Compose checks runs-on: ubuntu-latest timeout-minutes: 20 + concurrency: + group: ${{ github.workflow }}-local-checks-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max env: TF_IN_AUTOMATION: 'true' AWS_EC2_METADATA_DISABLED: 'true' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2e90e4b19..a5c669f83e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,21 +8,25 @@ on: branches: [main] pull_request: -# Each pull request author runs one PR at a time per workflow, so one person -# opening many PRs cannot fill the runner queue ahead of everyone else. Their -# other runs wait in order instead of being cancelled, and every job has a -# timeout so a hung run cannot hold that queue for hours. Bot-authored PRs and -# other events get their own group from the run ID and are never held back. +# A newer push to a pull request cancels that PR's superseded run. Each job +# then joins a per-author queue for that job (its own `concurrency` block), so +# one person opening many PRs runs one copy of each job at a time and cannot +# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub +# keeps up to 100 pending per queue and cancels arrivals past that. Every job +# has a timeout so a hung job cannot hold its queue for hours. Bot-authored +# PRs and non-PR events key on the run ID and never wait on a queue. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} - cancel-in-progress: false - queue: max + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: test-rust: name: cargo test runs-on: ubuntu-latest timeout-minutes: 45 + concurrency: + group: ${{ github.workflow }}-test-rust-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -79,6 +83,9 @@ jobs: name: HTML EOF runtime (Fastly) runs-on: ubuntu-latest timeout-minutes: 25 + concurrency: + group: ${{ github.workflow }}-test-fastly-eof-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -128,6 +135,9 @@ jobs: name: cargo test (axum native) runs-on: ubuntu-latest timeout-minutes: 30 + concurrency: + group: ${{ github.workflow }}-test-axum-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -199,6 +209,9 @@ jobs: name: cargo check (cloudflare native + wasm32-unknown-unknown) runs-on: ubuntu-latest timeout-minutes: 15 + concurrency: + group: ${{ github.workflow }}-test-cloudflare-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -225,6 +238,9 @@ jobs: name: cargo check/build/test (spin native + wasm32-wasip1) runs-on: ubuntu-latest timeout-minutes: 20 + concurrency: + group: ${{ github.workflow }}-test-spin-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -263,6 +279,9 @@ jobs: name: cargo test (cross-adapter parity) runs-on: ubuntu-latest timeout-minutes: 15 + concurrency: + group: ${{ github.workflow }}-test-parity-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -289,6 +308,9 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 45 + concurrency: + group: ${{ github.workflow }}-test-cli-${{ matrix.os }}-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 @@ -354,6 +376,9 @@ jobs: name: vitest runs-on: ubuntu-latest timeout-minutes: 10 + concurrency: + group: ${{ github.workflow }}-test-typescript-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max defaults: run: working-directory: crates/trusted-server-js/lib From 600f6a9c19e4e2a9efbe59eab240290646aa0282 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 21:46:40 +0530 Subject: [PATCH 4/5] Guard the CI concurrency blocks and job timeouts The required cargo fmt job now fails when a PR-triggered workflow lacks the per-PR cancellation block from test.yml, or when a job lacks timeout-minutes or a per-author queue keyed on pull_request.user.login with queue: max. Matrix jobs must also include the matrix in their group. Jobs that call a reusable workflow are skipped. The step uses the runner image's yq. Signed-off-by: dhruv8sh --- .github/workflows/format.yml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 1d03f47364..0379093897 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -64,6 +64,48 @@ jobs: steps: - uses: actions/checkout@v4 + # A hung job, a PR-triggered workflow without per-PR cancellation, or a + # job outside its per-author queue brings back the runner pile-up that + # the concurrency blocks prevent. Every PR-triggered workflow must carry + # test.yml's workflow-level block, and every job must set timeout-minutes + # and a per-author queue (keyed on the matrix too, for matrix jobs). + # Jobs that call a reusable workflow are skipped: they cannot set these. + - name: Require per-PR cancellation, per-author job queues and timeouts + shell: bash + run: | + reference=.github/workflows/test.yml + if [ "$(yq '.concurrency."cancel-in-progress" == true and (.concurrency.group // "" | test("pull_request\.number"))' "$reference")" != true ]; then + echo "::error file=$reference::test.yml must cancel superseded runs per pull request." + exit 1 + fi + expected=$(yq -o=json -I=0 '.concurrency' "$reference") + status=0 + for f in .github/workflows/*.yml .github/workflows/*.yaml; do + [ -f "$f" ] || continue + on_pr=$(yq '.on | ((tag == "!!map" and has("pull_request")) or (tag == "!!str" and . == "pull_request") or (tag == "!!seq" and ([.[] | select(. == "pull_request")] | length > 0)))' "$f") + [ "$on_pr" = true ] || continue + if [ "$(yq -o=json -I=0 '.concurrency' "$f")" != "$expected" ]; then + echo "::error file=$f::PR-triggered workflows must use the workflow-level concurrency block from test.yml." + status=1 + fi + while read -r job; do + [ -n "$job" ] || continue + echo "::error file=$f::Job '$job' must set timeout-minutes." + status=1 + done < <(yq '.jobs | to_entries[] | select(.value.uses == null and .value["timeout-minutes"] == null) | .key' "$f") + while read -r job; do + [ -n "$job" ] || continue + echo "::error file=$f::Job '$job' must join a per-author queue: concurrency.group keyed on pull_request.user.login and queue: max." + status=1 + done < <(yq '.jobs | to_entries[] | select(.value.uses == null and (.value.concurrency.queue != "max" or (.value.concurrency.group // "" | test("pull_request\.user\.login") | not))) | .key' "$f") + while read -r job; do + [ -n "$job" ] || continue + echo "::error file=$f::Matrix job '$job' must include the matrix in its concurrency group so its legs do not queue behind each other." + status=1 + done < <(yq '.jobs | to_entries[] | select(.value.strategy.matrix != null and (.value.concurrency.group // "" | test("matrix\.") | not)) | .key' "$f") + done + exit "$status" + # Rust jobs must set up the toolchain through ./.github/actions/setup-rust, # which removes the image's stable and only saves caches from main. A # direct setup-rust-toolchain or rust-cache step skips both silently. From 32523f1848a62ff2c8f8d06c6ca2bcb31cdb938e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 30 Oct 2026 12:48:24 +0530 Subject: [PATCH 5/5] Strengthen and document CI concurrency policy checks --- .github/workflows/codeql.yml | 13 +- .github/workflows/format.yml | 76 +++----- .github/workflows/integration-tests.yml | 13 +- .github/workflows/pbs-example.yml | 13 +- .github/workflows/test.yml | 17 +- AGENTS.md | 1 + scripts/README.md | 1 + scripts/check-ci-concurrency.sh | 164 +++++++++++++++++ scripts/check-ci-concurrency.test.mjs | 233 ++++++++++++++++++++++++ 9 files changed, 453 insertions(+), 78 deletions(-) create mode 100755 scripts/check-ci-concurrency.sh create mode 100644 scripts/check-ci-concurrency.test.mjs diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1d3f825ffe..45ea7f8a5e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,13 +19,12 @@ on: schedule: - cron: "42 6 * * 0" -# A newer push to a pull request cancels that PR's superseded run. Each job -# then joins a per-author queue for that job (its own `concurrency` block), so -# one person opening many PRs runs one copy of each job at a time and cannot -# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub -# keeps up to 100 pending per queue and cancels arrivals past that. Every job -# has a timeout so a hung job cannot hold its queue for hours. Bot-authored -# PRs and non-PR events key on the run ID and never wait on a queue. +# New pushes cancel older runs for the same pull request. Each job queues by +# workflow, job, and author, so one author runs one copy of each job at a time. +# GitHub orders jobs by when they start waiting, but does not guarantee FIFO. +# Each queue holds up to 100 pending jobs; later arrivals are canceled. +# Timeouts release queues after hung jobs. Bots and non-PR events use unique +# run IDs and do not queue together. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 5e80456a57..2398eb8a91 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -8,13 +8,12 @@ on: permissions: contents: read -# A newer push to a pull request cancels that PR's superseded run. Each job -# then joins a per-author queue for that job (its own `concurrency` block), so -# one person opening many PRs runs one copy of each job at a time and cannot -# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub -# keeps up to 100 pending per queue and cancels arrivals past that. Every job -# has a timeout so a hung job cannot hold its queue for hours. Bot-authored -# PRs and non-PR events key on the run ID and never wait on a queue. +# New pushes cancel older runs for the same pull request. Each job queues by +# workflow, job, and author, so one author runs one copy of each job at a time. +# GitHub orders jobs by when they start waiting, but does not guarantee FIFO. +# Each queue holds up to 100 pending jobs; later arrivals are canceled. +# Timeouts release queues after hung jobs. Bots and non-PR events use unique +# run IDs and do not queue together. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true @@ -57,9 +56,22 @@ jobs: check-ci-node-pin: name: CI Node pin guard runs-on: ubuntu-latest + timeout-minutes: 5 + concurrency: + group: ${{ github.workflow }}-check-ci-node-pin-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: - uses: actions/checkout@v4 + - name: Install yq + shell: bash + run: | + curl --fail --location --silent --show-error \ + https://github.com/mikefarah/yq/releases/download/v4.54.1/yq_linux_amd64 \ + --output "$RUNNER_TEMP/yq" + chmod +x "$RUNNER_TEMP/yq" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + # The guard's tests check the tree before the guard landed; fetch only # that commit instead of the full history. - name: Fetch the pre-guard commit @@ -70,7 +82,7 @@ jobs: with: node-version-file: .tool-versions - - name: Test the guard + - name: Test the Node pin guard run: node --test scripts/check-ci-node-pin.test.mjs # Every job that compiles trusted-server-core also runs the TSJS build, so @@ -78,6 +90,12 @@ jobs: - name: Verify jobs pin Node before compiling Rust run: node scripts/check-ci-node-pin.mjs + - name: Test the CI concurrency policy guard + run: node --test scripts/check-ci-concurrency.test.mjs + + - name: Require per-PR cancellation, per-author job queues and timeouts + run: scripts/check-ci-concurrency.sh + format-rust: name: cargo fmt runs-on: ubuntu-latest @@ -88,48 +106,6 @@ jobs: steps: - uses: actions/checkout@v4 - # A hung job, a PR-triggered workflow without per-PR cancellation, or a - # job outside its per-author queue brings back the runner pile-up that - # the concurrency blocks prevent. Every PR-triggered workflow must carry - # test.yml's workflow-level block, and every job must set timeout-minutes - # and a per-author queue (keyed on the matrix too, for matrix jobs). - # Jobs that call a reusable workflow are skipped: they cannot set these. - - name: Require per-PR cancellation, per-author job queues and timeouts - shell: bash - run: | - reference=.github/workflows/test.yml - if [ "$(yq '.concurrency."cancel-in-progress" == true and (.concurrency.group // "" | test("pull_request\.number"))' "$reference")" != true ]; then - echo "::error file=$reference::test.yml must cancel superseded runs per pull request." - exit 1 - fi - expected=$(yq -o=json -I=0 '.concurrency' "$reference") - status=0 - for f in .github/workflows/*.yml .github/workflows/*.yaml; do - [ -f "$f" ] || continue - on_pr=$(yq '.on | ((tag == "!!map" and has("pull_request")) or (tag == "!!str" and . == "pull_request") or (tag == "!!seq" and ([.[] | select(. == "pull_request")] | length > 0)))' "$f") - [ "$on_pr" = true ] || continue - if [ "$(yq -o=json -I=0 '.concurrency' "$f")" != "$expected" ]; then - echo "::error file=$f::PR-triggered workflows must use the workflow-level concurrency block from test.yml." - status=1 - fi - while read -r job; do - [ -n "$job" ] || continue - echo "::error file=$f::Job '$job' must set timeout-minutes." - status=1 - done < <(yq '.jobs | to_entries[] | select(.value.uses == null and .value["timeout-minutes"] == null) | .key' "$f") - while read -r job; do - [ -n "$job" ] || continue - echo "::error file=$f::Job '$job' must join a per-author queue: concurrency.group keyed on pull_request.user.login and queue: max." - status=1 - done < <(yq '.jobs | to_entries[] | select(.value.uses == null and (.value.concurrency.queue != "max" or (.value.concurrency.group // "" | test("pull_request\.user\.login") | not))) | .key' "$f") - while read -r job; do - [ -n "$job" ] || continue - echo "::error file=$f::Matrix job '$job' must include the matrix in its concurrency group so its legs do not queue behind each other." - status=1 - done < <(yq '.jobs | to_entries[] | select(.value.strategy.matrix != null and (.value.concurrency.group // "" | test("matrix\.") | not)) | .key' "$f") - done - exit "$status" - # Rust jobs must set up the toolchain through ./.github/actions/setup-rust, # which removes the image's stable and only saves caches from main. A # direct setup-rust-toolchain or rust-cache step skips both silently. diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 2956834fbb..b1d1262750 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,13 +10,12 @@ on: types: [opened, synchronize, reopened] workflow_dispatch: -# A newer push to a pull request cancels that PR's superseded run. Each job -# then joins a per-author queue for that job (its own `concurrency` block), so -# one person opening many PRs runs one copy of each job at a time and cannot -# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub -# keeps up to 100 pending per queue and cancels arrivals past that. Every job -# has a timeout so a hung job cannot hold its queue for hours. Bot-authored -# PRs and non-PR events key on the run ID and never wait on a queue. +# New pushes cancel older runs for the same pull request. Each job queues by +# workflow, job, and author, so one author runs one copy of each job at a time. +# GitHub orders jobs by when they start waiting, but does not guarantee FIFO. +# Each queue holds up to 100 pending jobs; later arrivals are canceled. +# Timeouts release queues after hung jobs. Bots and non-PR events use unique +# run IDs and do not queue together. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/pbs-example.yml b/.github/workflows/pbs-example.yml index d61072867a..0282e5a066 100644 --- a/.github/workflows/pbs-example.yml +++ b/.github/workflows/pbs-example.yml @@ -17,13 +17,12 @@ on: permissions: contents: read -# A newer push to a pull request cancels that PR's superseded run. Each job -# then joins a per-author queue for that job (its own `concurrency` block), so -# one person opening many PRs runs one copy of each job at a time and cannot -# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub -# keeps up to 100 pending per queue and cancels arrivals past that. Every job -# has a timeout so a hung job cannot hold its queue for hours. Bot-authored -# PRs and non-PR events key on the run ID and never wait on a queue. +# New pushes cancel older runs for the same pull request. Each job queues by +# workflow, job, and author, so one author runs one copy of each job at a time. +# GitHub orders jobs by when they start waiting, but does not guarantee FIFO. +# Each queue holds up to 100 pending jobs; later arrivals are canceled. +# Timeouts release queues after hung jobs. Bots and non-PR events use unique +# run IDs and do not queue together. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 173b83d05b..0642b6a959 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,13 +8,12 @@ on: branches: [main] pull_request: -# A newer push to a pull request cancels that PR's superseded run. Each job -# then joins a per-author queue for that job (its own `concurrency` block), so -# one person opening many PRs runs one copy of each job at a time and cannot -# fill the runner pool ahead of everyone else. Queued jobs start in the order they began waiting; GitHub -# keeps up to 100 pending per queue and cancels arrivals past that. Every job -# has a timeout so a hung job cannot hold its queue for hours. Bot-authored -# PRs and non-PR events key on the run ID and never wait on a queue. +# New pushes cancel older runs for the same pull request. Each job queues by +# workflow, job, and author, so one author runs one copy of each job at a time. +# GitHub orders jobs by when they start waiting, but does not guarantee FIFO. +# Each queue holds up to 100 pending jobs; later arrivals are canceled. +# Timeouts release queues after hung jobs. Bots and non-PR events use unique +# run IDs and do not queue together. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true @@ -415,6 +414,10 @@ jobs: name: ts dev lint domains (changed lines) if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 10 + concurrency: + group: ${{ github.workflow }}-lint-domains-${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }} + queue: max steps: # A PR checkout is GitHub's merge commit; depth 2 also fetches its # parents, so `HEAD^1` is the exact base commit the PR merges into. diff --git a/AGENTS.md b/AGENTS.md index 7688c1829f..28f815c938 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -375,6 +375,7 @@ Every PR must pass: 11. Documentation proposal scripts: `shellcheck scripts/docs-proposal/*.sh && node --test scripts/docs-proposal/hunks.test.mjs && scripts/docs-proposal/test.sh` 12. URL-host linter on lines the PR adds (`lint-domains` job, PRs only); run it locally with `cargo run --manifest-path crates/trusted-server-cli/Cargo.toml --target --bin ts -- dev lint domains --changed-vs origin/main`, using the PR's base branch in place of `main` when it targets another branch 13. CI Node pin guard: `node --test scripts/check-ci-node-pin.test.mjs && node scripts/check-ci-node-pin.mjs` with the `.tool-versions` Node (every CI job that compiles Rust sets up the `.tool-versions` Node before its first cargo step) +14. CI concurrency policy guard: `node --test scripts/check-ci-concurrency.test.mjs && scripts/check-ci-concurrency.sh` (requires yq v4); every PR-triggered workflow must use `test.yml`'s per-PR cancellation block, and every non-reusable job must set a timeout and a per-author `queue: max` group. Matrix jobs must also include their matrix values in the group. CI also runs these HTML EOF checks. Run them locally only when changing EOF finalization, publisher or HTML-processor code: diff --git a/scripts/README.md b/scripts/README.md index 21fee51f52..c86cad849a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,6 +8,7 @@ their documented orchestration handles a probe deliberately. | `batch-sync.sh` | Endpoint, bearer token, EC ID, partner UID; `curl` and Python | Sends one authenticated batch-sync request. Its temporary response file is removed on exit. | | `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | | `check-ci-node-pin.mjs` | Pinned Node; tests in `check-ci-node-pin.test.mjs` | Read-only. Fails when a CI job compiles Rust before an unconditional `actions/setup-node` step pinned to `.tool-versions`; `--root` checks another checkout. | +| `check-ci-concurrency.sh` | yq v4; tests in `check-ci-concurrency.test.mjs` | Read-only. Checks per-PR workflow cancellation, per-job author queues, matrix groups, and timeouts; `--root` checks another checkout and `--yq` selects yq. | | `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | | `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | | `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | diff --git a/scripts/check-ci-concurrency.sh b/scripts/check-ci-concurrency.sh new file mode 100755 index 0000000000..36d3a06b89 --- /dev/null +++ b/scripts/check-ci-concurrency.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/.." && pwd) +root=$repo_root +yq_bin=${YQ_BIN:-yq} + +usage() { + printf 'usage: check-ci-concurrency.sh [--root ROOT] [--yq PATH]\n' >&2 +} + +while (($#)); do + case "$1" in + --root) + [[ $# -ge 2 ]] || { usage; exit 2; } + root=$2 + shift 2 + ;; + --yq) + [[ $# -ge 2 ]] || { usage; exit 2; } + yq_bin=$2 + shift 2 + ;; + *) + usage + exit 2 + ;; + esac +done + +workflow_dir=$root/.github/workflows +reference=$workflow_dir/test.yml +if [[ ! -d "$workflow_dir" || ! -f "$reference" ]]; then + printf 'error: expected workflow directory and test.yml under %s\n' "$root" >&2 + exit 1 +fi +if ! command -v "$yq_bin" >/dev/null 2>&1; then + printf 'error: yq v4 is required to check workflow policies\n' >&2 + exit 1 +fi + +workflow_prefix='${{ github.workflow }}' +expected_workflow_group="${workflow_prefix}-\${{ github.event.pull_request.number || github.run_id }}" +expected_author_suffix="\${{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }}" +matrix_prefix="\${{ matrix." +matrix_suffix=' }}-' +matrix_key_pattern='^[[:alnum:]_.-]+$' + +status=0 + +report_error() { + printf '::error file=%s::%s\n' "$1" "$2" + status=1 +} + +has_pull_request_trigger() { + "$yq_bin" -r ' + .on + | ((tag == "!!map" and has("pull_request")) + or (tag == "!!str" and . == "pull_request") + or (tag == "!!seq" and ([.[] | select(. == "pull_request")] | length > 0))) + ' "$1" +} + +job_value() { + local file=$1 + local job=$2 + local expression=$3 + YQ_JOB=$job "$yq_bin" -r ".jobs[strenv(YQ_JOB)].${expression}" "$file" +} + +validate_matrix_segment() { + local remaining=$1 + local matrix_key + + while [[ "$remaining" == "$matrix_prefix"* ]]; do + remaining=${remaining#"$matrix_prefix"} + [[ "$remaining" == *"$matrix_suffix"* ]] || return 1 + matrix_key=${remaining%%"$matrix_suffix"*} + [[ "$matrix_key" =~ $matrix_key_pattern ]] || return 1 + remaining=${remaining#*"$matrix_suffix"} + done + + [[ -z "$remaining" ]] +} + +validate_job_group() { + local file=$1 + local job=$2 + local group=$3 + local has_matrix=$4 + local prefix="${workflow_prefix}-${job}-" + local remainder + local matrix_segment + + if [[ "$group" != "$prefix"* ]]; then + report_error "$file" "Job '$job' concurrency group must start with the workflow name and its own job ID." + return + fi + remainder=${group#"$prefix"} + if [[ "$remainder" != *"$expected_author_suffix" ]]; then + report_error "$file" "Job '$job' concurrency group must use the pull-request author for human PRs and the run ID otherwise." + return + fi + matrix_segment=${remainder%"$expected_author_suffix"} + + if [[ "$has_matrix" == true ]]; then + if [[ -z "$matrix_segment" ]] || ! validate_matrix_segment "$matrix_segment"; then + report_error "$file" "Matrix job '$job' concurrency group must include each matrix leg before the per-author suffix." + fi + elif [[ -n "$matrix_segment" ]]; then + report_error "$file" "Non-matrix job '$job' concurrency group must not add a shared queue segment." + fi +} + +check_workflow() { + local file=$1 + local is_pr + local group + local cancel + local queue + local job + local timeout + local job_group + local job_queue + local has_matrix + + is_pr=$(has_pull_request_trigger "$file") + [[ "$is_pr" == true ]] || return 0 + + group=$("$yq_bin" -r '.concurrency.group // ""' "$file") + cancel=$("$yq_bin" -r '.concurrency."cancel-in-progress" // false' "$file") + queue=$("$yq_bin" -r '.concurrency.queue // ""' "$file") + if [[ "$group" != "$expected_workflow_group" || "$cancel" != true || "$queue" == max ]]; then + report_error "$file" "PR workflow concurrency group must be per-PR cancellation from test.yml, without queue: max." + fi + + while IFS= read -r job; do + [[ -n "$job" ]] || continue + if [[ "$(job_value "$file" "$job" "uses")" != null ]]; then + continue + fi + + timeout=$(job_value "$file" "$job" '"timeout-minutes" // 0') + if [[ ! "$timeout" =~ ^[1-9][0-9]*$ ]]; then + report_error "$file" "Job '$job' must set a positive timeout-minutes value." + fi + + job_group=$(job_value "$file" "$job" "concurrency.group") + job_queue=$(job_value "$file" "$job" "concurrency.queue") + if [[ "$job_queue" != max ]]; then + report_error "$file" "Job '$job' must set queue: max on its per-author concurrency group." + fi + has_matrix=$(job_value "$file" "$job" "strategy.matrix != null") + validate_job_group "$file" "$job" "$job_group" "$has_matrix" + done < <("$yq_bin" -r '.jobs | to_entries[] | select(.value.uses == null) | .key' "$file") +} + +while IFS= read -r -d '' workflow_file; do + check_workflow "$workflow_file" +done < <(find "$workflow_dir" -type f \( -name '*.yml' -o -name '*.yaml' \) -print0) + +exit "$status" diff --git a/scripts/check-ci-concurrency.test.mjs b/scripts/check-ci-concurrency.test.mjs new file mode 100644 index 0000000000..e69d0c8b25 --- /dev/null +++ b/scripts/check-ci-concurrency.test.mjs @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import test from 'node:test' + +const repositoryRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..' +) +const guardPath = path.join(repositoryRoot, 'scripts/check-ci-concurrency.sh') +const workflowPrefix = '$' + '{{ github.workflow }}' +const workflowGroup = + workflowPrefix + + '-' + + '$' + + '{{ github.event.pull_request.number || github.run_id }}' +const authorGroupSuffix = + '$' + + "{{ github.event_name == 'pull_request' && github.event.pull_request.user.type != 'Bot' && github.event.pull_request.user.login || github.run_id }}" + +function authorGroup(job, matrix = '') { + return [workflowPrefix, job, matrix, authorGroupSuffix] + .filter(Boolean) + .join('-') +} + +function workflow(job, group, options = {}) { + const lines = [ + 'name: ' + job, + 'on:', + ' pull_request:', + 'concurrency:', + ' group: ' + workflowGroup, + ' cancel-in-progress: true', + 'jobs:', + ' ' + job + ':', + ' runs-on: ubuntu-latest', + ] + if (options.timeout !== false) lines.push(' timeout-minutes: 5') + if (options.matrix) { + lines.push(' strategy:', ' matrix:', ' os: [ubuntu-latest]') + } + lines.push(' concurrency:', ' group: ' + group) + if (options.queue !== false) lines.push(' queue: max') + lines.push(' steps:', ' - run: echo test') + return lines.join('\n') + '\n' +} + +function fixtureRoot() { + const root = mkdtempSync(path.join(os.tmpdir(), 'check-ci-concurrency-')) + const workflowsPath = path.join(root, '.github/workflows') + mkdirSync(workflowsPath, { recursive: true }) + writeFileSync( + path.join(workflowsPath, 'test.yml'), + workflow('test-rust', authorGroup('test-rust')) + ) + writeFileSync( + path.join(workflowsPath, 'format.yml'), + workflow('format-rust', authorGroup('format-rust')) + ) + writeFileSync( + path.join(workflowsPath, 'push-only.yml'), + [ + 'name: Push only', + 'on:', + ' push:', + 'jobs:', + ' build:', + ' runs-on: ubuntu-latest', + ' steps:', + ' - run: echo test', + '', + ].join('\n') + ) + writeFileSync( + path.join(workflowsPath, 'reusable.yml'), + [ + 'name: Reusable caller', + 'on:', + ' pull_request:', + 'concurrency:', + ' group: ' + workflowGroup, + ' cancel-in-progress: true', + 'jobs:', + ' call:', + ' uses: example/reusable/.github/workflows/test.yml@main', + '', + ].join('\n') + ) + return { root, workflowsPath } +} + +function runGuard(root) { + return spawnSync('bash', [guardPath, '--root', root], { encoding: 'utf8' }) +} + +function withFixture(callback) { + const fixture = fixtureRoot() + try { + callback(fixture) + } finally { + rmSync(fixture.root, { recursive: true, force: true }) + } +} + +function assertRejected(root, expectedMessage) { + const result = runGuard(root) + assert.equal( + result.error, + undefined, + 'should start the guard: ' + result.error + ) + assert.notEqual( + result.status, + 0, + 'should reject the policy:\n' + result.stdout + '\n' + result.stderr + ) + assert.match(result.stdout + result.stderr, expectedMessage) +} + +test('accepts per-PR cancellation and distinct per-author job queues', () => { + withFixture(({ root }) => { + const result = runGuard(root) + assert.equal( + result.error, + undefined, + 'should start the guard: ' + result.error + ) + assert.equal(result.status, 0, result.stdout + '\n' + result.stderr) + }) +}) + +test('rejects author groups that use the actor or queue bot PRs', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + const source = workflow('format-rust', authorGroup('format-rust')) + writeFileSync( + formatPath, + source.replace('pull_request.user.login', 'github.actor') + ) + assertRejected(root, /concurrency group/i) + writeFileSync(formatPath, source.replace("user.type != 'Bot' && ", '')) + assertRejected(root, /concurrency group/i) + }) +}) + +test('rejects a per-author group without the non-PR run-ID fallback', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + const group = authorGroup('format-rust').replace(' || github.run_id', '') + writeFileSync(formatPath, workflow('format-rust', group)) + assertRejected(root, /concurrency group/i) + }) +}) + +test('rejects copied job groups and groups without the workflow prefix', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + writeFileSync(formatPath, workflow('format-rust', authorGroup('test-rust'))) + assertRejected(root, /concurrency group/i) + writeFileSync( + formatPath, + workflow( + 'format-rust', + authorGroup('format-rust').replace(workflowPrefix + '-', '') + ) + ) + assertRejected(root, /concurrency group/i) + }) +}) + +test('requires matrix jobs to separate their legs in the author queue', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + writeFileSync( + formatPath, + workflow( + 'format-rust', + authorGroup('format-rust', '$' + '{{ matrix.os }}'), + { + matrix: true, + } + ) + ) + const passing = runGuard(root) + assert.equal( + passing.error, + undefined, + 'should start the guard: ' + passing.error + ) + assert.equal(passing.status, 0, passing.stdout + '\n' + passing.stderr) + + writeFileSync( + formatPath, + workflow('format-rust', authorGroup('format-rust'), { matrix: true }) + ) + assertRejected(root, /matrix job/i) + }) +}) + +test('requires every non-reusable PR job to set a timeout and queue max', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + writeFileSync( + formatPath, + workflow('format-rust', authorGroup('format-rust'), { timeout: false }) + ) + assertRejected(root, /timeout-minutes/) + writeFileSync( + formatPath, + workflow('format-rust', authorGroup('format-rust'), { queue: false }) + ) + assertRejected(root, /queue: max/) + }) +}) + +test('requires the exact per-PR cancellation group on every PR workflow', () => { + withFixture(({ root, workflowsPath }) => { + const formatPath = path.join(workflowsPath, 'format.yml') + const wrongGroup = workflow( + 'format-rust', + authorGroup('format-rust') + ).replace( + 'github.event.pull_request.number || github.run_id', + 'github.actor' + ) + writeFileSync(formatPath, wrongGroup) + assertRejected(root, /workflow concurrency group/) + }) +})