ci.yml

name: ci

# Runner provider toggle, read from the CI_PROVIDER repo variable:
#
#   gh variable set CI_PROVIDER --body github   # fall back to GitHub-hosted
#   gh variable delete CI_PROVIDER              # back to Blacksmith (default)
#
# A repo variable, not a committed value: during a Blacksmith outage there is no
# working CI to merge a switchover through. Only unset/'blacksmith' selects
# Blacksmith; anything unrecognized selects GitHub so a typo can't queue jobs
# against the provider you're escaping. Paid build/browser jobs and their cache actions share
# this predicate. Task planning, the aggregate gate, HTTP acceptance and integration tests use free GitHub runners.
#
# GitHub mode is break-glass, not a peer — cold layers, slower runs. The app image
# is the one job on a paid larger runner: next build needs ~32 GB and OOM-kills
# (exit 137) on the free 16 GB runners at any heap ceiling.

on:
  push:
    branches: [main, staging, dev]
  # No path filter: a pull request whose run never starts never reports `checks / ci`. The
  # `changes` job in checks.yml scopes the jobs to what the diff can affect instead.
  pull_request:
    branches: [main, staging, dev]

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

jobs:
  checks:
    name: checks
    if: github.ref != 'refs/heads/dev' || github.event_name == 'pull_request'
    uses: ./.github/workflows/checks.yml

  # Detect if this is a version release commit (e.g., "v0.5.24: ...")
  # Smallest runner on purpose: a few seconds of pure shell over the commit
  # message, no checkout and no install.
  version:
    name: version
    runs-on: &runner-2vcpu ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
    timeout-minutes: 5
    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev')
    outputs:
      version: ${{ steps.extract.outputs.version }}
      is_release: ${{ steps.extract.outputs.is_release }}
    steps:
      - name: Extract version from commit message
        id: extract
        env:
          COMMIT_MSG: ${{ github.event.head_commit.message }}
        run: |
          # Only tag versions on main branch
          if [ "$GITHUB_REF" = "refs/heads/main" ] && [[ "$COMMIT_MSG" =~ ^(v[0-9]+\.[0-9]+\.[0-9]+): ]]; then
            VERSION="${BASH_REMATCH[1]}"
            echo "version=${VERSION}" >> $GITHUB_OUTPUT
            echo "is_release=true" >> $GITHUB_OUTPUT
            echo "✅ Detected release commit: ${VERSION}"
          else
            echo "version=" >> $GITHUB_OUTPUT
            echo "is_release=false" >> $GITHUB_OUTPUT
            echo "ℹ️ Not a release commit"
          fi

  # Detect shell-code changes on dev/staging pushes. Web-only changes never
  # need a desktop build (installed shells load the web app live); changes to
  # the Electron app or the bridge packages trigger a per-env prerelease build
  # (dev → dev stream, staging → staging) that the env's update feed
  # (/api/desktop/update) starts offering automatically.
  desktop-changes:
    name: desktop-changes
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/staging')
    outputs:
      changed: ${{ steps.diff.outputs.changed }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
        with:
          fetch-depth: 50

      - name: Diff desktop paths
        id: diff
        env:
          BEFORE: ${{ github.event.before }}
        run: |
          # Force pushes (dev resets) can reference a BEFORE we don't have;
          # fall back to the previous commit, and to no build when even that
          # is unavailable.
          if [ -z "$BEFORE" ] || ! git cat-file -e "$BEFORE" 2>/dev/null; then
            BEFORE="$(git rev-parse HEAD^ 2>/dev/null || echo '')"
          fi
          if [ -z "$BEFORE" ]; then
            echo "changed=false" >> "$GITHUB_OUTPUT"
            echo "ℹ️ No comparable base commit; skipping desktop prerelease"
            exit 0
          fi
          if git diff --name-only "$BEFORE" HEAD | grep -qE '^(apps/desktop/|packages/desktop-bridge/|packages/browser-protocol/|\.github/workflows/(ci|desktop-release)\.yml$)'; then
            echo "changed=true" >> "$GITHUB_OUTPUT"
            echo "✅ Desktop shell code changed"
          else
            echo "changed=false" >> "$GITHUB_OUTPUT"
            echo "ℹ️ No desktop shell changes"
          fi

  # Run database migrations before images are promoted: the ECR latest/staging
  # tag push triggers CodePipeline, so migrating first guarantees the schema is
  # in place before the new app version deploys (replaces the removed ECS
  # migration sidecar)
  migrate:
    permissions:
      contents: read
    name: migrate
    needs: [checks]
    # Explicit need results instead of the implicit success(): a skipped job
    # anywhere in the transitive needs chain silently fails implicit success()
    # and cascade-skips the deploy chain (migrate -> promote ->
    # CodeDeploy) — this bit us on 2026-07-23. State requirements explicitly.
    if: >-
      !cancelled() &&
      needs.checks.result == 'success' &&
      github.event_name == 'push' &&
      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
    uses: ./.github/workflows/migrate.yml
    with:
      environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
    secrets:
      DATABASE_URL: ${{ secrets.DATABASE_URL }}
      STAGING_DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
      MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }}
      STAGING_MIGRATION_DATABASE_URL: ${{ secrets.STAGING_MIGRATION_DATABASE_URL }}

  # Same ordering for dev (schema push before the dev image lands in ECR)
  migrate-dev:
    permissions:
      contents: read
      id-token: write
    name: migrate-dev
    if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
    uses: ./.github/workflows/migrate.yml
    with:
      environment: dev
    secrets:
      DEV_DATABASE_URL: ${{ secrets.DEV_DATABASE_URL }}

  # Dev: build immutable images alongside the schema push and Trigger upload.
  image-dev:
    name: image-dev (${{ matrix.id }})
    if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
    runs-on: &runner-matrix ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.bs_runner || matrix.gh_runner }}
    timeout-minutes: 30
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        include:
          # Only the app image needs a large runner: next build exhausts the free
          # 16 GB one (exit 137). The others build in <5 min and idle at 12-15%
          # CPU on 8 vCPU, so they stay on the smaller tiers.
          #
          # 16 vCPU on Blacksmith because this build is the critical path to a
          # deploy — nothing ships until the image is pushed — and its two
          # dominant steps both scale with cores (`bun install` ~300-400s, `next
          # build` ~260s). The same `next build` runs on 16 vCPU in the separate
          # checks `build` job, which does not gate anything; this one
          # was doing comparable work on half the cores.
          #
          # cache_mb is the layer cache the post-job prune retains, and it is the
          # only reason the sticky disks stay bounded — see the image action's
          # action.yml. Rows that omit it take the small-image default there. The
          # app image overrides because it carries ~34 layers plus apt and bun
          # cache mounts for the whole monorepo; 100 GB is several builds' worth
          # of headroom over that working set.
          - id: app
            dockerfile: ./docker/app.Dockerfile
            cache_mb: '102400'
            ecr_repo_secret: ECR_APP
            gh_runner: linux-x64-8-core
            bs_runner: blacksmith-16vcpu-ubuntu-2404
          - id: db
            dockerfile: ./docker/db.Dockerfile
            ecr_repo_secret: ECR_MIGRATIONS
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
          - id: realtime
            dockerfile: ./docker/realtime.Dockerfile
            ecr_repo_secret: ECR_REALTIME
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
          - id: pii
            dockerfile: ./docker/pii.Dockerfile
            ecr_repo_secret: ECR_PII
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
        with:
          role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ secrets.DEV_AWS_REGION }}

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2

      - name: Login to Docker Hub
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Resolve ECR repo name
        id: ecr-repo
        run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
        env:
          ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}

      - name: Build and push
        uses: ./.github/actions/image
        with:
          provider: ${{ vars.CI_PROVIDER }}
          file: ${{ matrix.dockerfile }}
          platforms: linux/amd64
          tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev
          build-args: |
            MSHIP_PLAN_MODE_DEFAULT=true
          max-cache-size-mb: ${{ matrix.cache_mb }}

  # Staging/production coordinate task releases with app traffic cutover.
  # Dev tasks deploy independently in trigger-dev.yml.
  trigger-upload:
    name: trigger-upload
    if: >-
      github.event_name == 'push' &&
      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
    runs-on: &runner-4vcpu ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
    timeout-minutes: 30
    outputs:
      version: ${{ steps.deploy.outputs.deploymentVersion }}
      environment: ${{ steps.target.outputs.environment }}
      preview_branch: ${{ steps.target.outputs.preview_branch }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Setup workspace
        uses: ./.github/actions/setup
        with:
          provider: ${{ vars.CI_PROVIDER }}
          node-version: ''

      - name: Select Trigger environment
        id: target
        run: |
          case "$GITHUB_REF" in
            refs/heads/main) TRIGGER_ENV=prod; TRIGGER_BRANCH='' ;;
            refs/heads/staging) TRIGGER_ENV=staging; TRIGGER_BRANCH='' ;;
            *) echo "ERROR: unsupported Trigger release ref: $GITHUB_REF" >&2; exit 1 ;;
          esac
          echo "environment=$TRIGGER_ENV" >> "$GITHUB_OUTPUT"
          echo "preview_branch=$TRIGGER_BRANCH" >> "$GITHUB_OUTPUT"

      - name: Upload Trigger.dev version without promotion
        id: deploy
        working-directory: ./apps/sim
        env:
          TRIGGER_ACCESS_TOKEN: ${{ (steps.target.outputs.environment == 'prod' && secrets.TRIGGER_ACCESS_TOKEN_PROD) || (steps.target.outputs.environment == 'staging' && secrets.TRIGGER_ACCESS_TOKEN_STAGING) || secrets.TRIGGER_ACCESS_TOKEN }}
          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
          TRIGGER_ENV: ${{ steps.target.outputs.environment }}
          TRIGGER_BRANCH: ${{ steps.target.outputs.preview_branch }}
        run: |
          set -eo pipefail
          if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then
            echo "ERROR: TRIGGER_PROJECT_ID and a Trigger.dev access token (TRIGGER_ACCESS_TOKEN_${TRIGGER_ENV^^} or TRIGGER_ACCESS_TOKEN) repo secrets must be set" >&2
            exit 1
          fi
          TARGET_ARGS=(--env "$TRIGGER_ENV")
          if [ -n "$TRIGGER_BRANCH" ]; then
            TARGET_ARGS+=(--branch "$TRIGGER_BRANCH")
          fi
          bunx trigger.dev@4.8.2 deploy "${TARGET_ARGS[@]}" --skip-promotion

      - name: Validate deployment version output
        env:
          VERSION: ${{ steps.deploy.outputs.deploymentVersion }}
        run: |
          if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then
            echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2
            exit 1
          fi

  # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR.
  # Runs in parallel with tests — only immutable sha tags are pushed here, and
  # the CodePipeline EventBridge triggers filter on exactly the
  # latest/staging/dev ECR tags, so nothing deploys and no mutable tag moves
  # until promote / ghcr retag after the gate.
  image-amd64:
    name: image-amd64 (${{ matrix.id }})
    if: >-
      github.event_name == 'push' &&
      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
    runs-on: *runner-matrix
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        include:
          - id: app
            dockerfile: ./docker/app.Dockerfile
            cache_mb: '102400'
            ghcr_image: ghcr.io/simstudioai/simstudio
            ecr_repo_secret: ECR_APP
            gh_runner: linux-x64-8-core
            bs_runner: blacksmith-16vcpu-ubuntu-2404
          - id: db
            dockerfile: ./docker/db.Dockerfile
            ghcr_image: ghcr.io/simstudioai/migrations
            ecr_repo_secret: ECR_MIGRATIONS
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
          - id: realtime
            dockerfile: ./docker/realtime.Dockerfile
            ghcr_image: ghcr.io/simstudioai/realtime
            ecr_repo_secret: ECR_REALTIME
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
          - id: pii
            dockerfile: ./docker/pii.Dockerfile
            ghcr_image: ghcr.io/simstudioai/pii
            ecr_repo_secret: ECR_PII
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
          # No ECR repo is provisioned for cron, so it publishes to GHCR only.
          # The tag step below omits the ECR tag when the repo name is empty.
          - id: cron
            dockerfile: ./docker/cron.Dockerfile
            ghcr_image: ghcr.io/simstudioai/cron
            gh_runner: ubuntu-latest
            bs_runner: blacksmith-2vcpu-ubuntu-2404
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
        with:
          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }}

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2

      - name: Login to Docker Hub
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GHCR
        if: github.ref == 'refs/heads/main'
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Resolve ECR repo name
        id: ecr-repo
        run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
        env:
          ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}

      # Only sha tags here — the ECR deploy tags (latest/staging) are applied
      # by promote and the GHCR latest-amd64/version tags by
      # ghcr, both after tests and migrations pass.
      - name: Generate tags
        id: meta
        run: |
          ECR_REGISTRY="${{ steps.login-ecr.outputs.registry }}"
          ECR_REPO="${{ steps.ecr-repo.outputs.name }}"
          GHCR_IMAGE="${{ matrix.ghcr_image }}"

          TAGS=""
          if [ -n "$ECR_REPO" ]; then
            TAGS="${ECR_REGISTRY}/${ECR_REPO}:${{ github.sha }}"
          fi

          if [ "${{ github.ref }}" = "refs/heads/main" ] && [ -n "$GHCR_IMAGE" ]; then
            if [ -n "$TAGS" ]; then
              TAGS="${TAGS},${GHCR_IMAGE}:${{ github.sha }}-amd64"
            else
              TAGS="${GHCR_IMAGE}:${{ github.sha }}-amd64"
            fi
          fi

          # An entry can legitimately resolve to no tags — e.g. the cron image has
          # no ECR repo, so on staging/dev (where GHCR tags are not applied) there
          # is nothing to push. Skip that build instead of failing the job.
          if [ -z "$TAGS" ]; then
            echo "No ECR repo and no GHCR tag for this entry on ${{ github.ref }} — skipping push."
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

          echo "tags=${TAGS}" >> $GITHUB_OUTPUT

      - name: Build and push images
        if: steps.meta.outputs.skip != 'true'
        uses: ./.github/actions/image
        with:
          provider: ${{ vars.CI_PROVIDER }}
          file: ${{ matrix.dockerfile }}
          platforms: linux/amd64
          tags: ${{ steps.meta.outputs.tags }}
          max-cache-size-mb: ${{ matrix.cache_mb }}

  # Promote the sha-tagged ECR images once their build and migrations pass.
  # Staging/production also require the Trigger upload; dev never waits for it.
  # Pushing the ECR latest/staging/dev tag is what triggers
  # CodePipeline, so this seconds-long manifest retag is the deploy gate —
  # the image builds themselves run in parallel with the tests. A single job
  # (not a matrix) so all four sha manifests are verified before any tag
  # moves; a missing image can't produce a partial mixed-version deploy.
  promote:
    name: promote
    needs: [migrate, image-amd64, trigger-upload, migrate-dev, image-dev]
    # Explicit results: see migrate's comment.
    if: >-
      !cancelled() && github.event_name == 'push' &&
      (
        ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') &&
          needs.trigger-upload.result == 'success' &&
          needs.migrate.result == 'success' &&
          needs.image-amd64.result == 'success') ||
        (github.ref == 'refs/heads/dev' &&
          needs.migrate-dev.result == 'success' &&
          needs.image-dev.result == 'success')
      )
    runs-on: *runner-2vcpu
    timeout-minutes: 10
    permissions:
      contents: read
      id-token: write
    outputs:
      # Whether the deploy tag was actually moved (false on a stale-run guard
      # skip). trigger-promote keys off this so tasks are never promoted when
      # the app itself wasn't.
      promoted: ${{ steps.guard.outputs.fresh }}
      # Epoch when the deploy tag was retagged (this push's ECS pipeline trigger).
      # trigger-promote passes it to the poll script so a stale pipeline execution
      # reusing the same image digest can't satisfy the cutover gate.
      retag_epoch: ${{ steps.promote.outputs.retag_epoch }}
      # Unchanged tags verify the latest execution's cutover without an epoch bound.
      app_image_changed: ${{ steps.promote.outputs.app_image_changed }}
      app_image_digest: ${{ steps.promote.outputs.app_image_digest }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
        with:
          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_REGION || secrets.STAGING_AWS_REGION }}

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@03f1aad4c6c7ffd436567f42f9384779290529bd # v2

      # Deploy-tag moves must be monotonic: a re-run of an old run must never
      # retag latest/staging back to stale code. A superseded first-attempt
      # run still promotes — the ci-<ref> concurrency group executes runs
      # serially in commit order, so an ancestor of head is a forward deploy.
      - name: Guard against stale promotion
        id: guard
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          STATUS="$(gh api "repos/${{ github.repository }}/compare/${{ github.sha }}...${GITHUB_REF_NAME}" --jq '.status' || echo "unknown")"
          if [ "$STATUS" = "identical" ] || { [ "$STATUS" = "ahead" ] && [ "${{ github.run_attempt }}" = "1" ]; }; then
            echo "fresh=true" >> $GITHUB_OUTPUT
          else
            echo "::warning::Skipping promotion of ${{ github.sha }} (branch compare: ${STATUS}, attempt ${{ github.run_attempt }}). Moving the deploy tags here could deploy stale code; push a revert commit to roll back instead."
            echo "fresh=false" >> $GITHUB_OUTPUT
          fi

      # Fail before moving tags if the live cutover observer lacks permission.
      # Requires codepipeline:GetPipelineState on the app pipeline.
      - name: Verify pipeline state access
        if: steps.guard.outputs.fresh == 'true'
        env:
          PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || github.ref == 'refs/heads/dev' && 'dev' || 'staging' }}-us-east-1-app-deployment
        run: aws codepipeline get-pipeline-state --name "$PIPELINE" --query pipelineName --output text > /dev/null

      - name: Promote images to deploy tags
        id: promote
        if: steps.guard.outputs.fresh == 'true'
        env:
          SOURCE_TAG: ${{ github.ref == 'refs/heads/dev' && format('{0}-dev', github.sha) || github.sha }}
          ECR_REPOS: >-
            ${{ secrets.ECR_MIGRATIONS }}
            ${{ secrets.ECR_REALTIME }}
            ${{ secrets.ECR_PII }}
            ${{ secrets.ECR_APP }}
        run: |
          REGISTRY="${{ steps.login-ecr.outputs.registry }}"

          RELEASE_TAG=""
          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
            ECR_TAG="latest"
            # ECR lifecycle retains release- aliases as prod rollback targets.
            RELEASE_TAG="release-${SOURCE_TAG}"
          elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then
            ECR_TAG="dev"
          else
            ECR_TAG="staging"
          fi

          APP_REPO="${{ secrets.ECR_APP }}"

          # Verify every sha image exists before moving any deploy tag, so a
          # missing/expired image aborts the whole promotion up front.
          for repo in $ECR_REPOS; do
            echo "🔍 Verifying ${repo}:${SOURCE_TAG}"
            docker buildx imagetools inspect "${REGISTRY}/${repo}:${SOURCE_TAG}" > /dev/null
          done

          # Move the app last so a preceding tag failure cannot start app rollout
          # and then skip the downstream Trigger promotion job.
          for repo in $ECR_REPOS; do
            echo "🚀 Promoting ${repo}:${SOURCE_TAG} to ${ECR_TAG}${RELEASE_TAG:+ and ${RELEASE_TAG}}"
            if [ "$repo" = "$APP_REPO" ]; then
              bash .github/scripts/promote-app-image.sh "$REGISTRY" "$APP_REPO" "$SOURCE_TAG" "$ECR_TAG" "$RELEASE_TAG"
            else
              bash .github/scripts/promote-image.sh "$REGISTRY" "$repo" "$SOURCE_TAG" "$ECR_TAG" "$RELEASE_TAG"
            fi
          done

  # Staging/production: promote the parked Trigger.dev version after observing the ECS
  # traffic cutover (CodeDeploy AllowTraffic on every target). The image retag
  # triggers the ECS pipeline; this job correlates it via the digest + retag epoch
  # (rejecting a stale execution reusing the digest) and promotes at cutover.
  # Skipped when promote skipped the tag move (stale run) — tasks then
  # correctly stay on the old version. If the app deploy fails or never cuts over,
  # promote never fires and this job fails visibly.
  trigger-promote:
    name: trigger-promote
    needs: [promote, trigger-upload]
    # Explicit results also suppress skip propagation from optional ancestors.
    if: >-
      !cancelled() &&
      github.event_name == 'push' &&
      (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') &&
      needs.promote.result == 'success' &&
      needs.trigger-upload.result == 'success' &&
      needs.promote.outputs.promoted == 'true'
    outputs:
      # The new app serves traffic; indexnow keys off this, not the later Trigger.dev promotion.
      cutover: ${{ steps.cutover.outcome }}
    # Mostly waiting on the ECS cutover after a dependency install: the smallest runner is enough.
    runs-on: *runner-2vcpu
    # Leave setup/promotion headroom above the 70-minute cutover poll.
    timeout-minutes: 90
    permissions:
      contents: read
      id-token: write
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Setup workspace
        uses: ./.github/actions/setup
        with:
          provider: ${{ vars.CI_PROVIDER }}
          node-version: ''

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
        with:
          role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_REGION || secrets.STAGING_AWS_REGION }}
          role-duration-seconds: 5400

      # An unchanged tag may belong to a failed or still-running earlier deploy.
      # Verify its latest cutover rather than treating tag equality as success.
      - name: Wait for ECS traffic cutover
        id: cutover
        env:
          OVERALL_TIMEOUT: 4200
          APP_IMAGE_CHANGED: ${{ needs.promote.outputs.app_image_changed }}
          DIGEST: ${{ needs.promote.outputs.app_image_digest }}
          PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || github.ref == 'refs/heads/dev' && 'dev' || 'staging' }}-us-east-1-app-deployment
          RETAG_EPOCH: ${{ needs.promote.outputs.retag_epoch }}
        run: |
          set -eo pipefail
          case "$APP_IMAGE_CHANGED" in
            true) ;;
            false) RETAG_EPOCH=0 ;;
            *) echo "ERROR: invalid app image change metadata" >&2; exit 1 ;;
          esac
          bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH"

      - name: Promote Trigger.dev version
        working-directory: ./apps/sim
        env:
          TRIGGER_ACCESS_TOKEN: ${{ (needs.trigger-upload.outputs.environment == 'prod' && secrets.TRIGGER_ACCESS_TOKEN_PROD) || (needs.trigger-upload.outputs.environment == 'staging' && secrets.TRIGGER_ACCESS_TOKEN_STAGING) || secrets.TRIGGER_ACCESS_TOKEN }}
          TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }}
          TRIGGER_ENV: ${{ needs.trigger-upload.outputs.environment }}
          TRIGGER_BRANCH: ${{ needs.trigger-upload.outputs.preview_branch }}
          VERSION: ${{ needs.trigger-upload.outputs.version }}
        run: |
          set -eo pipefail
          if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then
            echo "ERROR: TRIGGER_PROJECT_ID and a Trigger.dev access token (TRIGGER_ACCESS_TOKEN_${TRIGGER_ENV^^} or TRIGGER_ACCESS_TOKEN) repo secrets must be set" >&2
            exit 1
          fi
          if [ -z "$VERSION" ]; then
            echo "ERROR: no deployed version passed from trigger-upload" >&2
            exit 1
          fi
          echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV)"
          TARGET_ARGS=(--env "$TRIGGER_ENV")
          if [ -n "$TRIGGER_BRANCH" ]; then
            TARGET_ARGS+=(--branch "$TRIGGER_BRANCH")
          fi
          bunx trigger.dev@4.8.2 promote "$VERSION" "${TARGET_ARGS[@]}"

  # Production only, once the new app serves traffic: submit the library and comparison URLs this
  # push changed to IndexNow. Best effort: nothing depends on it, and a failure never fails the run.
  indexnow:
    name: indexnow
    needs: [promote, trigger-promote]
    if: >-
      !cancelled() &&
      github.event_name == 'push' &&
      github.ref == 'refs/heads/main' &&
      needs.promote.outputs.promoted == 'true' &&
      needs.trigger-promote.outputs.cutover == 'success'
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    continue-on-error: true
    permissions:
      contents: read
      actions: read
      checks: read
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
        with:
          bun-version: 1.4.4

      # Diff from the last earlier main commit whose `indexnow` check succeeded, not the previous
      # push: a push that never deployed, or whose submission failed, would otherwise drop its
      # changed URLs for good. The check-runs lookup asks for this one check per commit, so it
      # stays small and counts every attempt of a re-run; a lookup that errors counts as no
      # success. With no success in view, the base is the first parent of the oldest run in view,
      # so URLs that no submission ever covered, that run's own included, are sent now; a failed
      # parent lookup fails the step, so the next run retries from the same base. The diff
      # needs only the two trees, so that one commit is fetched alone; an all-zero `before` (the
      # branch's first push) has no base, which the script skips.
      - name: Submit changed URLs to IndexNow
        env:
          GH_TOKEN: ${{ github.token }}
          PUSH_BEFORE: ${{ github.event.before }}
        run: |
          BASE_SHA=""
          OLDEST_SHA=""
          while read -r head_sha; do
            OLDEST_SHA="$head_sha"
            if succeeded="$(gh api "repos/$GITHUB_REPOSITORY/commits/$head_sha/check-runs?check_name=indexnow&filter=all" \
              --jq '[.check_runs[] | select(.conclusion == "success")] | length')" && [[ "$succeeded" -gt 0 ]]; then
              BASE_SHA="$head_sha"
              break
            fi
          done < <(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=30" \
            --jq ".workflow_runs[] | select(.id < $GITHUB_RUN_ID) | .head_sha")
          if [[ -z "$BASE_SHA" && -n "$OLDEST_SHA" ]]; then
            BASE_SHA="$(gh api "repos/$GITHUB_REPOSITORY/commits/$OLDEST_SHA" --jq '.parents[0].sha // empty')"
          fi
          BASE_SHA="${BASE_SHA:-$PUSH_BEFORE}"
          echo "Submitting URLs changed in $BASE_SHA..$GITHUB_SHA"
          if [[ ! "$BASE_SHA" =~ ^0+$ ]]; then
            git fetch --no-tags --depth=1 origin "$BASE_SHA"
          fi
          bun scripts/ping-indexnow.ts "$BASE_SHA" "$GITHUB_SHA"

  # Build ARM64 images for GHCR (main branch only, runs in parallel with
  # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64
  # are applied by ghcr after the gate, so a failing run
  # never moves a documented tag.
  image-arm64:
    name: image-arm64 (${{ matrix.id }})
    runs-on: *runner-matrix
    timeout-minutes: 30
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        # Non-app images sit at 4 vCPU rather than the finer x64 split: the ARM
        # sizing data is job-level (8 -> 4 for the whole matrix), not per-image,
        # and this job only runs on push to main — an unprovisioned label would
        # hang a release in `queued` rather than fail a PR.
        include:
          - id: app
            dockerfile: ./docker/app.Dockerfile
            cache_mb: '102400'
            image: ghcr.io/simstudioai/simstudio
            gh_runner: linux-arm64-8-core
            bs_runner: blacksmith-16vcpu-ubuntu-2404-arm
          - id: db
            dockerfile: ./docker/db.Dockerfile
            image: ghcr.io/simstudioai/migrations
            gh_runner: ubuntu-24.04-arm
            bs_runner: blacksmith-4vcpu-ubuntu-2404-arm
          - id: realtime
            dockerfile: ./docker/realtime.Dockerfile
            image: ghcr.io/simstudioai/realtime
            gh_runner: ubuntu-24.04-arm
            bs_runner: blacksmith-4vcpu-ubuntu-2404-arm
          - id: pii
            dockerfile: ./docker/pii.Dockerfile
            image: ghcr.io/simstudioai/pii
            gh_runner: ubuntu-24.04-arm
            bs_runner: blacksmith-4vcpu-ubuntu-2404-arm
          - id: cron
            dockerfile: ./docker/cron.Dockerfile
            image: ghcr.io/simstudioai/cron
            gh_runner: ubuntu-24.04-arm
            bs_runner: blacksmith-4vcpu-ubuntu-2404-arm

    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Login to GHCR
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push ARM64 to GHCR
        uses: ./.github/actions/image
        with:
          provider: ${{ vars.CI_PROVIDER }}
          file: ${{ matrix.dockerfile }}
          platforms: linux/arm64
          tags: ${{ matrix.image }}:${{ github.sha }}-arm64
          max-cache-size-mb: ${{ matrix.cache_mb }}

  # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags)
  # and the multi-arch manifests from the immutable sha tags — only on main,
  # after the deploy gate (promote) and the ARM64 build both pass.
  ghcr:
    name: ghcr (${{ matrix.id }})
    runs-on: *runner-2vcpu
    timeout-minutes: 10
    needs: [promote, image-arm64, version]
    # Explicit results: see migrate's comment.
    if: >-
      !cancelled() &&
      needs.promote.result == 'success' &&
      needs.image-arm64.result == 'success' &&
      needs.version.result == 'success' &&
      github.event_name == 'push' && github.ref == 'refs/heads/main'
    permissions:
      contents: read
      packages: write
    # Every matrix leg evaluates the same guard against the same commit, so the
    # value is identical whichever leg reports it last. attest-subjects needs it
    # to tell "the guard withheld latest" from "the registry read was stale".
    outputs:
      latest_fresh: ${{ steps.guard.outputs.fresh }}
    strategy:
      matrix:
        include:
          - id: simstudio
            image: ghcr.io/simstudioai/simstudio
          - id: migrations
            image: ghcr.io/simstudioai/migrations
          - id: realtime
            image: ghcr.io/simstudioai/realtime
          - id: pii
            image: ghcr.io/simstudioai/pii
          - id: cron
            image: ghcr.io/simstudioai/cron

    steps:
      - name: Login to GHCR
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # Same monotonic guard as promote, applied to the public latest
      # tags only — immutable sha and version tags are always published.
      - name: Guard against stale latest tags
        id: guard
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          STATUS="$(gh api "repos/${{ github.repository }}/compare/${{ github.sha }}...${GITHUB_REF_NAME}" --jq '.status' || echo "unknown")"
          if [ "$STATUS" = "identical" ] || { [ "$STATUS" = "ahead" ] && [ "${{ github.run_attempt }}" = "1" ]; }; then
            echo "fresh=true" >> $GITHUB_OUTPUT
          else
            echo "::warning::Publishing immutable tags for ${{ github.sha }} but skipping the latest tags (branch compare: ${STATUS}, attempt ${{ github.run_attempt }})."
            echo "fresh=false" >> $GITHUB_OUTPUT
          fi

      - name: Publish tags and manifests
        run: |
          IMAGE="${{ matrix.image }}"
          SHA="${{ github.sha }}"

          # Multi-arch manifest from the immutable per-arch sha tags
          docker buildx imagetools create -t "${IMAGE}:${SHA}" \
            "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"

          if [ "${{ needs.version.outputs.is_release }}" = "true" ]; then
            VERSION="${{ needs.version.outputs.version }}"
            echo "📦 Publishing version tags: ${VERSION}"
            docker buildx imagetools create -t "${IMAGE}:${VERSION}-amd64" "${IMAGE}:${SHA}-amd64"
            docker buildx imagetools create -t "${IMAGE}:${VERSION}-arm64" "${IMAGE}:${SHA}-arm64"
            docker buildx imagetools create -t "${IMAGE}:${VERSION}" \
              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
          fi

          if [ "${{ steps.guard.outputs.fresh }}" = "true" ]; then
            docker buildx imagetools create -t "${IMAGE}:latest-amd64" "${IMAGE}:${SHA}-amd64"
            docker buildx imagetools create -t "${IMAGE}:latest-arm64" "${IMAGE}:${SHA}-arm64"
            docker buildx imagetools create -t "${IMAGE}:latest" \
              "${IMAGE}:${SHA}-amd64" "${IMAGE}:${SHA}-arm64"
          fi

  # Sign the published images and attach SLSA provenance and an SBOM to each.
  #
  # This runs after ghcr rather than inside the build because
  # buildx's own provenance/sbom attestations stay off (see the note in
  # .github/actions/image): the extra manifests they add to an index
  # break the `imagetools create` retagging that promote depends on.
  # Attaching attestations here instead leaves the index itself untouched — they
  # are stored as separate referrer manifests that point at it.
  #
  # Resolve the set of digests that actually got published, so the attestation
  # job below covers every tag a customer can pull.
  #
  # A static list is not enough. `imagetools create` always writes an INDEX, so
  # `:<version>-amd64` is a single-entry index whose digest differs from the
  # `:<sha>-amd64` manifest it wraps — attesting the manifest leaves the tag
  # people actually pin unverifiable. Which tags exist also varies per run:
  # version tags only on a release, and the latest tags only when the monotonic
  # guard in ghcr passed. Resolving tag -> digest here and
  # de-duplicating is what keeps the two in step without hardcoding that logic
  # twice.
  attest-subjects:
    name: attest-subjects
    runs-on: *runner-2vcpu
    timeout-minutes: 10
    needs: [ghcr, version]
    if: >-
      !cancelled() &&
      needs.ghcr.result == 'success' &&
      needs.version.result == 'success' &&
      github.event_name == 'push' && github.ref == 'refs/heads/main'
    permissions:
      contents: read
      packages: read
    outputs:
      subjects: ${{ steps.resolve.outputs.subjects }}
      count: ${{ steps.resolve.outputs.count }}
    steps:
      - name: Login to GHCR
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Resolve published tags to digests
        id: resolve
        env:
          IS_RELEASE: ${{ needs.version.outputs.is_release }}
          VERSION: ${{ needs.version.outputs.version }}
          SHA: ${{ github.sha }}
          LATEST_FRESH: ${{ needs.ghcr.outputs.latest_fresh }}
        run: |
          set -euo pipefail

          IMAGES="simstudio migrations realtime pii cron"

          # Prints the digest, or nothing when the tag is genuinely absent.
          #
          # An absent tag and a registry hiccup both make `inspect` fail, and
          # treating them alike is how a published image silently ends up
          # unsigned while this job still goes green. So: retry, and only report
          # "absent" when the registry actually says the manifest is unknown.
          # Anything else fails the step.
          digest_of() {
            local ref="$1" attempt raw err absent=0
            for attempt in 1 2 3; do
              if raw="$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest}}' 2>/tmp/inspect.err)"; then
                printf '%s' "$raw" | jq -r '.digest // empty'
                return 0
              fi
              err="$(cat /tmp/inspect.err)"
              # Absence is retried like any other failure: GHCR can report a
              # just-published alias as unknown for a moment, and accepting that
              # on the first attempt would skip a tag this run did publish.
              case "$err" in
                *"not found"*|*MANIFEST_UNKNOWN*|*"no such manifest"*|*"NAME_UNKNOWN"*) absent=1 ;;
                *) absent=0 ;;
              esac
              if [ "$attempt" -lt 3 ]; then
                sleep "$((attempt * 3))"
              fi
            done
            # Only call it absent if the registry said so on the final attempt.
            if [ "$absent" -eq 1 ]; then
              return 0
            fi
            echo "::error::Could not inspect ${ref} after 3 attempts: ${err}" >&2
            return 1
          }

          # Records a subject. `platform` tells the attestation job whether this
          # digest is a single-architecture image, which is the only case where
          # a Syft SBOM describes what the puller actually gets.
          emit() {
            jq -nc --arg image "$1" --arg digest "$2" --arg platform "$3" \
              '{image: $image, digest: $digest, platform: $platform}' >> /tmp/subjects.jsonl
          }

          : > /tmp/subjects.jsonl
          for name in $IMAGES; do
            image="ghcr.io/simstudioai/${name}"

            # The sha tags are this run's own output. All three must resolve —
            # a missing one means the publish did not complete, not that the tag
            # is optional.
            seen=""
            sha_index=""
            for tag in "${SHA}" "${SHA}-amd64" "${SHA}-arm64"; do
              digest="$(digest_of "${image}:${tag}")"
              if [ -z "$digest" ]; then
                echo "::error::${image}:${tag} was not published by this run"
                exit 1
              fi
              case "$tag" in
                *-amd64) platform=amd64 ;;
                *-arm64) platform=arm64 ;;
                *) platform=index; sha_index="$digest" ;;
              esac
              seen="$seen $digest"
              emit "$image" "$digest" "$platform"
            done

            # A moving alias is only taken when it resolves to the same index
            # digest this run published — content identity, which is what a
            # digest can prove. ghcr holds the latest tags back when
            # its monotonic guard sees a newer commit, and they then still point
            # at an older build — attesting those would put this run's signature
            # and provenance on an image it did not produce. The per-arch
            # aliases are published in the same guarded block as `latest`, so
            # that one comparison gates all three.
            alias_groups="latest"
            if [ "${IS_RELEASE}" = "true" ]; then
              alias_groups="${alias_groups} ${VERSION}"
            fi

            for alias in $alias_groups; do
              # A mismatch has two very different causes: the guard deliberately
              # held the tag back, or GHCR is still serving the previous digest
              # moments after this run wrote it. Re-read before concluding the
              # former, or a read landing a second early silently drops three
              # subjects from the matrix.
              alias_index=""
              for alias_attempt in 1 2 3; do
                alias_index="$(digest_of "${image}:${alias}")"
                [ "$alias_index" = "$sha_index" ] && break
                [ "$alias_attempt" -lt 3 ] && sleep 5 || true
              done

              if [ "$alias_index" != "$sha_index" ]; then
                # `latest` is allowed to lag, but only when the guard actually
                # withheld it. If the guard published latest this run, a mismatch
                # here is a stale read, not a deliberate skip — and silently
                # dropping it would leave a published tag unsigned.
                if [ "$alias" = "latest" ]; then
                  if [ "${LATEST_FRESH}" = "true" ]; then
                    echo "::error::${image}:latest was published by this run but resolves to ${alias_index:-nothing}"
                    exit 1
                  fi
                  echo "Skipping latest* for ${image}: the monotonic guard withheld it this run."
                  continue
                fi
                # A version tag has no such carve-out. This run published it, so
                # a release must not ship a version image nothing has attested.
                echo "::error::${image}:${alias} does not resolve to this run's index (${sha_index:-none}); refusing to publish an unattested release image"
                exit 1
              fi
              for tag in "${alias}" "${alias}-amd64" "${alias}-arm64"; do
                digest="$(digest_of "${image}:${tag}")"
                if [ -z "$digest" ]; then
                  echo "::error::${image}:${tag} is missing though ${image}:${alias} is current"
                  exit 1
                fi
                case " $seen " in *" $digest "*) continue ;; esac
                seen="$seen $digest"
                case "$tag" in
                  *-amd64) emit "$image" "$digest" amd64 ;;
                  *-arm64) emit "$image" "$digest" arm64 ;;
                  *) emit "$image" "$digest" index ;;
                esac
              done
            done
          done

          if [ ! -s /tmp/subjects.jsonl ]; then
            echo "::error::Resolved no image digests to attest"
            exit 1
          fi

          echo "Resolved $(wc -l < /tmp/subjects.jsonl) distinct subjects:"
          cat /tmp/subjects.jsonl
          echo "subjects=$(jq -sc . /tmp/subjects.jsonl)" >> "$GITHUB_OUTPUT"
          echo "count=$(wc -l < /tmp/subjects.jsonl | tr -d ' ')" >> "$GITHUB_OUTPUT"

  # One leg per distinct published digest. Attesting each subject separately is
  # also what makes the SBOMs truthful: the amd64 and arm64 images contain
  # different packages, and one SBOM attached to the index cannot describe both.
  attest:
    name: attest
    runs-on: *runner-2vcpu
    timeout-minutes: 15
    needs: [attest-subjects]
    if: >-
      !cancelled() &&
      needs.attest-subjects.result == 'success'
    permissions:
      contents: read
      packages: write
      # Sigstore signs against the runner's OIDC identity; no key material is stored.
      id-token: write
      attestations: write
      # Records each attestation against the image as an artifact storage record.
      artifact-metadata: write
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.attest-subjects.outputs.subjects) }}

    steps:
      - name: Login to GHCR
        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # Skipped for index subjects: Syft resolves an index to one platform, so
      # the SBOM it produces would describe amd64 while the index also serves
      # arm64. The per-arch subjects below carry an accurate SBOM each, and the
      # index still gets a signature and provenance.
      #
      # Scanned by the `<sha>-<arch>` tag rather than by `matrix.digest`. Half
      # the per-arch subjects are single-entry INDEXES (`imagetools create`
      # writes an index even from one manifest), and Syft resolves an index
      # against the RUNNER's platform — so an arm64-only index fails outright on
      # an amd64 runner with "no child with platform linux/arm64". The sha tag is
      # the plain manifest that index wraps: identical content, no platform
      # resolution, and one pull shared by both subjects instead of two.
      - name: Generate SBOM
        if: matrix.platform != 'index'
        uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3
        with:
          image: ${{ matrix.image }}:${{ github.sha }}-${{ matrix.platform }}
          format: spdx-json
          output-file: sbom.spdx.json
          # The action's own release upload is for workflows triggered by a
          # release; these attach to the image instead.
          upload-artifact: false
          upload-release-assets: false

      - name: Attest SBOM
        if: matrix.platform != 'index'
        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
        with:
          subject-name: ${{ matrix.image }}
          subject-digest: ${{ matrix.digest }}
          sbom-path: sbom.spdx.json
          # Stored alongside the image so a mirrored registry carries the
          # attestation with it, rather than only being retrievable from GitHub.
          push-to-registry: true

      - name: Attest build provenance
        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
        with:
          subject-name: ${{ matrix.image }}
          subject-digest: ${{ matrix.digest }}
          push-to-registry: true

      - name: Install Cosign
        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2

      # The attestations above prove how the image was built; this is the plain
      # signature that admission controllers (Kyverno, the Sigstore policy
      # controller) verify before admitting a pod.
      - name: Sign image
        run: cosign sign --yes "${{ matrix.image }}@${{ matrix.digest }}"

  # Check if docs changed
  # Smallest runner on purpose: a depth-2 checkout plus a path filter, no
  # install and no build.
  docs-changes:
    name: docs-changes
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    outputs:
      docs_changed: ${{ steps.filter.outputs.docs }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
        with:
          fetch-depth: 2 # Need at least 2 commits to detect changes
      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4
        id: filter
        with:
          filters: |
            docs:
              - 'apps/docs/content/**'
              - 'apps/docs/components/workflow-preview/examples.ts'
              - 'apps/docs/components/workflow-preview/block-display-workflows.ts'
              - 'apps/docs/scripts/compile-content.ts'
              - 'apps/docs/scripts/index-search.ts'
              - 'apps/docs/lib/content/**'
              - 'apps/docs/lib/search/**'
              - 'apps/docs/lib/openapi-specs.ts'
              - 'apps/docs/lib/urls.ts'
              - 'apps/docs/openapi*.json'
              - 'apps/docs/package.json'
              - 'bun.lock'
              - '.github/workflows/docs.yml'
              - '.github/workflows/ci.yml'

  # Process docs embeddings (only when docs change, after images are promoted)
  docs:
    name: docs
    needs: [promote, docs-changes]
    # Explicit results: see migrate's comment.
    if: >-
      !cancelled() &&
      needs.promote.result == 'success' &&
      needs.docs-changes.result == 'success' &&
      needs.docs-changes.outputs.docs_changed == 'true'
    uses: ./.github/workflows/docs.yml
    secrets:
      DATABASE_URL: ${{ secrets.DATABASE_URL }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

  # Create GitHub Release (only for version commits on main, after all builds complete)
  release:
    name: release
    runs-on: *runner-4vcpu
    timeout-minutes: 10
    needs: [ghcr, attest-subjects, attest, version]
    # Explicit results: see migrate's comment. attest is a gate, not just
    # an ordering edge — a release must not advertise images whose signature or
    # attestation failed to publish. The count check is belt and braces: today
    # attest-subjects already fails on an empty subject set, so this only bites
    # if that guard is ever removed.
    #
    # Note this gates the GitHub release, not the production deploy: CodePipeline
    # fires from the ECR tags moved by promote, upstream of this job, and
    # only the GHCR mirrors are attested.
    if: >-
      !cancelled() &&
      needs.ghcr.result == 'success' &&
      needs.attest-subjects.result == 'success' &&
      needs.attest-subjects.outputs.count != '0' &&
      needs.attest.result == 'success' &&
      needs.version.result == 'success' &&
      needs.version.outputs.is_release == 'true'
    permissions:
      contents: write
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
        with:
          fetch-depth: 0

      - name: Setup workspace
        uses: ./.github/actions/setup
        with:
          provider: ${{ vars.CI_PROVIDER }}
          node-version: ''

      - name: Create release
        env:
          GH_PAT: ${{ secrets.GITHUB_TOKEN }}
        run: bun run scripts/create-single-release.ts ${{ needs.version.outputs.version }}

  # Desktop release: builds, signs, notarizes, and attaches the macOS app to
  # the GitHub release created above. Gated on the Apple signing secrets so a
  # release pipeline run skips cleanly (instead of failing) until the Apple
  # Developer account is provisioned — the moment the six secrets exist, the
  # next vX.Y.Z release ships desktop artifacts with no further changes.
  # Job-level `if:` cannot read the secrets context, hence the probe job.
  desktop-signing:
    name: desktop-signing
    runs-on: *runner-2vcpu
    timeout-minutes: 2
    needs: [version, desktop-changes]
    # !cancelled(): desktop-changes is skipped on main (and
    # version tags only on main); either path may need the probe.
    if: ${{ !cancelled() && (needs.version.outputs.is_release == 'true' || needs.desktop-changes.outputs.changed == 'true') }}
    outputs:
      configured: ${{ steps.check.outputs.configured }}
    steps:
      - name: Probe Apple signing secrets
        id: check
        env:
          CONFIGURED: ${{ secrets.CSC_LINK != '' && secrets.CSC_KEY_PASSWORD != '' && secrets.APPLE_API_KEY_P8 != '' && secrets.APPLE_API_KEY_ID != '' && secrets.APPLE_API_ISSUER != '' && secrets.APPLE_TEAM_ID != '' }}
        run: |
          echo "configured=${CONFIGURED}" >> "$GITHUB_OUTPUT"
          if [ "$CONFIGURED" != "true" ]; then
            echo "::warning::Desktop release skipped: Apple signing secrets are not configured (CSC_LINK, CSC_KEY_PASSWORD, APPLE_API_KEY_P8, APPLE_API_KEY_ID, APPLE_API_ISSUER, APPLE_TEAM_ID)."
          fi

  desktop-release:
    name: desktop-release
    needs: [release, desktop-signing, version]
    # Suppress the implicit success() check: desktop-signing has an
    # intentionally skipped transitive dependency on main, which would
    # otherwise cascade-skip this job even when every direct need succeeded.
    if: >-
      !cancelled() &&
      needs.release.result == 'success' &&
      needs.desktop-signing.result == 'success' &&
      needs.version.result == 'success' &&
      needs.desktop-signing.outputs.configured == 'true'
    permissions:
      contents: write
    uses: ./.github/workflows/desktop-release.yml
    with:
      version: ${{ needs.version.outputs.version }}
      publish: true
    secrets:
      DESKTOP_RELEASE_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
      CSC_LINK: ${{ secrets.CSC_LINK }}
      CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
      APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }}
      APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
      APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
      APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}

  # Per-env desktop prereleases: a dev/staging push that touches shell code
  # publishes an environment-tagged GitHub prerelease to the public,
  # release-only simstudioai/sim-desktop-releases repository. Keeping these
  # builds out of this source repository prevents its followers from receiving
  # every internal shell release. Each environment's /api/desktop/update feed
  # still offers only its own signed stream.
  prerelease-create:
    name: prerelease-create
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    needs: [desktop-changes, desktop-signing]
    # Requires the signing probe to have actually succeeded (not just "not
    # cancelled") so a probe failure can't produce a release with no build.
    if: >-
      ${{
        !cancelled() &&
        needs.desktop-changes.outputs.changed == 'true' &&
        needs.desktop-signing.result == 'success' &&
        needs.desktop-signing.outputs.configured == 'true'
      }}
    permissions:
      contents: read
    outputs:
      version: ${{ steps.version.outputs.version }}
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6

      - name: Compute prerelease version and create draft release
        id: version
        env:
          DESKTOP_RELEASE_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
          GH_TOKEN: ${{ github.token }}
          PRERELEASE_REPOSITORY: simstudioai/sim-desktop-releases
          SOURCE_REPOSITORY: ${{ github.repository }}
        run: |
          if [ -z "$DESKTOP_RELEASE_TOKEN" ]; then
            echo "::error::DESKTOP_RELEASE_TOKEN is required to publish desktop prereleases."
            exit 1
          fi
          if [ "$GITHUB_REF" = "refs/heads/dev" ]; then CHANNEL=dev; else CHANNEL=staging; fi
          # Prerelease core = next patch after the latest stable release, so
          # channel builds always outrank the stable they are built on top of
          # and are always superseded by the next stable. The run-attempt
          # suffix keeps re-runs of the same workflow from colliding on the
          # tag while preserving semver ordering.
          # Fail loudly if the query itself fails: silently falling back to
          # v0.0.0 would publish a channel build that sorts below the shipped
          # stable, and installed shells would never see it as an update.
          if ! LATEST="$(gh release list --repo "$SOURCE_REPOSITORY" --exclude-pre-releases --limit 1 --json tagName --jq '.[0].tagName')"; then
            echo "::error::Could not query the latest stable release."
            exit 1
          fi
          # An empty release list makes jq print "null", which ${VAR:-default}
          # does not treat as empty. Anything that is not a bare vX.Y.Z means
          # "no stable release to build on top of" — start the channel at 0.0.1.
          if [[ ! "$LATEST" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            LATEST="v0.0.0"
          fi
          IFS='.' read -r MAJOR MINOR PATCH <<< "${LATEST#v}"
          TAG="v${MAJOR}.${MINOR}.$((PATCH + 1))-${CHANNEL}.${GITHUB_RUN_NUMBER}.${GITHUB_RUN_ATTEMPT}"
          NOTES="Automated ${CHANNEL}-channel desktop build from ${GITHUB_REF_NAME} @ ${GITHUB_SHA::7}."
          # Draft until the build uploads its artifacts: drafts are invisible
          # to the update feed, so a failed or in-flight build can never take
          # the channel down with an assetless release. The release-only repo
          # has no source commit for this SHA, so its tag intentionally targets
          # that repository's main branch; the notes retain the source SHA.
          GH_TOKEN="$DESKTOP_RELEASE_TOKEN" gh release create "$TAG" \
            --repo "$PRERELEASE_REPOSITORY" \
            --draft \
            --prerelease \
            --target main \
            --title "$TAG" \
            --notes "$NOTES"
          echo "version=$TAG" >> "$GITHUB_OUTPUT"
          echo "✅ Created draft prerelease $TAG"

  prerelease-build:
    name: prerelease-build
    needs: [prerelease-create, desktop-signing]
    # The reusable workflow declares contents: write for its stable-release
    # path. GitHub cannot elevate a caller's token, even though this prerelease
    # path uses the dedicated cross-repository token for its actual upload.
    permissions:
      contents: write
    uses: ./.github/workflows/desktop-release.yml
    with:
      version: ${{ needs.prerelease-create.outputs.version }}
      publish: true
      sign: true
    secrets:
      DESKTOP_RELEASE_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
      CSC_LINK: ${{ secrets.CSC_LINK }}
      CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
      APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }}
      APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
      APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
      APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}

  # The draft only becomes visible to the update feed once its artifacts are
  # attached — this is what makes a dev/staging push atomic from the shell's
  # point of view.
  prerelease-publish:
    name: prerelease-publish
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    needs: [prerelease-create, prerelease-build]
    permissions:
      contents: read
    env:
      GH_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
      GH_REPO: simstudioai/sim-desktop-releases
      TAG: ${{ needs.prerelease-create.outputs.version }}
    steps:
      - name: Publish the draft release
        run: |
          if [ -z "$GH_TOKEN" ]; then
            echo "::error::DESKTOP_RELEASE_TOKEN is required to publish desktop prereleases."
            exit 1
          fi
          gh release edit "$TAG" --draft=false

  # Keep the release list tidy: per channel, retain the newest 5 prereleases
  # and delete the rest (with their tags, so dev force-resets don't strand
  # commits behind stale tags). Leftover drafts (failed or superseded builds)
  # are always garbage by this point — the current run's release is published.
  prerelease-prune:
    name: prerelease-prune
    runs-on: *runner-2vcpu
    timeout-minutes: 5
    needs: [prerelease-create, prerelease-publish]
    permissions:
      contents: read
    env:
      GH_TOKEN: ${{ secrets.DESKTOP_RELEASE_TOKEN }}
      GH_REPO: simstudioai/sim-desktop-releases
      CURRENT_TAG: ${{ needs.prerelease-create.outputs.version }}
    steps:
      - name: Delete stale prereleases
        run: |
          set -euo pipefail
          : "${CURRENT_TAG:?Current publication tag is required before pruning}"
          if [ -z "$GH_TOKEN" ]; then
            echo "::error::DESKTOP_RELEASE_TOKEN is required to prune desktop prereleases."
            exit 1
          fi
          if [ "$GITHUB_REF" = "refs/heads/dev" ]; then CHANNELS='(dev|alpha)'; else CHANNELS='(staging|beta)'; fi
          # createdAt follows the tag's commit: published releases in the release-only
          # repository can all share it. Retain by publication time, with deterministic ties.
          gh release list --limit 100 --json tagName,isPrerelease,isDraft,publishedAt \
            --jq "[.[] | select(.isPrerelease and (.isDraft | not) and (.tagName | test(\"-${CHANNELS}\\\\.\")))] | sort_by(.publishedAt, .tagName) | reverse | .[5:] | .[].tagName" |
          while read -r TAG; do
            [ -n "$TAG" ] || continue
            [ "$TAG" != "$CURRENT_TAG" ] || continue
            echo "Deleting stale prerelease $TAG"
            gh release delete "$TAG" --cleanup-tag --yes
          done

      - name: Delete leftover draft prereleases
        run: |
          if [ "$GITHUB_REF" = "refs/heads/dev" ]; then CHANNELS='(dev|alpha)'; else CHANNELS='(staging|beta)'; fi
          # Drafts have no tag ref, so delete by release id via the API
          # (gh release delete resolves by tag, which is ambiguous for drafts).
          gh api "repos/${GH_REPO}/releases?per_page=100" \
            --jq ".[] | select(.draft and (.tag_name | test(\"-${CHANNELS}\\\\.\"))) | .id" |
          while read -r ID; do
            [ -n "$ID" ] || continue
            echo "Deleting leftover draft release $ID"
            gh api -X DELETE "repos/${GH_REPO}/releases/${ID}"
          done