From f8c188a9042c8dc0b6c9de2ef2b7face5f24a8e4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 08:51:51 -0700 Subject: [PATCH 1/2] fix(ci): bound a stalled bun install and let the helm test probe wait for routing --- .github/actions/setup/action.yml | 15 ++++++++++- .github/workflows/helm.yml | 9 ++++--- helm/sim/Chart.yaml | 2 +- helm/sim/templates/tests/test-connection.yaml | 25 +++++++++++++------ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8ae0c89dae4..efc0b49d914 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -66,6 +66,19 @@ runs: key: ${{ github.repository }}-${{ inputs.turbo-cache-key }}-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./.turbo + # Bun retries a failed request itself, but a registry connection that stalls + # without failing leaves `bun install` waiting until the job's own timeout + # cancels it, the whole budget spent on one hung step. Healthy installs finish + # in under three minutes cold, so an attempt past five is stalled: kill it and + # retry. Any other failure is real and fails at once. - name: Install dependencies shell: bash - run: bun install --frozen-lockfile --ignore-scripts + run: | + for attempt in 1 2 3; do + timeout --kill-after=30s 5m bun install --frozen-lockfile --ignore-scripts && exit 0 + status=$? + if [ "$status" -ne 124 ] && [ "$status" -ne 137 ]; then exit "$status"; fi + echo "::warning title=bun install stalled::attempt ${attempt} of 3 ran past 5 minutes" + done + echo "::error title=bun install stalled::all 3 attempts ran past 5 minutes" + exit 1 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 9118f417092..5f891b91694 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -188,6 +188,11 @@ jobs: --values helm/sim/ci/kind-overlay.yaml \ --wait --timeout 15m + # `--logs` prints the probe's output, so a failure says which URL refused. + - name: Run helm test + run: helm test sim --namespace sim --timeout 5m --logs + + # After the test, so it also explains a failed probe, not only a failed install. - name: Diagnostics on failure if: failure() run: | @@ -196,9 +201,7 @@ jobs: kubectl -n sim describe pods | tail -100 || true kubectl -n sim logs deploy/sim-app -c migrations --tail=50 || true kubectl -n sim logs deploy/sim-app --tail=80 || true - - - name: Run helm test - run: helm test sim --namespace sim --timeout 5m + kubectl -n sim logs deploy/sim-realtime --tail=80 || true # Resolved once and shared, so the two publish paths cannot package the same # immutable chart version with different appVersions -- they run in parallel, diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index 51d97537731..482676a7a34 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.11.8 +version: 1.11.9 appVersion: "v0.8.26" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/templates/tests/test-connection.yaml b/helm/sim/templates/tests/test-connection.yaml index f05c976ea4f..654c69410f7 100644 --- a/helm/sim/templates/tests/test-connection.yaml +++ b/helm/sim/templates/tests/test-connection.yaml @@ -40,15 +40,26 @@ spec: - -c - | set -eu - APP_URL="http://{{ include "sim.fullname" . }}-app:{{ .Values.app.service.port }}/" - echo "Probing $APP_URL" - wget -q -T {{ .Values.tests.timeoutSeconds }} --tries=1 --spider "$APP_URL" + # A pod turns Ready before every node routes its Service to it, so a + # probe sent the moment `helm install --wait` returns can be refused. + # Each URL gets a minute to answer before the test fails. + probe() { + attempt=1 + while ! wget -q -T {{ .Values.tests.timeoutSeconds }} --spider "$1"; do + if [ "$attempt" -ge 12 ]; then + echo "FAILED: $1 did not answer after $attempt attempts" + return 1 + fi + echo "Attempt $attempt: $1 did not answer; retrying in 5s" + attempt=$((attempt + 1)) + sleep 5 + done + echo "OK: $1" + } + probe "http://{{ include "sim.fullname" . }}-app:{{ .Values.app.service.port }}/" {{- if .Values.realtime.enabled }} - RT_URL="http://{{ include "sim.fullname" . }}-realtime:{{ .Values.realtime.service.port }}/health" - echo "Probing $RT_URL" - wget -q -T {{ .Values.tests.timeoutSeconds }} --tries=1 --spider "$RT_URL" + probe "http://{{ include "sim.fullname" . }}-realtime:{{ .Values.realtime.service.port }}/health" {{- end }} - echo "OK" resources: {{- toYaml .Values.tests.resources | nindent 8 }} {{- end }} From 77827fd356f293a9534b85549715d75342681d4d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 09:14:29 -0700 Subject: [PATCH 2/2] fix(ci): fit the install retry inside every caller and read a failed probe's log from its kept pod --- .github/actions/setup/action.yml | 13 +++++++------ .github/workflows/helm.yml | 8 +++++--- helm/sim/templates/tests/test-connection.yaml | 11 +++++------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index efc0b49d914..575623a6fb9 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -69,16 +69,17 @@ runs: # Bun retries a failed request itself, but a registry connection that stalls # without failing leaves `bun install` waiting until the job's own timeout # cancels it, the whole budget spent on one hung step. Healthy installs finish - # in under three minutes cold, so an attempt past five is stalled: kill it and - # retry. Any other failure is real and fails at once. + # in under two and a half minutes cold, so an attempt past three is stalled: + # kill it and retry once, which leaves room inside the shortest caller's + # 10-minute job timeout. Any other failure is real and fails at once. - name: Install dependencies shell: bash run: | - for attempt in 1 2 3; do - timeout --kill-after=30s 5m bun install --frozen-lockfile --ignore-scripts && exit 0 + for attempt in 1 2; do + timeout --kill-after=15s 3m bun install --frozen-lockfile --ignore-scripts && exit 0 status=$? if [ "$status" -ne 124 ] && [ "$status" -ne 137 ]; then exit "$status"; fi - echo "::warning title=bun install stalled::attempt ${attempt} of 3 ran past 5 minutes" + echo "::warning title=bun install stalled::attempt ${attempt} of 2 ran past 3 minutes" done - echo "::error title=bun install stalled::all 3 attempts ran past 5 minutes" + echo "::error title=bun install stalled::both attempts ran past 3 minutes" exit 1 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 5f891b91694..85fa8dec01d 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -188,11 +188,12 @@ jobs: --values helm/sim/ci/kind-overlay.yaml \ --wait --timeout 15m - # `--logs` prints the probe's output, so a failure says which URL refused. - name: Run helm test - run: helm test sim --namespace sim --timeout 5m --logs + run: helm test sim --namespace sim --timeout 5m - # After the test, so it also explains a failed probe, not only a failed install. + # After the test, so it also explains a failed probe, not only a failed + # install. A failed test pod is kept (the hook deletes it only on success), + # so its log says which URL refused. - name: Diagnostics on failure if: failure() run: | @@ -202,6 +203,7 @@ jobs: kubectl -n sim logs deploy/sim-app -c migrations --tail=50 || true kubectl -n sim logs deploy/sim-app --tail=80 || true kubectl -n sim logs deploy/sim-realtime --tail=80 || true + kubectl -n sim logs sim-test-connection || true # Resolved once and shared, so the two publish paths cannot package the same # immutable chart version with different appVersions -- they run in parallel, diff --git a/helm/sim/templates/tests/test-connection.yaml b/helm/sim/templates/tests/test-connection.yaml index 654c69410f7..12c5dd952fd 100644 --- a/helm/sim/templates/tests/test-connection.yaml +++ b/helm/sim/templates/tests/test-connection.yaml @@ -42,16 +42,15 @@ spec: set -eu # A pod turns Ready before every node routes its Service to it, so a # probe sent the moment `helm install --wait` returns can be refused. - # Each URL gets a minute to answer before the test fails. + # Each URL is retried until a minute has passed, then the test fails. probe() { - attempt=1 + deadline=$(($(date +%s) + 60)) while ! wget -q -T {{ .Values.tests.timeoutSeconds }} --spider "$1"; do - if [ "$attempt" -ge 12 ]; then - echo "FAILED: $1 did not answer after $attempt attempts" + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "FAILED: $1 did not answer within 60s" return 1 fi - echo "Attempt $attempt: $1 did not answer; retrying in 5s" - attempt=$((attempt + 1)) + echo "$1 did not answer; retrying in 5s" sleep 5 done echo "OK: $1"