diff --git a/.docker/web/nextcloud.conf b/.docker/web/nextcloud.conf index 401e537..39ef1fd 100644 --- a/.docker/web/nextcloud.conf +++ b/.docker/web/nextcloud.conf @@ -1,6 +1,8 @@ upstream php-handler { - server app:9000; - #server unix:/run/php/php8.2-fpm.sock; + zone php-handler 64k; + resolver 127.0.0.11 valid=10s ipv6=off; + resolver_timeout 5s; + server app:9000 resolve; } # Set the `immutable` cache control options only for assets with a cache busting `v` argument diff --git a/.github/actions/build-and-scan/action.yml b/.github/actions/build-and-scan/action.yml index f16457c..42a32e8 100644 --- a/.github/actions/build-and-scan/action.yml +++ b/.github/actions/build-and-scan/action.yml @@ -16,7 +16,6 @@ runs: using: composite steps: - name: Setup Bats - if: inputs.component == 'app' uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0 with: support-install: false @@ -32,6 +31,8 @@ runs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + # Refresh OS security updates: invalidate the app runtime stage and + # rebuild the small web image so cached apt/apk upgrades cannot go stale. - name: Build app image (linux/amd64) if: inputs.component == 'app' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -42,6 +43,7 @@ runs: build-args: | NEXTCLOUD_BASE_IMAGE=${{ inputs.nextcloud_base_image }} tags: scan/app:amd64 + no-cache-filters: runtime cache-from: type=gha,scope=app-${{ inputs.nextcloud_version }}-amd64 cache-to: type=gha,mode=max,scope=app-${{ inputs.nextcloud_version }}-amd64 @@ -55,6 +57,7 @@ runs: build-args: | NEXTCLOUD_BASE_IMAGE=${{ inputs.nextcloud_base_image }} tags: scan/app:arm64 + no-cache-filters: runtime cache-from: type=gha,scope=app-${{ inputs.nextcloud_version }}-arm64 cache-to: type=gha,mode=max,scope=app-${{ inputs.nextcloud_version }}-arm64 @@ -66,6 +69,7 @@ runs: platforms: linux/amd64 load: true tags: scan/web:amd64 + no-cache: true cache-from: type=gha,scope=web-amd64 cache-to: type=gha,mode=max,scope=web-amd64 @@ -77,6 +81,7 @@ runs: platforms: linux/arm64 load: true tags: scan/web:arm64 + no-cache: true cache-from: type=gha,scope=web-arm64 cache-to: type=gha,mode=max,scope=web-arm64 @@ -108,11 +113,18 @@ runs: shell: bash env: APP_IMAGE: scan/app:amd64 - run: bats tests/app-image.bats + run: make test-app-image - name: Runtime acceptance app image (linux/arm64) if: inputs.component == 'app' shell: bash env: APP_IMAGE: scan/app:arm64 - run: bats tests/app-image.bats + run: make test-app-image + + - name: Runtime acceptance web image (linux/amd64) + if: inputs.component == 'web' + shell: bash + env: + WEB_IMAGE: scan/web:amd64 + run: make test-web-image diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c802473..c0f5bcf 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -41,9 +41,6 @@ jobs: - name: Run offline regression tests run: make test-ci-scripts - - name: Test Trivy wrapper - run: bash tests/test-scan-images.sh - - name: ShellCheck image publication scripts run: | shellcheck scripts/nextcloud-versions.sh scripts/publish-images.sh \ diff --git a/.github/workflows/ncdd-cli.yml b/.github/workflows/ncdd-cli.yml index ed91a9f..e9e4ead 100644 --- a/.github/workflows/ncdd-cli.yml +++ b/.github/workflows/ncdd-cli.yml @@ -5,7 +5,7 @@ on: paths: - 'bin/ncdd' - 'lib/ncdd-test.sh' - - 'tests/ncdd.bats' + - 'tests/cli/**' - 'docs/ncdd.md' - 'AGENTS.md' - 'Makefile' @@ -20,7 +20,7 @@ on: paths: - 'bin/ncdd' - 'lib/ncdd-test.sh' - - 'tests/ncdd.bats' + - 'tests/cli/**' - 'docs/ncdd.md' - 'AGENTS.md' - 'Makefile' @@ -48,4 +48,4 @@ jobs: run: shellcheck -x -P SCRIPTDIR bin/ncdd lib/ncdd-test.sh - name: Test - run: bats tests/ncdd.bats \ No newline at end of file + run: make test-ncdd \ No newline at end of file diff --git a/.github/workflows/nextcloud-development.yml b/.github/workflows/nextcloud-development.yml index a97ac8c..1067a96 100644 --- a/.github/workflows/nextcloud-development.yml +++ b/.github/workflows/nextcloud-development.yml @@ -6,27 +6,27 @@ on: - main paths: - '.docker/app/**' - - 'tests/app-image.bats' + - 'tests/app/**' - 'scripts/scan-images.sh' - 'trivy.yaml' - '.github/workflows/nextcloud-development.yml' - 'scripts/resolve-nextcloud-upstream.sh' - 'scripts/push-master-architecture.sh' - 'scripts/publish-master-manifest.sh' - - 'tests/nextcloud-development-image.bats' + - 'tests/development/**' push: branches: - main paths: - '.docker/app/**' - - 'tests/app-image.bats' + - 'tests/app/**' - 'scripts/scan-images.sh' - 'trivy.yaml' - '.github/workflows/nextcloud-development.yml' - 'scripts/resolve-nextcloud-upstream.sh' - 'scripts/push-master-architecture.sh' - 'scripts/publish-master-manifest.sh' - - 'tests/nextcloud-development-image.bats' + - 'tests/development/**' schedule: - cron: '17 3 * * *' workflow_dispatch: @@ -63,7 +63,7 @@ jobs: file-install: false - name: Test development image scripts - run: bats tests/nextcloud-development-image.bats + run: make test-development-image-scripts validate: needs: @@ -150,7 +150,7 @@ jobs: shell: bash env: APP_IMAGE: scan/master:${{ matrix.arch }} - run: bats tests/app-image.bats + run: make test-app-image - name: Log in to GitHub Container Registry if: github.event_name != 'pull_request' diff --git a/Makefile b/Makefile index 410f5ed..94c24ea 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ APP_TEST_IMAGE ?= nextcloud-app:acceptance # Maintainer builds use the latest reviewed upstream base, not the deployment image channel. APP_BASE_IMAGE ?= $(shell jq -er '.versions[-1].image' .github/nextcloud-supported.json) -.PHONY: up-garages3 down-garages3 bootstrap-garages3 garage-status-garages3 start-garages3 wait-nextcloud-garages3 setup-garages3 test-scan-images test-ci-scripts test-development-image-scripts test-ncdd test-app-image test-current-app-image scan-images +.PHONY: up-garages3 down-garages3 bootstrap-garages3 garage-status-garages3 start-garages3 wait-nextcloud-garages3 setup-garages3 test-scan-images test-ci-scripts test-development-image-scripts test-ncdd test-app-image test-web-image test-current-app-image scan-images up-garages3: $(COMPOSE) -f $(GARAGES3_COMPOSE_FILE) up -d garage @@ -30,23 +30,26 @@ setup-garages3: $(MAKE) wait-nextcloud-garages3 test-scan-images: - bash tests/test-scan-images.sh + bats tests/ci/scan-images.bats +# Each Bats suite auto-discovers tests from its category directory. +# New tests do not require any Makefile or CI workflow changes. test-ci-scripts: - bats tests/check-action-pins.bats \ - tests/compose-images.bats \ - tests/nextcloud-versions.bats \ - tests/publish-images.bats + bats tests/ci/*.bats test-development-image-scripts: - bats tests/nextcloud-development-image.bats + bats tests/development/*.bats test-ncdd: - bats tests/ncdd.bats + bats tests/cli/*.bats test-app-image: @test -n "$(APP_IMAGE)" || { echo 'Usage: make test-app-image APP_IMAGE=' >&2; exit 2; } - APP_IMAGE="$(APP_IMAGE)" bats tests/app-image.bats + APP_IMAGE="$(APP_IMAGE)" bats tests/app/*.bats + +test-web-image: + @test -n "$(WEB_IMAGE)" || { echo 'Usage: make test-web-image WEB_IMAGE=' >&2; exit 2; } + WEB_IMAGE="$(WEB_IMAGE)" bats tests/web/*.bats test-current-app-image: @set -e; \ diff --git a/docs/maintenance.md b/docs/maintenance.md index 79624a1..6c67302 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -74,8 +74,10 @@ The repository uses Bash, AWK, `jq`, and Bats rather than a separate Python impl ```bash make test-ci-scripts make test-development-image-scripts -bash tests/test-scan-images.sh +make test-ncdd +make test-scan-images make test-app-image APP_IMAGE=scan/app:amd64 +make test-web-image WEB_IMAGE=scan/web:amd64 make test-current-app-image ``` @@ -83,6 +85,12 @@ The wiki parser regression tests include real historical snapshots from 2023 to The runtime acceptance test uses an isolated Docker network and PostgreSQL container, verifies Nextcloud installation and `occ` status, and checks the PHP-FPM interface. Test-created resources are cleaned up. CI must run these tests against the images actually built by the workflow. +The Trivy wrapper tests are part of the offline CI suite. Bats tests are grouped into `tests/ci/` (offline checks), `tests/development/`, +`tests/cli/`, `tests/app/`, and `tests/web/`. The Makefile discovers every +`*.bats` file in the corresponding directory. Add tests there without changing +GitHub Actions or maintaining a file list. The app and web suites require +locally built images passed through `APP_IMAGE` and `WEB_IMAGE`. + For YAML syntax, Shell scripts, and publication changes, use `actionlint`, `ShellCheck`, and the Bats suite before requesting review. Do not weaken vulnerability gates to pass CI. ## Other maintenance decisions diff --git a/tests/app-image.bats b/tests/app/app-image.bats similarity index 100% rename from tests/app-image.bats rename to tests/app/app-image.bats diff --git a/tests/check-action-pins.bats b/tests/ci/check-action-pins.bats similarity index 98% rename from tests/check-action-pins.bats rename to tests/ci/check-action-pins.bats index 04262bb..1aa0c16 100644 --- a/tests/check-action-pins.bats +++ b/tests/ci/check-action-pins.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats setup() { - cd "$BATS_TEST_DIRNAME/.." + cd "$BATS_TEST_DIRNAME/../.." } check_pins() { diff --git a/tests/compose-images.bats b/tests/ci/compose-images.bats similarity index 97% rename from tests/compose-images.bats rename to tests/ci/compose-images.bats index 409711b..b99114a 100644 --- a/tests/compose-images.bats +++ b/tests/ci/compose-images.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats setup() { - REPO_ROOT=$(cd "$BATS_TEST_DIRNAME/.." && pwd) + REPO_ROOT=$(cd "$BATS_TEST_DIRNAME/../.." && pwd) } assert_published_images() { diff --git a/tests/nextcloud-versions.bats b/tests/ci/nextcloud-versions.bats similarity index 99% rename from tests/nextcloud-versions.bats rename to tests/ci/nextcloud-versions.bats index ab5a7ee..85ef8f5 100644 --- a/tests/nextcloud-versions.bats +++ b/tests/ci/nextcloud-versions.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # Offline regression tests for the official wiki and reviewed JSON format. setup() { - cd "$BATS_TEST_DIRNAME/.." + cd "$BATS_TEST_DIRNAME/../.." TEST_ROOT=$(mktemp -d) export TEST_ROOT mkdir -p "$TEST_ROOT/bin" diff --git a/tests/publish-images.bats b/tests/ci/publish-images.bats similarity index 99% rename from tests/publish-images.bats rename to tests/ci/publish-images.bats index 36d26b6..628f107 100644 --- a/tests/publish-images.bats +++ b/tests/ci/publish-images.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # Docker is mocked; these tests never push real images to a registry. setup() { - cd "$BATS_TEST_DIRNAME/.." + cd "$BATS_TEST_DIRNAME/../.." TEST_ROOT=$(mktemp -d) export TEST_ROOT mkdir -p "$TEST_ROOT/bin" diff --git a/tests/ci/scan-images.bats b/tests/ci/scan-images.bats new file mode 100644 index 0000000..44a1f2a --- /dev/null +++ b/tests/ci/scan-images.bats @@ -0,0 +1,96 @@ +#!/usr/bin/env bats + +setup() { + repo_root="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + tmp_root="$(mktemp -d)" + bin_dir="$tmp_root/bin" + report_dir="$tmp_root/reports" + log_file="$tmp_root/trivy.log" + mkdir -p "$bin_dir" "$report_dir" + + cat > "$bin_dir/trivy" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail + +args=("$@") +platform='' +image='' +output='' +format='' +for ((i = 0; i < ${#args[@]}; i++)); do + case "${args[$i]}" in + --platform) + ((i + 1 < ${#args[@]})) || exit 90 + platform="${args[$((i + 1))]}" + ;; + --format) + ((i + 1 < ${#args[@]})) || exit 91 + format="${args[$((i + 1))]}" + ;; + --output) + ((i + 1 < ${#args[@]})) || exit 92 + output="${args[$((i + 1))]}" + ;; + esac +done +image="${args[-1]}" + +printf '%s|%s|%s\n' "$platform" "$format" "$image" >> "$TRIVY_TEST_LOG" + +if [[ "$platform" != "$TRIVY_EXPECTED_PLATFORM" ]]; then + printf 'unexpected platform: %s\n' "$platform" >&2 + exit 93 +fi + +if [[ "$format" == sarif ]]; then + [[ -n "$output" ]] || exit 94 + printf '{"version":"2.1.0"}\n' > "$output" +fi + +if [[ -n "${TRIVY_FAIL_IMAGE:-}" && "$image" == "$TRIVY_FAIL_IMAGE" ]]; then + exit 1 +fi +MOCK + chmod +x "$bin_dir/trivy" + + export TRIVY_BIN="$bin_dir/trivy" + export TRIVY_TEST_LOG="$log_file" + export TRIVY_REPORT_DIR="$report_dir" +} + +teardown() { + rm -rf "$tmp_root" +} + +@test "amd64 scans produce table and SARIF results" { + export TRIVY_EXPECTED_PLATFORM=linux/amd64 + run bash "$repo_root/scripts/scan-images.sh" 'app@linux/amd64=example/app:amd64' + [ "$status" -eq 0 ] + [ -f "$report_dir/app-linux-amd64.sarif" ] + grep -Fqx 'linux/amd64|table|example/app:amd64' "$log_file" + grep -Fqx 'linux/amd64|sarif|example/app:amd64' "$log_file" +} + +@test "arm64 scans produce table and SARIF results" { + export TRIVY_EXPECTED_PLATFORM=linux/arm64 + run bash "$repo_root/scripts/scan-images.sh" 'app@linux/arm64=example/app:arm64' + [ "$status" -eq 0 ] + [ -f "$report_dir/app-linux-arm64.sarif" ] + grep -Fqx 'linux/arm64|table|example/app:arm64' "$log_file" + grep -Fqx 'linux/arm64|sarif|example/app:arm64' "$log_file" +} + +@test "a Trivy failure blocks the security scan" { + export TRIVY_EXPECTED_PLATFORM=linux/arm64 + export TRIVY_FAIL_IMAGE=example/app:arm64 + run bash "$repo_root/scripts/scan-images.sh" 'app-failure@linux/arm64=example/app:arm64' + [ "$status" -ne 0 ] +} + +@test "unsupported architectures are rejected before invoking Trivy" { + export TRIVY_EXPECTED_PLATFORM=linux/amd64 + run bash "$repo_root/scripts/scan-images.sh" 'app@linux/s390x=example/app:amd64' + [ "$status" -ne 0 ] + [[ "$output" == *"Unsupported platform"* ]] + [ ! -s "$log_file" ] +} diff --git a/tests/ncdd.bats b/tests/cli/ncdd.bats similarity index 98% rename from tests/ncdd.bats rename to tests/cli/ncdd.bats index aa899b5..75b68d7 100644 --- a/tests/ncdd.bats +++ b/tests/cli/ncdd.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats setup() { - REPO_ROOT=$(cd "$BATS_TEST_DIRNAME/.." && pwd) + REPO_ROOT=$(cd "$BATS_TEST_DIRNAME/../.." && pwd) TMP_ROOT=$(mktemp -d) mkdir -p "$TMP_ROOT/bin" export NCDD_TEST_LOG="$TMP_ROOT/docker.log" diff --git a/tests/nextcloud-development-image.bats b/tests/development/nextcloud-development-image.bats similarity index 100% rename from tests/nextcloud-development-image.bats rename to tests/development/nextcloud-development-image.bats diff --git a/tests/test-scan-images.sh b/tests/test-scan-images.sh deleted file mode 100644 index f551dfe..0000000 --- a/tests/test-scan-images.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -tmp_root=$(mktemp -d) -trap 'rm -rf "$tmp_root"' EXIT - -bin_dir="$tmp_root/bin" -report_dir="$tmp_root/reports" -log_file="$tmp_root/trivy.log" -mkdir -p "$bin_dir" "$report_dir" - -cat > "$bin_dir/trivy" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail - -args=("$@") -platform='' -image='' -output='' -format='' -for ((i = 0; i < ${#args[@]}; i++)); do - case "${args[$i]}" in - --platform) - ((i + 1 < ${#args[@]})) || exit 90 - platform="${args[$((i + 1))]}" - ;; - --format) - ((i + 1 < ${#args[@]})) || exit 91 - format="${args[$((i + 1))]}" - ;; - --output) - ((i + 1 < ${#args[@]})) || exit 92 - output="${args[$((i + 1))]}" - ;; - esac -done -image="${args[-1]}" - -printf '%s|%s|%s\n' "$platform" "$format" "$image" >> "$TRIVY_TEST_LOG" - -if [[ "$platform" != "$TRIVY_EXPECTED_PLATFORM" ]]; then - printf 'unexpected platform: %s\n' "$platform" >&2 - exit 93 -fi - -if [[ "$format" == sarif ]]; then - [[ -n "$output" ]] || exit 94 - printf '{"version":"2.1.0"}\n' > "$output" -fi - -if [[ -n "${TRIVY_FAIL_IMAGE:-}" && "$image" == "$TRIVY_FAIL_IMAGE" ]]; then - exit 1 -fi -EOF -chmod +x "$bin_dir/trivy" - -export TRIVY_BIN="$bin_dir/trivy" -export TRIVY_TEST_LOG="$log_file" -export TRIVY_REPORT_DIR="$report_dir" -export TRIVY_EXPECTED_PLATFORM=linux/amd64 - -bash "$repo_root/scripts/scan-images.sh" \ - 'app@linux/amd64=example/app:amd64' -test -f "$report_dir/app-linux-amd64.sarif" -grep -Fqx 'linux/amd64|table|example/app:amd64' "$log_file" -grep -Fqx 'linux/amd64|sarif|example/app:amd64' "$log_file" - -export TRIVY_EXPECTED_PLATFORM=linux/arm64 -bash "$repo_root/scripts/scan-images.sh" \ - 'app@linux/arm64=example/app:arm64' -test -f "$report_dir/app-linux-arm64.sarif" -grep -Fqx 'linux/arm64|table|example/app:arm64' "$log_file" -grep -Fqx 'linux/arm64|sarif|example/app:arm64' "$log_file" - -if TRIVY_EXPECTED_PLATFORM=linux/arm64 TRIVY_FAIL_IMAGE=example/app:arm64 \ - bash "$repo_root/scripts/scan-images.sh" \ - 'app-failure@linux/arm64=example/app:arm64'; then - printf 'expected a Trivy failure to make the helper fail\n' >&2 - exit 1 -fi - -if TRIVY_EXPECTED_PLATFORM=linux/amd64 \ - bash "$repo_root/scripts/scan-images.sh" \ - 'app@linux/s390x=example/app:amd64'; then - printf 'expected an unsupported architecture to be rejected\n' >&2 - exit 1 -fi - -echo 'scan-images tests passed' diff --git a/tests/web/nginx-dns-recovery.bats b/tests/web/nginx-dns-recovery.bats new file mode 100644 index 0000000..44c38ea --- /dev/null +++ b/tests/web/nginx-dns-recovery.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats + +setup() { + : "${WEB_IMAGE:?Set WEB_IMAGE to the built Nextcloud web image}" + FPM_IMAGE="${FPM_IMAGE:-php:8.4-fpm-alpine}" + test_id="nc-fpm-dns-${BATS_TEST_NUMBER}-$$" + network="$test_id" + web="$test_id-web" + app="$test_id-app" + holder="$test_id-holder" + workdir="$(mktemp -d)" + + mkdir -p "$workdir/htdocs" "$workdir/includes" + chmod 755 "$workdir" + printf '%s\n' " "$workdir/htdocs/status.php" + + # Docker only permits --ip on networks with a user-configured subnet. + # Let Docker select an available subnet instead of hardcoding an address. + allocator="$network-allocator" + docker network create "$allocator" >/dev/null + subnet="$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' "$allocator")" + docker network rm "$allocator" >/dev/null + docker network create --subnet "$subnet" "$network" >/dev/null +} + +teardown() { + docker rm -f "$web" "$app" "$holder" >/dev/null 2>&1 || true + docker network rm "$network" >/dev/null 2>&1 || true + docker network rm "$allocator" >/dev/null 2>&1 || true + if [[ -n "${workdir:-}" ]]; then + rm -rf "$workdir" + fi +} + +start_app() { + docker run -d --name "$app" --network "$network" --network-alias app \ + --volume "$workdir/htdocs:/var/www/html:ro" "$FPM_IMAGE" >/dev/null +} + +wait_for_php() { + local response attempt + for ((attempt = 0; attempt < 30; attempt++)); do + if response="$(docker exec "$web" wget -q -T 3 -O - http://127.0.0.1/status.php 2>/dev/null)" \ + && [[ "$response" == "healthy" ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +assert_healthy() { + if ! wait_for_php; then + docker logs --tail 25 "$web" >&2 + echo "nginx did not recover its PHP-FPM connection" >&2 + return 1 + fi +} + +@test "nginx reconnects to a recreated PHP-FPM at a different IP without reload" { + start_app + docker run -d --name "$web" --network "$network" \ + --volume "$workdir/htdocs:/var/www/html:ro" \ + --volume "$workdir/includes:/etc/nginx/conf.d/includes:ro" \ + "$WEB_IMAGE" >/dev/null + + run docker exec "$web" nginx -t + [ "$status" -eq 0 ] || { echo "$output" >&2; return 1; } + assert_healthy + + old_ip="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$app")" + web_start="$(docker inspect --format '{{.State.StartedAt}}' "$web")" + docker rm -f "$app" >/dev/null + + # Occupy the previous address so Docker allocates another one to PHP-FPM. + docker run -d --name "$holder" --network "$network" --ip "$old_ip" \ + "$FPM_IMAGE" sleep 180 >/dev/null + start_app + + new_ip="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$app")" + [ "$old_ip" != "$new_ip" ] || { echo "PHP-FPM IP did not change" >&2; return 1; } + assert_healthy + [ "$web_start" = "$(docker inspect --format '{{.State.StartedAt}}' "$web")" ] || { + echo "nginx container unexpectedly restarted" >&2 + return 1 + } +}