Make lint enforcing and run the sample tests in CI

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
         run: sudo apt-get install -y shellcheck
       - name: Run shellcheck
         run: |
-          find . -name '*.sh' -type f | xargs shellcheck --severity=warning || true
+          find . -name '*.sh' -type f -print0 | xargs -0 shellcheck --severity=error
 
   terraform-fmt:
     name: Terraform Format
@@ -25,9 +25,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v3
         with:
-          terraform_version: '1.5.0'
+          terraform_version: '1.14.9'
       - name: Check terraform format
-        run: |
-          find . -name '*.tf' -type f -exec dirname {} \; | sort -u | while read dir; do
-            terraform fmt -check "$dir" || true
-          done
+        run: terraform fmt -check -recursive -diff .

.github/workflows/test.yml

@@ -0,0 +1,162 @@
+name: Sample Tests
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    # “At 00:00 on Sunday.”
+    - cron: "0 0 * * 0"
+  workflow_dispatch:
+
+jobs:
+  prepare-list:
+    name: Find testable samples
+    runs-on: ubuntu-latest
+    outputs:
+      scripts: ${{ steps.set-matrix.outputs.scripts }}
+      make: ${{ steps.set-matrix.outputs.make }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: set-matrix
+        run: |
+          # Samples with scripts/deploy.sh and scripts/test.sh.
+          # glue-etl-jobs is skipped for now: Glue tries an SSL connection to RDS first, and an
+          # aborted TLS handshake stops the LocalStack Postgres proxy for that cluster.
+          mapfile -t scripts < <(find . -path '*/scripts/test.sh' | xargs -n1 dirname | xargs -n1 dirname | sed 's|^\./||' | grep -v '^glue-etl-jobs/' | sort)
+          # Samples with a Makefile test-ci target
+          mapfile -t make < <(grep -l '^test-ci:' */Makefile | xargs -n1 dirname | sort)
+          echo "scripts=$(printf '%s\n' "${scripts[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
+          echo "make=$(printf '%s\n' "${make[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
+
+  script-samples:
+    name: Test ${{ matrix.directory }}
+    runs-on: ubuntu-latest
+    needs: prepare-list
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        directory: ${{ fromJSON(needs.prepare-list.outputs.scripts) }}
+    env:
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_DEFAULT_REGION: us-east-1
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install tools
+        run: |
+          pip install awscli-local uv
+          npm install -g @localstack/lstk
+          # The runner ships AWS CLI v2, which treats blob parameters as base64 by default
+          aws configure set cli_binary_format raw-in-base64-out
+
+      - name: Start LocalStack
+        env:
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+        run: lstk start
+
+      - name: Deploy
+        timeout-minutes: 15
+        working-directory: ${{ matrix.directory }}
+        run: bash scripts/deploy.sh
+
+      - name: Test
+        timeout-minutes: 10
+        working-directory: ${{ matrix.directory }}
+        run: bash scripts/test.sh
+
+      - name: Show LocalStack logs
+        if: failure()
+        run: lstk logs
+
+      - name: Send a Slack notification
+        if: failure() || github.event_name != 'pull_request'
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          notification_title: "Sample `${{ matrix.directory }}` ({workflow}) has {status_message}"
+          message_format: "{emoji} *${{ matrix.directory }}* {status_message} in <{repo_url}|{repo}>"
+          footer: "Linked Repo <{repo_url}|{repo}> | <{run_url}|View Workflow run>"
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  make-samples:
+    name: Test ${{ matrix.directory }}
+    runs-on: ubuntu-latest
+    needs: prepare-list
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        directory: ${{ fromJSON(needs.prepare-list.outputs.make) }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: '1.14.9'
+          terraform_wrapper: false
+
+      - name: Install tools
+        run: |
+          pip install localstack awscli-local[ver1] aws-sam-cli-local virtualenv
+          npm install -g @localstack/lstk
+
+      - name: Run make test-ci
+        timeout-minutes: 30
+        working-directory: ${{ matrix.directory }}
+        env:
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+        run: make test-ci
+
+      - name: Upload LocalStack logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.directory }}
+          path: ${{ matrix.directory }}/logs.txt
+          if-no-files-found: ignore
+
+      - name: Send a Slack notification
+        if: failure() || github.event_name != 'pull_request'
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          notification_title: "Sample `${{ matrix.directory }}` ({workflow}) has {status_message}"
+          message_format: "{emoji} *${{ matrix.directory }}* {status_message} in <{repo_url}|{repo}>"
+          footer: "Linked Repo <{repo_url}|{repo}> | <{run_url}|View Workflow run>"
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

apigw-custom-domain/python/terraform/providers.tf

@@ -1,10 +1,10 @@
 terraform {
   required_providers {
     aws = {
-      source  = "hashicorp/aws"
+      source = "hashicorp/aws"
     }
     tls = {
-      source  = "hashicorp/tls"
+      source = "hashicorp/tls"
     }
   }
 }

apigw-websockets/javascript/handler.js

@@ -26,10 +26,11 @@ module.exports.handler = function(event, context, callback) {
   } else {
     // Echo back the event for $default and custom actions
     console.log(`Message received on route: ${routeKey}`);
+    const response = { statusCode: 200, body: JSON.stringify(event) };
     if (callback) {
-      callback(null, event);
+      callback(null, response);
     } else {
-      context.succeed(event);
+      context.succeed(response);
     }
   }
 };

apigw-websockets/javascript/scripts/test.sh

@@ -147,7 +147,8 @@ PYEOF
 )
 
 # Run Python WebSocket test using uv
-WS_RESULT=$(uv run --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT" 2>&1)
+# The stage is part of the WebSocket URL, as on AWS
+WS_RESULT=$(uv run -q --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT/$STAGE" 2>&1 || true)
 
 case "$WS_RESULT" in
     SUCCESS)

apigw-websockets/javascript/terraform/main.tf

@@ -20,9 +20,9 @@ provider "aws" {
   skip_requesting_account_id  = true
 
   endpoints {
-    lambda         = "http://localhost.localstack.cloud:4566"
-    iam            = "http://localhost.localstack.cloud:4566"
-    apigatewayv2   = "http://localhost.localstack.cloud:4566"
+    lambda       = "http://localhost.localstack.cloud:4566"
+    iam          = "http://localhost.localstack.cloud:4566"
+    apigatewayv2 = "http://localhost.localstack.cloud:4566"
   }
 }
 

appsync-graphql-api/python/terraform/main.tf

@@ -19,11 +19,11 @@ provider "aws" {
   skip_requesting_account_id  = true
 
   endpoints {
-    appsync          = "http://localhost.localstack.cloud:4566"
-    dynamodb         = "http://localhost.localstack.cloud:4566"
-    iam              = "http://localhost.localstack.cloud:4566"
-    rds              = "http://localhost.localstack.cloud:4566"
-    secretsmanager   = "http://localhost.localstack.cloud:4566"
+    appsync        = "http://localhost.localstack.cloud:4566"
+    dynamodb       = "http://localhost.localstack.cloud:4566"
+    iam            = "http://localhost.localstack.cloud:4566"
+    rds            = "http://localhost.localstack.cloud:4566"
+    secretsmanager = "http://localhost.localstack.cloud:4566"
   }
 }
 
@@ -61,11 +61,11 @@ resource "aws_dynamodb_table" "posts" {
 
 # RDS Aurora PostgreSQL cluster
 resource "aws_rds_cluster" "main" {
-  cluster_identifier = var.db_cluster_id
-  engine             = "aurora-postgresql"
-  master_username    = var.db_user
-  master_password    = var.db_password
-  database_name      = var.db_name
+  cluster_identifier  = var.db_cluster_id
+  engine              = "aurora-postgresql"
+  master_username     = var.db_user
+  master_password     = var.db_password
+  database_name       = var.db_name
   skip_final_snapshot = true
 }
 
@@ -159,7 +159,7 @@ resource "aws_appsync_datasource" "rds" {
     source_type = "RDS_HTTP_ENDPOINT"
 
     http_endpoint_config {
-      aws_secret_store_arn = aws_secretsmanager_secret.rds.arn
+      aws_secret_store_arn  = aws_secretsmanager_secret.rds.arn
       db_cluster_identifier = aws_rds_cluster.main.arn
       database_name         = var.db_name
       region                = "us-east-1"

ecs-ecr-app/python/terraform/main.tf

@@ -16,11 +16,11 @@ provider "aws" {
   skip_requesting_account_id  = true
 
   endpoints {
-    ecr            = "http://localhost.localstack.cloud:4566"
-    ecs            = "http://localhost.localstack.cloud:4566"
-    ec2            = "http://localhost.localstack.cloud:4566"
-    iam            = "http://localhost.localstack.cloud:4566"
-    logs           = "http://localhost.localstack.cloud:4566"
+    ecr  = "http://localhost.localstack.cloud:4566"
+    ecs  = "http://localhost.localstack.cloud:4566"
+    ec2  = "http://localhost.localstack.cloud:4566"
+    iam  = "http://localhost.localstack.cloud:4566"
+    logs = "http://localhost.localstack.cloud:4566"
   }
 }
 

elb-load-balancing/javascript/terraform/main.tf

@@ -19,10 +19,10 @@ provider "aws" {
   skip_requesting_account_id  = true
 
   endpoints {
-    ec2            = var.localstack_endpoint
-    lambda         = var.localstack_endpoint
-    elbv2          = var.localstack_endpoint
-    iam            = var.localstack_endpoint
+    ec2    = var.localstack_endpoint
+    lambda = var.localstack_endpoint
+    elbv2  = var.localstack_endpoint
+    iam    = var.localstack_endpoint
   }
 }
 

emr-serverless-python-dependencies/Makefile

@@ -15,6 +15,8 @@ build-aws:
 
 deploy:
 	docker compose up --detach
+	@echo Waiting for LocalStack...
+	@for i in $$(seq 1 60); do curl -sf localhost:4566/_localstack/health > /dev/null && break; sleep 2; done
 	terraform workspace select local
 	AWS_ENDPOINT_URL=https://localhost.localstack.cloud:4566 terraform apply --auto-approve
 

emr-serverless-python-dependencies/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~>1.9.1"
+  required_version = ">= 1.9.1"
 
   required_providers {
     aws = {

emr-serverless-python-dependencies/start_job.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 local="${1:-local}"
 tfoutput=$(terraform output -json)
 
@@ -43,7 +44,12 @@ job_run_result=$($AWS emr-serverless start-job-run \
 echo $job_run_result
 job_id=$(echo $job_run_result | jq .jobRunId | tr -d '"' )
 
+deadline=$((SECONDS + ${MAX_WAIT_SECONDS:-900}))
 while true; do
+    if [ $SECONDS -ge $deadline ]; then
+        echo "Job '${job_id}' did not finish in time"
+        exit 1
+    fi
     job_run=$($AWS emr-serverless get-job-run --job-run-id $job_id --application-id $application_id)
     state=$(echo $job_run | jq .jobRun.state | tr -d '"' )
     echo "Job '${job_id}', State '${state}'. First run might take a few minutes."

emr-serverless-python-dependencies/stop-application.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 local="${1:-local}"
 tfoutput=$(terraform output -json)
 application_id=$(echo $tfoutput | jq .application_id.value | tr -d '"')

glue-etl-jobs/python/scripts/deploy.sh

@@ -20,6 +20,7 @@ echo "Creating RDS Aurora PostgreSQL cluster..."
 awslocal rds create-db-cluster \
     --db-cluster-identifier "$CLUSTER_IDENTIFIER" \
     --engine aurora-postgresql \
+    --enable-http-endpoint \
     --database-name test > /dev/null
 
 echo "Waiting for RDS cluster to be available..."

glue-etl-jobs/python/scripts/teardown.sh

@@ -3,12 +3,16 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+# deploy.sh writes .env only after a successful run; fall back to its fixed
+# resource names so a failed deploy can still be cleaned up
 if [ -f "$SCRIPT_DIR/.env" ]; then
     source "$SCRIPT_DIR/.env"
-else
-    echo "No .env file found. Nothing to tear down."
-    exit 0
 fi
+BUCKET="${BUCKET:-glue-pyspark-test}"
+TARGET_BUCKET="${TARGET_BUCKET:-glue-sample-target}"
+JOB_NAME="${JOB_NAME:-test-job1}"
+CLUSTER_IDENTIFIER="${CLUSTER_IDENTIFIER:-glue-etl-cluster1}"
+CONNECTION_NAME="${CONNECTION_NAME:-glue-etl-cluster1-connection}"
 
 echo "Cleaning up Glue ETL resources..."