From af85c43ceb0854bc9ec7ec9a27787e5ad94ea6dd Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Mon, 12 Oct 2026 18:10:13 +0530 Subject: [PATCH 1/4] Make lint enforcing and run the sample tests in CI Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- .github/workflows/lint.yml | 9 +- .github/workflows/test.yml | 152 ++++++++++++++++++ .../python/terraform/providers.tf | 4 +- apigw-websockets/javascript/handler.js | 5 +- apigw-websockets/javascript/scripts/test.sh | 3 +- apigw-websockets/javascript/terraform/main.tf | 6 +- appsync-graphql-api/python/terraform/main.tf | 22 +-- ecs-ecr-app/python/terraform/main.tf | 10 +- .../javascript/terraform/main.tf | 8 +- .../start_job.sh | 1 + .../stop-application.sh | 1 + glue-etl-jobs/python/scripts/deploy.sh | 1 + glue-etl-jobs/python/terraform/main.tf | 14 +- .../python/terraform/main.tf | 8 +- .../python/terraform/main.tf | 16 +- lambda-s3-http/python/terraform/main.tf | 12 +- mq-broker/python/terraform/main.tf | 12 +- route53-dns-failover/python/terraform/main.tf | 10 +- stepfunctions-lambda/python/terraform/main.tf | 12 +- terraform-resources/test.tf | 22 +-- web-app-rds/python/terraform/main.tf | 4 +- 21 files changed, 243 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6f51ae2..9817df6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: run: sudo apt-get install -y shellcheck - name: Run shellcheck run: | - find . -name '*.sh' -type f | xargs shellcheck --severity=warning || true + find . -name '*.sh' -type f -print0 | xargs -0 shellcheck --severity=error terraform-fmt: name: Terraform Format @@ -25,9 +25,6 @@ jobs: - uses: actions/checkout@v4 - uses: hashicorp/setup-terraform@v3 with: - terraform_version: '1.5.0' + terraform_version: '1.14.9' - name: Check terraform format - run: | - find . -name '*.tf' -type f -exec dirname {} \; | sort -u | while read dir; do - terraform fmt -check "$dir" || true - done + run: terraform fmt -check -recursive -diff . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..75bb46b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,152 @@ +name: Sample Tests + +on: + push: + paths-ignore: + - '**.md' + branches: + - main + pull_request: + branches: + - main + schedule: + # “At 00:00 on Sunday.” + - cron: "0 0 * * 0" + workflow_dispatch: + +jobs: + prepare-list: + name: Find testable samples + runs-on: ubuntu-latest + outputs: + scripts: ${{ steps.set-matrix.outputs.scripts }} + make: ${{ steps.set-matrix.outputs.make }} + steps: + - uses: actions/checkout@v4 + + - id: set-matrix + run: | + # Samples with scripts/deploy.sh and scripts/test.sh + mapfile -t scripts < <(find . -path '*/scripts/test.sh' | xargs -n1 dirname | xargs -n1 dirname | sed 's|^\./||' | sort) + # Samples with a Makefile test-ci target + mapfile -t make < <(grep -l '^test-ci:' */Makefile | xargs -n1 dirname | sort) + echo "scripts=$(printf '%s\n' "${scripts[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT" + echo "make=$(printf '%s\n' "${make[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT" + + script-samples: + name: Test ${{ matrix.directory }} + runs-on: ubuntu-latest + needs: prepare-list + strategy: + fail-fast: false + max-parallel: 8 + matrix: + directory: ${{ fromJSON(needs.prepare-list.outputs.scripts) }} + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install tools + run: | + pip install awscli-local + npm install -g @localstack/lstk + + - name: Start LocalStack + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: lstk start + + - name: Deploy + timeout-minutes: 15 + working-directory: ${{ matrix.directory }} + run: bash scripts/deploy.sh + + - name: Test + timeout-minutes: 10 + working-directory: ${{ matrix.directory }} + run: bash scripts/test.sh + + - name: Show LocalStack logs + if: failure() + run: lstk logs + + - name: Send a Slack notification + if: failure() || github.event_name != 'pull_request' + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + token: ${{ secrets.GITHUB_TOKEN }} + notification_title: "Sample `${{ matrix.directory }}` ({workflow}) has {status_message}" + message_format: "{emoji} *${{ matrix.directory }}* {status_message} in <{repo_url}|{repo}>" + footer: "Linked Repo <{repo_url}|{repo}> | <{run_url}|View Workflow run>" + notify_when: "failure" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + make-samples: + name: Test ${{ matrix.directory }} + runs-on: ubuntu-latest + needs: prepare-list + strategy: + fail-fast: false + max-parallel: 8 + matrix: + directory: ${{ fromJSON(needs.prepare-list.outputs.make) }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install tools + run: | + pip install localstack awscli-local[ver1] aws-sam-cli-local virtualenv + npm install -g @localstack/lstk + + - name: Run make test-ci + timeout-minutes: 15 + working-directory: ${{ matrix.directory }} + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: make test-ci + + - name: Upload LocalStack logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.directory }} + path: ${{ matrix.directory }}/logs.txt + if-no-files-found: ignore + + - name: Send a Slack notification + if: failure() || github.event_name != 'pull_request' + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + token: ${{ secrets.GITHUB_TOKEN }} + notification_title: "Sample `${{ matrix.directory }}` ({workflow}) has {status_message}" + message_format: "{emoji} *${{ matrix.directory }}* {status_message} in <{repo_url}|{repo}>" + footer: "Linked Repo <{repo_url}|{repo}> | <{run_url}|View Workflow run>" + notify_when: "failure" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/apigw-custom-domain/python/terraform/providers.tf b/apigw-custom-domain/python/terraform/providers.tf index c60cfc0..ff65ac3 100644 --- a/apigw-custom-domain/python/terraform/providers.tf +++ b/apigw-custom-domain/python/terraform/providers.tf @@ -1,10 +1,10 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" + source = "hashicorp/aws" } tls = { - source = "hashicorp/tls" + source = "hashicorp/tls" } } } diff --git a/apigw-websockets/javascript/handler.js b/apigw-websockets/javascript/handler.js index 19f01af..74a5f30 100644 --- a/apigw-websockets/javascript/handler.js +++ b/apigw-websockets/javascript/handler.js @@ -26,10 +26,11 @@ module.exports.handler = function(event, context, callback) { } else { // Echo back the event for $default and custom actions console.log(`Message received on route: ${routeKey}`); + const response = { statusCode: 200, body: JSON.stringify(event) }; if (callback) { - callback(null, event); + callback(null, response); } else { - context.succeed(event); + context.succeed(response); } } }; diff --git a/apigw-websockets/javascript/scripts/test.sh b/apigw-websockets/javascript/scripts/test.sh index 737a5e9..2dbb20f 100755 --- a/apigw-websockets/javascript/scripts/test.sh +++ b/apigw-websockets/javascript/scripts/test.sh @@ -147,7 +147,8 @@ PYEOF ) # Run Python WebSocket test using uv -WS_RESULT=$(uv run --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT" 2>&1) +# The stage is part of the WebSocket URL, as on AWS +WS_RESULT=$(uv run --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT/$STAGE" 2>&1 || true) case "$WS_RESULT" in SUCCESS) diff --git a/apigw-websockets/javascript/terraform/main.tf b/apigw-websockets/javascript/terraform/main.tf index c6064d0..40dba98 100644 --- a/apigw-websockets/javascript/terraform/main.tf +++ b/apigw-websockets/javascript/terraform/main.tf @@ -20,9 +20,9 @@ provider "aws" { skip_requesting_account_id = true endpoints { - lambda = "http://localhost.localstack.cloud:4566" - iam = "http://localhost.localstack.cloud:4566" - apigatewayv2 = "http://localhost.localstack.cloud:4566" + lambda = "http://localhost.localstack.cloud:4566" + iam = "http://localhost.localstack.cloud:4566" + apigatewayv2 = "http://localhost.localstack.cloud:4566" } } diff --git a/appsync-graphql-api/python/terraform/main.tf b/appsync-graphql-api/python/terraform/main.tf index 05209f9..bcad734 100644 --- a/appsync-graphql-api/python/terraform/main.tf +++ b/appsync-graphql-api/python/terraform/main.tf @@ -19,11 +19,11 @@ provider "aws" { skip_requesting_account_id = true endpoints { - appsync = "http://localhost.localstack.cloud:4566" - dynamodb = "http://localhost.localstack.cloud:4566" - iam = "http://localhost.localstack.cloud:4566" - rds = "http://localhost.localstack.cloud:4566" - secretsmanager = "http://localhost.localstack.cloud:4566" + appsync = "http://localhost.localstack.cloud:4566" + dynamodb = "http://localhost.localstack.cloud:4566" + iam = "http://localhost.localstack.cloud:4566" + rds = "http://localhost.localstack.cloud:4566" + secretsmanager = "http://localhost.localstack.cloud:4566" } } @@ -61,11 +61,11 @@ resource "aws_dynamodb_table" "posts" { # RDS Aurora PostgreSQL cluster resource "aws_rds_cluster" "main" { - cluster_identifier = var.db_cluster_id - engine = "aurora-postgresql" - master_username = var.db_user - master_password = var.db_password - database_name = var.db_name + cluster_identifier = var.db_cluster_id + engine = "aurora-postgresql" + master_username = var.db_user + master_password = var.db_password + database_name = var.db_name skip_final_snapshot = true } @@ -159,7 +159,7 @@ resource "aws_appsync_datasource" "rds" { source_type = "RDS_HTTP_ENDPOINT" http_endpoint_config { - aws_secret_store_arn = aws_secretsmanager_secret.rds.arn + aws_secret_store_arn = aws_secretsmanager_secret.rds.arn db_cluster_identifier = aws_rds_cluster.main.arn database_name = var.db_name region = "us-east-1" diff --git a/ecs-ecr-app/python/terraform/main.tf b/ecs-ecr-app/python/terraform/main.tf index f3e66ff..7630aaa 100644 --- a/ecs-ecr-app/python/terraform/main.tf +++ b/ecs-ecr-app/python/terraform/main.tf @@ -16,11 +16,11 @@ provider "aws" { skip_requesting_account_id = true endpoints { - ecr = "http://localhost.localstack.cloud:4566" - ecs = "http://localhost.localstack.cloud:4566" - ec2 = "http://localhost.localstack.cloud:4566" - iam = "http://localhost.localstack.cloud:4566" - logs = "http://localhost.localstack.cloud:4566" + ecr = "http://localhost.localstack.cloud:4566" + ecs = "http://localhost.localstack.cloud:4566" + ec2 = "http://localhost.localstack.cloud:4566" + iam = "http://localhost.localstack.cloud:4566" + logs = "http://localhost.localstack.cloud:4566" } } diff --git a/elb-load-balancing/javascript/terraform/main.tf b/elb-load-balancing/javascript/terraform/main.tf index 2c1c347..c324826 100644 --- a/elb-load-balancing/javascript/terraform/main.tf +++ b/elb-load-balancing/javascript/terraform/main.tf @@ -19,10 +19,10 @@ provider "aws" { skip_requesting_account_id = true endpoints { - ec2 = var.localstack_endpoint - lambda = var.localstack_endpoint - elbv2 = var.localstack_endpoint - iam = var.localstack_endpoint + ec2 = var.localstack_endpoint + lambda = var.localstack_endpoint + elbv2 = var.localstack_endpoint + iam = var.localstack_endpoint } } diff --git a/emr-serverless-python-dependencies/start_job.sh b/emr-serverless-python-dependencies/start_job.sh index 12779f6..64fd95a 100755 --- a/emr-serverless-python-dependencies/start_job.sh +++ b/emr-serverless-python-dependencies/start_job.sh @@ -1,3 +1,4 @@ +#!/bin/bash local="${1:-local}" tfoutput=$(terraform output -json) diff --git a/emr-serverless-python-dependencies/stop-application.sh b/emr-serverless-python-dependencies/stop-application.sh index 282c983..7e8cfb6 100755 --- a/emr-serverless-python-dependencies/stop-application.sh +++ b/emr-serverless-python-dependencies/stop-application.sh @@ -1,3 +1,4 @@ +#!/bin/bash local="${1:-local}" tfoutput=$(terraform output -json) application_id=$(echo $tfoutput | jq .application_id.value | tr -d '"') diff --git a/glue-etl-jobs/python/scripts/deploy.sh b/glue-etl-jobs/python/scripts/deploy.sh index acdc6e0..2678f47 100755 --- a/glue-etl-jobs/python/scripts/deploy.sh +++ b/glue-etl-jobs/python/scripts/deploy.sh @@ -20,6 +20,7 @@ echo "Creating RDS Aurora PostgreSQL cluster..." awslocal rds create-db-cluster \ --db-cluster-identifier "$CLUSTER_IDENTIFIER" \ --engine aurora-postgresql \ + --enable-http-endpoint \ --database-name test > /dev/null echo "Waiting for RDS cluster to be available..." diff --git a/glue-etl-jobs/python/terraform/main.tf b/glue-etl-jobs/python/terraform/main.tf index f9289b7..7ab7240 100644 --- a/glue-etl-jobs/python/terraform/main.tf +++ b/glue-etl-jobs/python/terraform/main.tf @@ -19,10 +19,10 @@ provider "aws" { skip_requesting_account_id = true endpoints { - s3 = "http://localhost.localstack.cloud:4566" - rds = "http://localhost.localstack.cloud:4566" - glue = "http://localhost.localstack.cloud:4566" - secretsmanager = "http://localhost.localstack.cloud:4566" + s3 = "http://localhost.localstack.cloud:4566" + rds = "http://localhost.localstack.cloud:4566" + glue = "http://localhost.localstack.cloud:4566" + secretsmanager = "http://localhost.localstack.cloud:4566" } s3_use_path_style = true @@ -58,9 +58,9 @@ resource "aws_s3_object" "job_script" { # Aurora PostgreSQL cluster resource "aws_rds_cluster" "glue_etl" { - cluster_identifier = local.cluster_id - engine = "aurora-postgresql" - database_name = "test" + cluster_identifier = local.cluster_id + engine = "aurora-postgresql" + database_name = "test" skip_final_snapshot = true } diff --git a/glue-msk-schema-registry/python/terraform/main.tf b/glue-msk-schema-registry/python/terraform/main.tf index 6257e12..29e56a2 100644 --- a/glue-msk-schema-registry/python/terraform/main.tf +++ b/glue-msk-schema-registry/python/terraform/main.tf @@ -16,10 +16,10 @@ provider "aws" { skip_requesting_account_id = true endpoints { - ec2 = "http://localhost.localstack.cloud:4566" - kafka = "http://localhost.localstack.cloud:4566" - glue = "http://localhost.localstack.cloud:4566" - sts = "http://localhost.localstack.cloud:4566" + ec2 = "http://localhost.localstack.cloud:4566" + kafka = "http://localhost.localstack.cloud:4566" + glue = "http://localhost.localstack.cloud:4566" + sts = "http://localhost.localstack.cloud:4566" } } diff --git a/glue-redshift-crawler/python/terraform/main.tf b/glue-redshift-crawler/python/terraform/main.tf index d0fd91a..12b2f52 100644 --- a/glue-redshift-crawler/python/terraform/main.tf +++ b/glue-redshift-crawler/python/terraform/main.tf @@ -61,12 +61,12 @@ resource "aws_iam_role" "glue" { # Redshift cluster resource "aws_redshift_cluster" "main" { - cluster_identifier = var.redshift_cluster_id - database_name = var.redshift_db_name - master_username = var.redshift_username - master_password = var.redshift_password - node_type = "dc2.large" - cluster_type = "single-node" + cluster_identifier = var.redshift_cluster_id + database_name = var.redshift_db_name + master_username = var.redshift_username + master_password = var.redshift_password + node_type = "dc2.large" + cluster_type = "single-node" skip_final_snapshot = true } @@ -80,8 +80,8 @@ resource "aws_glue_connection" "redshift" { name = "glueconn-tf" connection_properties = { - USERNAME = var.redshift_username - PASSWORD = var.redshift_password + USERNAME = var.redshift_username + PASSWORD = var.redshift_password JDBC_CONNECTION_URL = "jdbc:redshift://${aws_redshift_cluster.main.endpoint}/${var.redshift_db_name}" } diff --git a/lambda-s3-http/python/terraform/main.tf b/lambda-s3-http/python/terraform/main.tf index 16cd95a..cda5499 100644 --- a/lambda-s3-http/python/terraform/main.tf +++ b/lambda-s3-http/python/terraform/main.tf @@ -1,12 +1,12 @@ # Lambda S3 HTTP Sample - Terraform Configuration locals { - table_name = "${var.prefix}-game-scores" - bucket_name = "${var.prefix}-replays" - queue_name = "${var.prefix}-score-validation" - http_function = "${var.prefix}-http-handler" - s3_function = "${var.prefix}-s3-handler" - sqs_function = "${var.prefix}-sqs-handler" + table_name = "${var.prefix}-game-scores" + bucket_name = "${var.prefix}-replays" + queue_name = "${var.prefix}-score-validation" + http_function = "${var.prefix}-http-handler" + s3_function = "${var.prefix}-s3-handler" + sqs_function = "${var.prefix}-sqs-handler" } # IAM role for Lambda diff --git a/mq-broker/python/terraform/main.tf b/mq-broker/python/terraform/main.tf index 5f6d170..cbf431e 100644 --- a/mq-broker/python/terraform/main.tf +++ b/mq-broker/python/terraform/main.tf @@ -35,12 +35,12 @@ locals { } resource "aws_mq_broker" "main" { - broker_name = local.broker_name - engine_type = "ActiveMQ" - engine_version = "5.18" - host_instance_type = "mq.m5.large" - deployment_mode = "SINGLE_INSTANCE" - publicly_accessible = true + broker_name = local.broker_name + engine_type = "ActiveMQ" + engine_version = "5.18" + host_instance_type = "mq.m5.large" + deployment_mode = "SINGLE_INSTANCE" + publicly_accessible = true auto_minor_version_upgrade = true user { diff --git a/route53-dns-failover/python/terraform/main.tf b/route53-dns-failover/python/terraform/main.tf index 0e1d313..f2369db 100644 --- a/route53-dns-failover/python/terraform/main.tf +++ b/route53-dns-failover/python/terraform/main.tf @@ -34,11 +34,11 @@ resource "aws_route53_zone" "main" { # Health check against LocalStack health endpoint resource "aws_route53_health_check" "primary" { - fqdn = "localhost.localstack.cloud" - port = 4566 - resource_path = "/_localstack/health" - type = "HTTP" - request_interval = 10 + fqdn = "localhost.localstack.cloud" + port = 4566 + resource_path = "/_localstack/health" + type = "HTTP" + request_interval = 10 } # Target 1: primary CNAME diff --git a/stepfunctions-lambda/python/terraform/main.tf b/stepfunctions-lambda/python/terraform/main.tf index 5bc22df..8a92a92 100644 --- a/stepfunctions-lambda/python/terraform/main.tf +++ b/stepfunctions-lambda/python/terraform/main.tf @@ -132,8 +132,8 @@ resource "aws_sfn_state_machine" "parallel_workflow" { StartAt = "Adam" States = { Adam = { - Type = "Task" - Resource = "arn:aws:states:::lambda:invoke" + Type = "Task" + Resource = "arn:aws:states:::lambda:invoke" OutputPath = "$.Payload" Parameters = { FunctionName = aws_lambda_function.adam.arn @@ -149,8 +149,8 @@ resource "aws_sfn_state_machine" "parallel_workflow" { StartAt = "Cole" States = { Cole = { - Type = "Task" - Resource = "arn:aws:states:::lambda:invoke" + Type = "Task" + Resource = "arn:aws:states:::lambda:invoke" OutputPath = "$.Payload" Parameters = { FunctionName = aws_lambda_function.cole.arn @@ -165,8 +165,8 @@ resource "aws_sfn_state_machine" "parallel_workflow" { ] } Combine = { - Type = "Task" - Resource = "arn:aws:states:::lambda:invoke" + Type = "Task" + Resource = "arn:aws:states:::lambda:invoke" OutputPath = "$.Payload" Parameters = { FunctionName = aws_lambda_function.combine.arn diff --git a/terraform-resources/test.tf b/terraform-resources/test.tf index 43c48b3..7e8d8bf 100644 --- a/terraform-resources/test.tf +++ b/terraform-resources/test.tf @@ -44,10 +44,10 @@ resource "aws_s3_bucket_acl" "test-bucket-acl" { resource "aws_api_gateway_authorizer" "demo" { name = "demo" - rest_api_id = "${aws_api_gateway_rest_api.demo.id}" - authorizer_uri = "${aws_lambda_function.authorizer.invoke_arn}" - authorizer_credentials = "${aws_iam_role.invocation_role.arn}" - type = "TOKEN" + rest_api_id = aws_api_gateway_rest_api.demo.id + authorizer_uri = aws_lambda_function.authorizer.invoke_arn + authorizer_credentials = aws_iam_role.invocation_role.arn + type = "TOKEN" } resource "aws_api_gateway_rest_api" "demo" { @@ -72,7 +72,7 @@ EOF resource "aws_iam_role_policy" "invocation_policy" { name = "default" - role = "${aws_iam_role.invocation_role.id}" + role = aws_iam_role.invocation_role.id policy = < Date: Mon, 12 Oct 2026 18:37:14 +0530 Subject: [PATCH 2/4] Fix the sample test jobs on the CI runner - Set cli_binary_format for AWS CLI v2 on the runner, so raw JSON payloads in lambda invoke are not parsed as base64 - Install uv for the apigw-websockets client and Terraform for emr-serverless-python-dependencies - lambda-debugging-sam-*: build the function zip in test-ci before deploying - terraform-resources: build lambda.zip with archive_file, use python3.12, and use real trust policies for the IAM roles - glue-etl-jobs: disable SSL on the JDBC connection; the Glue job container fails the TLS handshake with the LocalStack Postgres proxy Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- .github/workflows/test.yml | 10 ++++++++- glue-etl-jobs/python/scripts/deploy.sh | 2 +- lambda-debugging-sam-java/Makefile | 2 +- lambda-debugging-sam-javascript/Makefile | 2 +- lambda-debugging-sam-python/Makefile | 2 +- lambda-debugging-sam-typescript/Makefile | 2 +- terraform-resources/test.tf | 27 ++++++++++++++++++------ 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75bb46b..deb4559 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,8 +61,10 @@ jobs: - name: Install tools run: | - pip install awscli-local + pip install awscli-local uv npm install -g @localstack/lstk + # The runner ships AWS CLI v2, which treats blob parameters as base64 by default + aws configure set cli_binary_format raw-in-base64-out - name: Start LocalStack env: @@ -118,6 +120,12 @@ jobs: with: node-version: 22 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: '1.14.9' + terraform_wrapper: false + - name: Install tools run: | pip install localstack awscli-local[ver1] aws-sam-cli-local virtualenv diff --git a/glue-etl-jobs/python/scripts/deploy.sh b/glue-etl-jobs/python/scripts/deploy.sh index 2678f47..8bcf1e9 100755 --- a/glue-etl-jobs/python/scripts/deploy.sh +++ b/glue-etl-jobs/python/scripts/deploy.sh @@ -56,7 +56,7 @@ awslocal glue create-table --database legislators \ awslocal glue create-table --database legislators \ --table-input '{"Name": "organizations_json", "Parameters": {"connectionName": "'"$CONNECTION_NAME"'"}, "StorageDescriptor": {"Location": "test.organizations"}}' awslocal glue create-connection \ - --connection-input '{"Name": "'"$CONNECTION_NAME"'", "ConnectionType": "JDBC", "ConnectionProperties": {"USERNAME": "test", "PASSWORD": "test", "JDBC_CONNECTION_URL": "jdbc:postgresql://localhost.localstack.cloud:'"$DB_PORT"'"}}' + --connection-input '{"Name": "'"$CONNECTION_NAME"'", "ConnectionType": "JDBC", "ConnectionProperties": {"USERNAME": "test", "PASSWORD": "test", "JDBC_CONNECTION_URL": "jdbc:postgresql://localhost.localstack.cloud:'"$DB_PORT"'/test?sslmode=disable"}}' SECRET_ARN=$(awslocal secretsmanager create-secret --name pass --secret-string "test" | jq -r ".ARN") diff --git a/lambda-debugging-sam-java/Makefile b/lambda-debugging-sam-java/Makefile index 1f2707c..5f12d05 100644 --- a/lambda-debugging-sam-java/Makefile +++ b/lambda-debugging-sam-java/Makefile @@ -83,5 +83,5 @@ ready: ## Wait until LocalStack is running @localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1) test-ci: - make start install ready deploy wait invoke; return_code=`echo $$?`;\ + make start install build ready deploy wait invoke; return_code=`echo $$?`;\ echo "Interactive debugging not tested in CI"; exit $$return_code; diff --git a/lambda-debugging-sam-javascript/Makefile b/lambda-debugging-sam-javascript/Makefile index e9f78a7..6cba8b8 100644 --- a/lambda-debugging-sam-javascript/Makefile +++ b/lambda-debugging-sam-javascript/Makefile @@ -102,5 +102,5 @@ ready: ## Wait until LocalStack is running @localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1) test-ci: - make start install ready deploy wait invoke; return_code=`echo $$?`;\ + make start install build ready deploy wait invoke; return_code=`echo $$?`;\ echo "Interactive debugging not tested in CI"; exit $$return_code; diff --git a/lambda-debugging-sam-python/Makefile b/lambda-debugging-sam-python/Makefile index 2956e0f..0ae0b77 100644 --- a/lambda-debugging-sam-python/Makefile +++ b/lambda-debugging-sam-python/Makefile @@ -102,5 +102,5 @@ ready: ## Wait until LocalStack is running @localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1) test-ci: - make start install ready deploy wait invoke; return_code=`echo $$?`;\ + make start install build ready deploy wait invoke; return_code=`echo $$?`;\ echo "Interactive debugging not tested in CI"; exit $$return_code; diff --git a/lambda-debugging-sam-typescript/Makefile b/lambda-debugging-sam-typescript/Makefile index 9e35569..a39d32e 100644 --- a/lambda-debugging-sam-typescript/Makefile +++ b/lambda-debugging-sam-typescript/Makefile @@ -88,5 +88,5 @@ ready: ## Wait until LocalStack is running @localstack wait -t 30 && echo LocalStack is ready to use! || (echo Gave up waiting on LocalStack, exiting. && exit 1) test-ci: - make start install ready deploy wait invoke; return_code=`echo $$?`;\ + make start install build ready deploy wait invoke; return_code=`echo $$?`;\ echo "Interactive debugging not tested in CI"; exit $$return_code; diff --git a/terraform-resources/test.tf b/terraform-resources/test.tf index 7e8d8bf..f3f355d 100644 --- a/terraform-resources/test.tf +++ b/terraform-resources/test.tf @@ -63,8 +63,8 @@ resource "aws_iam_role" "invocation_role" { "Version": "2012-10-17", "Statement": { "Effect": "Allow", - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::test-bucket" + "Principal": {"Service": "apigateway.amazonaws.com"}, + "Action": "sts:AssumeRole" } } EOF @@ -94,20 +94,33 @@ resource "aws_iam_role" "lambda" { "Version": "2012-10-17", "Statement": { "Effect": "Allow", - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::example_bucket" + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole" } } EOF } +data "archive_file" "lambda" { + type = "zip" + output_path = "${path.module}/lambda.zip" + + source { + filename = "lambda.py" + content = < Date: Mon, 12 Oct 2026 18:57:41 +0530 Subject: [PATCH 3/4] apigw-websockets: keep uv install output out of the test result Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- apigw-websockets/javascript/scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-websockets/javascript/scripts/test.sh b/apigw-websockets/javascript/scripts/test.sh index 2dbb20f..89a1a68 100755 --- a/apigw-websockets/javascript/scripts/test.sh +++ b/apigw-websockets/javascript/scripts/test.sh @@ -148,7 +148,7 @@ PYEOF # Run Python WebSocket test using uv # The stage is part of the WebSocket URL, as on AWS -WS_RESULT=$(uv run --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT/$STAGE" 2>&1 || true) +WS_RESULT=$(uv run -q --with websockets python -c "$PYTHON_TEST" "$WS_ENDPOINT/$STAGE" 2>&1 || true) case "$WS_RESULT" in SUCCESS) From 7eaf2e07bb723dee787a4c0da3d35887fcd6f769 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Mon, 12 Oct 2026 19:00:44 +0530 Subject: [PATCH 4/4] Fix emr-serverless in CI and skip glue-etl-jobs for now - emr-serverless-python-dependencies: allow Terraform >= 1.9.1, wait for LocalStack after docker compose up, and stop polling the job after 15 minutes - Give make test-ci 30 minutes, since the first EMR Spark run is slow - glue-etl-jobs: Glue always attempts SSL to RDS first, and an aborted TLS handshake stops the LocalStack Postgres proxy for that cluster, so the non-SSL fallback is refused. Skip it in the matrix until that is fixed. Revert the sslmode change, which did not help. - glue-etl-jobs teardown: fall back to the fixed resource names when .env is missing, so a failed deploy can be cleaned up Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- .github/workflows/test.yml | 8 +++++--- emr-serverless-python-dependencies/Makefile | 2 ++ emr-serverless-python-dependencies/main.tf | 2 +- emr-serverless-python-dependencies/start_job.sh | 5 +++++ glue-etl-jobs/python/scripts/deploy.sh | 2 +- glue-etl-jobs/python/scripts/teardown.sh | 10 +++++++--- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index deb4559..9028d62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,8 +26,10 @@ jobs: - id: set-matrix run: | - # Samples with scripts/deploy.sh and scripts/test.sh - mapfile -t scripts < <(find . -path '*/scripts/test.sh' | xargs -n1 dirname | xargs -n1 dirname | sed 's|^\./||' | sort) + # Samples with scripts/deploy.sh and scripts/test.sh. + # glue-etl-jobs is skipped for now: Glue tries an SSL connection to RDS first, and an + # aborted TLS handshake stops the LocalStack Postgres proxy for that cluster. + mapfile -t scripts < <(find . -path '*/scripts/test.sh' | xargs -n1 dirname | xargs -n1 dirname | sed 's|^\./||' | grep -v '^glue-etl-jobs/' | sort) # Samples with a Makefile test-ci target mapfile -t make < <(grep -l '^test-ci:' */Makefile | xargs -n1 dirname | sort) echo "scripts=$(printf '%s\n' "${scripts[@]}" | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT" @@ -132,7 +134,7 @@ jobs: npm install -g @localstack/lstk - name: Run make test-ci - timeout-minutes: 15 + timeout-minutes: 30 working-directory: ${{ matrix.directory }} env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} diff --git a/emr-serverless-python-dependencies/Makefile b/emr-serverless-python-dependencies/Makefile index e60f435..cd761c8 100644 --- a/emr-serverless-python-dependencies/Makefile +++ b/emr-serverless-python-dependencies/Makefile @@ -15,6 +15,8 @@ build-aws: deploy: docker compose up --detach + @echo Waiting for LocalStack... + @for i in $$(seq 1 60); do curl -sf localhost:4566/_localstack/health > /dev/null && break; sleep 2; done terraform workspace select local AWS_ENDPOINT_URL=https://localhost.localstack.cloud:4566 terraform apply --auto-approve diff --git a/emr-serverless-python-dependencies/main.tf b/emr-serverless-python-dependencies/main.tf index fcae899..2e076a4 100644 --- a/emr-serverless-python-dependencies/main.tf +++ b/emr-serverless-python-dependencies/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~>1.9.1" + required_version = ">= 1.9.1" required_providers { aws = { diff --git a/emr-serverless-python-dependencies/start_job.sh b/emr-serverless-python-dependencies/start_job.sh index 64fd95a..b698ab5 100755 --- a/emr-serverless-python-dependencies/start_job.sh +++ b/emr-serverless-python-dependencies/start_job.sh @@ -44,7 +44,12 @@ job_run_result=$($AWS emr-serverless start-job-run \ echo $job_run_result job_id=$(echo $job_run_result | jq .jobRunId | tr -d '"' ) +deadline=$((SECONDS + ${MAX_WAIT_SECONDS:-900})) while true; do + if [ $SECONDS -ge $deadline ]; then + echo "Job '${job_id}' did not finish in time" + exit 1 + fi job_run=$($AWS emr-serverless get-job-run --job-run-id $job_id --application-id $application_id) state=$(echo $job_run | jq .jobRun.state | tr -d '"' ) echo "Job '${job_id}', State '${state}'. First run might take a few minutes." diff --git a/glue-etl-jobs/python/scripts/deploy.sh b/glue-etl-jobs/python/scripts/deploy.sh index 8bcf1e9..2678f47 100755 --- a/glue-etl-jobs/python/scripts/deploy.sh +++ b/glue-etl-jobs/python/scripts/deploy.sh @@ -56,7 +56,7 @@ awslocal glue create-table --database legislators \ awslocal glue create-table --database legislators \ --table-input '{"Name": "organizations_json", "Parameters": {"connectionName": "'"$CONNECTION_NAME"'"}, "StorageDescriptor": {"Location": "test.organizations"}}' awslocal glue create-connection \ - --connection-input '{"Name": "'"$CONNECTION_NAME"'", "ConnectionType": "JDBC", "ConnectionProperties": {"USERNAME": "test", "PASSWORD": "test", "JDBC_CONNECTION_URL": "jdbc:postgresql://localhost.localstack.cloud:'"$DB_PORT"'/test?sslmode=disable"}}' + --connection-input '{"Name": "'"$CONNECTION_NAME"'", "ConnectionType": "JDBC", "ConnectionProperties": {"USERNAME": "test", "PASSWORD": "test", "JDBC_CONNECTION_URL": "jdbc:postgresql://localhost.localstack.cloud:'"$DB_PORT"'"}}' SECRET_ARN=$(awslocal secretsmanager create-secret --name pass --secret-string "test" | jq -r ".ARN") diff --git a/glue-etl-jobs/python/scripts/teardown.sh b/glue-etl-jobs/python/scripts/teardown.sh index 5a4069f..dfe934f 100755 --- a/glue-etl-jobs/python/scripts/teardown.sh +++ b/glue-etl-jobs/python/scripts/teardown.sh @@ -3,12 +3,16 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# deploy.sh writes .env only after a successful run; fall back to its fixed +# resource names so a failed deploy can still be cleaned up if [ -f "$SCRIPT_DIR/.env" ]; then source "$SCRIPT_DIR/.env" -else - echo "No .env file found. Nothing to tear down." - exit 0 fi +BUCKET="${BUCKET:-glue-pyspark-test}" +TARGET_BUCKET="${TARGET_BUCKET:-glue-sample-target}" +JOB_NAME="${JOB_NAME:-test-job1}" +CLUSTER_IDENTIFIER="${CLUSTER_IDENTIFIER:-glue-etl-cluster1}" +CONNECTION_NAME="${CONNECTION_NAME:-glue-etl-cluster1-connection}" echo "Cleaning up Glue ETL resources..."