From 8f4fa3664705819bec59c9b12b5868820c6f9814 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Wed, 16 Sep 2026 14:02:46 -0400 Subject: [PATCH 1/8] refactor(cd): ship Elastic Beanstalk versions from GitHub on main Keep application and Terraform changes in separate PRs so a merge cannot race an HCP apply against an app deploy. --- .github/workflows/architecture-quality.yml | 6 +- .github/workflows/ci-terraform.yaml | 60 +++ .github/workflows/ci.yml | 33 +- .github/workflows/deploy-tag.yaml | 43 ++ .github/workflows/deploy.yaml | 403 ++++++++++++++++++ .github/workflows/release.yaml | 90 ++++ QUALITY_GATES.md | 41 +- REVIEW_AND_PR_FRAMEWORK.md | 9 +- scripts/check_app_terraform_isolation.py | 69 +++ scripts/governance-check.sh | 22 +- scripts/next_release_tag.py | 95 +++++ scripts/require_commit_checks.py | 123 ++++++ scripts/test_check_app_terraform_isolation.py | 59 +++ scripts/test_next_release_tag.py | 62 +++ scripts/test_require_commit_checks.py | 112 +++++ terraform/README.md | 11 +- terraform/live/README.md | 95 ++--- terraform/live/dev/main.tf | 2 +- terraform/live/dev/variables.tf | 15 - .../live/modules/environment-owned/main.tf | 71 ++- .../modules/environment-owned/variables.tf | 17 +- terraform/live/staging/main.tf | 2 +- terraform/live/staging/variables.tf | 15 - 23 files changed, 1280 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/ci-terraform.yaml create mode 100644 .github/workflows/deploy-tag.yaml create mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 scripts/check_app_terraform_isolation.py create mode 100644 scripts/next_release_tag.py create mode 100644 scripts/require_commit_checks.py create mode 100644 scripts/test_check_app_terraform_isolation.py create mode 100644 scripts/test_next_release_tag.py create mode 100644 scripts/test_require_commit_checks.py delete mode 100644 terraform/live/dev/variables.tf delete mode 100644 terraform/live/staging/variables.tf diff --git a/.github/workflows/architecture-quality.yml b/.github/workflows/architecture-quality.yml index 14018b5..4df5842 100644 --- a/.github/workflows/architecture-quality.yml +++ b/.github/workflows/architecture-quality.yml @@ -2,6 +2,8 @@ name: Architecture and changed-file quality on: pull_request: + push: + branches: [main] permissions: contents: read @@ -24,6 +26,6 @@ jobs: - name: Repository quality gate shell: bash env: - BASE_REF: ${{ github.event.pull_request.base.sha }} - HEAD_REF: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} run: bash scripts/governance-check.sh diff --git a/.github/workflows/ci-terraform.yaml b/.github/workflows/ci-terraform.yaml new file mode 100644 index 0000000..cc937fb --- /dev/null +++ b/.github/workflows/ci-terraform.yaml @@ -0,0 +1,60 @@ +name: Terraform CI + +# Static checks only. Plans run in HCP Terraform as speculative VCS runs on the +# PR (shoc-backend-dev and shoc-backend-staging). Applies are HCP auto-apply +# on merge to main (dev) and on a vX.Y.Z-staging tag (staging). + +on: + pull_request: + branches: [main, dev] + paths: + - "terraform/**" + - "scripts/**" + - ".github/workflows/ci-terraform.yaml" + - ".github/workflows/deploy.yaml" + - ".github/workflows/deploy-tag.yaml" + - ".github/workflows/release.yaml" + push: + branches: [main] + paths: + - "terraform/**" + - "scripts/**" + - ".github/workflows/ci-terraform.yaml" + +permissions: + contents: read + +jobs: + terraform: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: "1.9.8" + terraform_wrapper: false + + - name: Terraform fmt + run: terraform fmt -check -recursive terraform + + - name: Validate live/dev + run: | + terraform -chdir=terraform/live/dev init -backend=false + terraform -chdir=terraform/live/dev validate + + - name: Validate live/staging + run: | + terraform -chdir=terraform/live/staging init -backend=false + terraform -chdir=terraform/live/staging validate + + - name: Import plan guard tests + run: python3 scripts/test-terraform-import-plan-check.py + + - name: Release promotion script tests + run: | + python3 scripts/test_next_release_tag.py + python3 scripts/test_require_commit_checks.py + python3 scripts/test_check_app_terraform_isolation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24a071d..6e50e2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,8 @@ name: Backend CI on: pull_request: branches: [main, dev, staging] + push: + branches: [main] permissions: contents: read @@ -24,11 +26,6 @@ jobs: with: dotnet-version: "8.0.x" - - name: Set up Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: "1.9.8" - - name: Restore run: dotnet restore SeaHavenIndustries.sln @@ -37,29 +34,3 @@ jobs: - name: Test run: dotnet test SeaHavenIndustries.sln --no-build --configuration Release - - - name: Terraform fmt and validate - run: | - set -euo pipefail - - directories=() - case "${{ github.base_ref }}" in - dev) - directories+=(terraform/live/dev) - ;; - staging) - directories+=(terraform/live/staging) - ;; - esac - - for dir in "${directories[@]}"; do - terraform -chdir="$dir" fmt -check -recursive - terraform -chdir="$dir" init -backend=false - terraform -chdir="$dir" validate - done - - - name: Terraform import plan guard tests - run: python scripts/test-terraform-import-plan-check.py - - - name: Terraform release plan guard tests - run: python scripts/test-terraform-release-plan-check.py diff --git a/.github/workflows/deploy-tag.yaml b/.github/workflows/deploy-tag.yaml new file mode 100644 index 0000000..581fe97 --- /dev/null +++ b/.github/workflows/deploy-tag.yaml @@ -0,0 +1,43 @@ +name: Deploy API from tag + +# Human CLI escape hatch. GITHUB_TOKEN tag pushes from release.yaml do not +# start this workflow. No path filters. + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: read + +jobs: + target: + name: Resolve tag + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + environment: ${{ steps.resolve.outputs.environment }} + ref: ${{ steps.resolve.outputs.ref }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: resolve + env: + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + environment="$(python3 -c 'import os, sys; sys.path.insert(0, "scripts"); from next_release_tag import parse_environment_from_tag; print(parse_environment_from_tag(os.environ["TAG"]))')" + { + echo "environment=${environment}" + echo "ref=${TAG}" + } >> "${GITHUB_OUTPUT}" + + deploy: + needs: target + uses: ./.github/workflows/deploy.yaml + with: + environment: ${{ needs.target.outputs.environment }} + ref: ${{ needs.target.outputs.ref }} + secrets: inherit diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..59791dc --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,403 @@ +name: Deploy API + +# GitHub owns Elastic Beanstalk application versions. Terraform ignores +# version_label. Do not put path filters on tag events; those live in +# deploy-tag.yaml. + +on: + workflow_call: + inputs: + environment: + required: true + type: string + ref: + required: true + type: string + workflow_dispatch: + inputs: + environment: + description: Target Environment + required: true + type: choice + options: [dev, staging, prod] + ref: + description: Git ref to build (tag, branch, or SHA). Empty means this run's SHA. + required: false + type: string + default: "" + push: + branches: [main] + paths-ignore: + - "terraform/**" + - "**/*.md" + - ".github/workflows/ci.yml" + - ".github/workflows/ci-terraform.yaml" + - ".github/workflows/architecture-quality.yml" + - ".github/workflows/release.yaml" + - ".github/workflows/deploy-tag.yaml" + +permissions: + contents: read + +jobs: + target: + name: Resolve target + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + environment: ${{ steps.resolve.outputs.environment }} + ref: ${{ steps.resolve.outputs.ref }} + steps: + - id: resolve + env: + EVENT_NAME: ${{ github.event_name }} + CALL_ENVIRONMENT: ${{ inputs.environment }} + CALL_REF: ${{ inputs.ref }} + INPUT_ENVIRONMENT: ${{ github.event.inputs.environment }} + INPUT_REF: ${{ github.event.inputs.ref }} + GITHUB_SHA_IN: ${{ github.sha }} + run: | + set -euo pipefail + case "${EVENT_NAME}" in + workflow_call) + environment="${CALL_ENVIRONMENT}" + ref="${CALL_REF}" + ;; + workflow_dispatch) + environment="${INPUT_ENVIRONMENT}" + ref="${INPUT_REF:-${GITHUB_SHA_IN}}" + ;; + push) + environment=dev + ref="${GITHUB_SHA_IN}" + ;; + *) + echo "unsupported event ${EVENT_NAME}" >&2 + exit 1 + ;; + esac + case "${environment}" in + dev|staging|prod) ;; + *) + echo "unknown environment ${environment}" >&2 + exit 1 + ;; + esac + { + echo "environment=${environment}" + echo "ref=${ref}" + } >> "${GITHUB_OUTPUT}" + echo "Deploying ${ref} to ${environment}" + + deploy: + name: Deploy ${{ needs.target.outputs.environment }} + needs: target + runs-on: ubuntu-latest + timeout-minutes: 180 + environment: ${{ needs.target.outputs.environment }} + concurrency: + group: deploy-api-${{ needs.target.outputs.environment }} + cancel-in-progress: false + permissions: + contents: read + id-token: write + checks: read + env: + AWS_REGION: us-east-1 + DEPLOY_ROLE_ARN: ${{ vars.DEPLOY_ROLE_ARN }} + TARGET_ENVIRONMENT: ${{ needs.target.outputs.environment }} + TARGET_REF: ${{ needs.target.outputs.ref }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.target.outputs.ref }} + persist-credentials: false + fetch-depth: 0 + + - name: Resolve commit + id: commit + run: | + set -euo pipefail + sha="$(git rev-parse HEAD)" + echo "sha=${sha}" >> "${GITHUB_OUTPUT}" + echo "Building ${sha}" + + - name: Require tag on main + if: needs.target.outputs.environment != 'dev' + env: + REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + TAG_OR_REF: ${{ needs.target.outputs.ref }} + run: | + set -euo pipefail + status="$(gh api "repos/${REPO}/compare/main...${TAG_OR_REF}" --jq .status)" + if [ "${status}" != "behind" ] && [ "${status}" != "identical" ]; then + echo "ref ${TAG_OR_REF} is not on main (compare status: ${status})" >&2 + exit 1 + fi + + - name: Require CI on the SHA + if: needs.target.outputs.environment != 'dev' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/require_commit_checks.py \ + --repo "${{ github.repository }}" \ + --sha "${{ steps.commit.outputs.sha }}" \ + --timeout-seconds 60 + + - name: Skip prod AWS until live/prod exists + id: prod-gate + if: needs.target.outputs.environment == 'prod' + run: | + set -euo pipefail + if [ "${{ vars.PROD_APP_CD_ENABLED }}" = "true" ]; then + echo "skip_aws=false" >> "${GITHUB_OUTPUT}" + else + echo "PROD_APP_CD_ENABLED is not true; reviewers already approved; skipping AWS." + echo "skip_aws=true" >> "${GITHUB_OUTPUT}" + fi + + - name: Set up .NET + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + + - name: Build Elastic Beanstalk source bundle + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + run: bash scripts/package-elastic-beanstalk.sh + + - name: Validate exact release bundle + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + run: bash scripts/validate-elastic-beanstalk-bundle.sh + + - name: Configure AWS credentials using OIDC + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ env.DEPLOY_ROLE_ARN }} + aws-region: us-east-1 + audience: sts.amazonaws.com + + - name: Get deploy parameters + id: deploy + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + run: | + set -euo pipefail + prefix="/shoc-backend/${TARGET_ENVIRONMENT}/deploy" + APPLICATION=$(aws ssm get-parameter --name "${prefix}/application-name" --query Parameter.Value --output text) + ENVIRONMENT_NAME=$(aws ssm get-parameter --name "${prefix}/environment-name" --query Parameter.Value --output text) + ARTIFACTS_BUCKET=$(aws ssm get-parameter --name "${prefix}/artifacts-bucket" --query Parameter.Value --output text) + SMOKE_URL=$(aws ssm get-parameter --name "${prefix}/smoke-url" --query Parameter.Value --output text) + { + echo "application=${APPLICATION}" + echo "environment_name=${ENVIRONMENT_NAME}" + echo "artifacts_bucket=${ARTIFACTS_BUCKET}" + echo "smoke_url=${SMOKE_URL}" + } >> "${GITHUB_OUTPUT}" + + - name: Capture current environment version + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + env: + ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }} + run: | + set -euo pipefail + prev="$(aws elasticbeanstalk describe-environments \ + --environment-names "${ENVIRONMENT_NAME}" \ + --region us-east-1 \ + --query 'Environments[0].VersionLabel' \ + --output text)" + echo "previous_version_label=${prev}" >> "${GITHUB_ENV}" + echo "Previous version label: ${prev}" + + - name: Upload bundle and update environment + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + env: + APPLICATION: ${{ steps.deploy.outputs.application }} + ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }} + ARTIFACTS_BUCKET: ${{ steps.deploy.outputs.artifacts_bucket }} + GIT_SHA: ${{ steps.commit.outputs.sha }} + run: | + set -euo pipefail + version_label="${GIT_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + s3_key="shoc-backend/releases/${TARGET_ENVIRONMENT}/${GIT_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip" + aws s3 cp .artifacts/elastic-beanstalk/site.zip \ + "s3://${ARTIFACTS_BUCKET}/${s3_key}" \ + --region us-east-1 + aws elasticbeanstalk create-application-version \ + --application-name "${APPLICATION}" \ + --version-label "${version_label}" \ + --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --source-bundle "S3Bucket=${ARTIFACTS_BUCKET},S3Key=${s3_key}" \ + --process \ + --region us-east-1 + status="UNPROCESSED" + for _ in $(seq 1 36); do + status="$(aws elasticbeanstalk describe-application-versions \ + --application-name "${APPLICATION}" \ + --version-labels "${version_label}" \ + --region us-east-1 \ + --query 'ApplicationVersions[0].Status' \ + --output text)" + echo "application version status: $status" + if [ "$status" = "PROCESSED" ]; then + break + fi + if [ "$status" = "FAILED" ]; then + echo "Elastic Beanstalk failed to process ${version_label}." >&2 + exit 1 + fi + sleep 5 + done + if [ "$status" != "PROCESSED" ]; then + echo "Application version did not become PROCESSED." >&2 + exit 1 + fi + aws elasticbeanstalk update-environment \ + --environment-name "${ENVIRONMENT_NAME}" \ + --version-label "${version_label}" \ + --region us-east-1 + echo "version_label=${version_label}" >> "${GITHUB_ENV}" + echo "environment_updated=true" >> "${GITHUB_ENV}" + + - name: Verify exact application version is active + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + env: + ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }} + run: | + set -euo pipefail + expected="${version_label}" + status="Unknown" + current="Unknown" + health="Unknown" + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names "${ENVIRONMENT_NAME}" \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + if [ "$status" = "Ready" ]; then + if [ "$current" != "$expected" ]; then + echo "Environment became Ready on version $current, not the expected $expected." >&2 + exit 1 + fi + if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then + echo "Expected application version is Ready and healthy." + exit 0 + fi + echo "Expected version is active; waiting for health to leave $health." + fi + sleep 15 + done + echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2 + exit 1 + + - name: Post-deploy smoke + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + run: bash scripts/smoke-elastic-beanstalk.sh "${{ steps.deploy.outputs.smoke_url }}" + + - name: Verify webhook secret source is operational + if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true' + env: + SMOKE_URL: ${{ steps.deploy.outputs.smoke_url }} + run: | + set -euo pipefail + response_file="$(mktemp)" + trap 'rm -f "$response_file"' EXIT + status="$(curl --silent --show-error \ + --output "$response_file" \ + --write-out '%{http_code}' \ + --request POST \ + --header 'Content-Type: application/json' \ + --header "X-SH-Timestamp: $(date +%s)" \ + --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \ + --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \ + --data '{}' \ + "${SMOKE_URL}/api/webhooks/work-orders")" + if [ "$status" != "401" ]; then + echo "Expected 401 from enabled webhook; received $status." >&2 + sed -n '1,20p' "$response_file" >&2 + exit 1 + fi + + - name: Restore previous application version on failure (schema is not reverted) + if: ${{ failure() && !cancelled() && (needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true') }} + env: + ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }} + run: | + set -euo pipefail + prev="${previous_version_label:-}" + if [ "${environment_updated:-}" != "true" ]; then + echo "Environment was not updated; nothing to roll back." + exit 0 + fi + if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then + echo "No previous version recorded; nothing to roll back." >&2 + exit 0 + fi + if [ "$prev" = "${version_label:-}" ]; then + echo "Previous version is the failed release; nothing to roll back." >&2 + exit 0 + fi + + echo "Waiting for any in-flight environment update to settle..." + status="Unknown" + current="Unknown" + health="Unknown" + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names "${ENVIRONMENT_NAME}" \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + if [ "$status" = "Ready" ]; then + break + fi + sleep 15 + done + if [ "$status" != "Ready" ]; then + echo "Environment did not settle before rollback." >&2 + exit 1 + fi + if [ "$current" = "$prev" ]; then + echo "Environment is already on previous version $prev." + exit 0 + fi + + echo "Restoring previous application version $prev." + aws elasticbeanstalk update-environment \ + --environment-name "${ENVIRONMENT_NAME}" \ + --version-label "${prev}" \ + --region us-east-1 + + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names "${ENVIRONMENT_NAME}" \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + if [ "$status" = "Ready" ]; then + if [ "$current" != "$prev" ]; then + echo "Environment became Ready on version $current, not the previous $prev." >&2 + exit 1 + fi + if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then + echo "Previous application version is Ready and healthy." + exit 0 + fi + echo "Previous version is active; waiting for health to leave $health." + fi + sleep 15 + done + echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2 + exit 1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..777a11c --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,90 @@ +name: Release + +# Cut a SemVer tag from main HEAD, then call deploy. GITHUB_TOKEN is enough; +# it cannot start other workflows via the tag push, so this file calls deploy. + +on: + workflow_dispatch: + inputs: + environment: + description: Target environment + required: true + type: choice + options: [staging, prod] + bump: + description: SemVer bump from the last prod core tag + required: true + type: choice + options: [patch, minor, major] + message: + description: Annotated tag message and GitHub Release body + required: true + type: string + +permissions: + contents: write + checks: read + +jobs: + cut: + name: Cut tag + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + tag: ${{ steps.tag.outputs.tag }} + sha: ${{ steps.tag.outputs.sha }} + environment: ${{ github.event.inputs.environment }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Require main + run: | + set -euo pipefail + if [ "${GITHUB_REF}" != "refs/heads/main" ]; then + echo "Release must run from main (Use workflow from: main). Got ${GITHUB_REF}." >&2 + exit 1 + fi + + - name: Require CI + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/require_commit_checks.py \ + --repo "${{ github.repository }}" \ + --sha "$(git rev-parse HEAD)" \ + --timeout-seconds 1200 + + - name: Compute and push tag + id: tag + env: + ENVIRONMENT: ${{ github.event.inputs.environment }} + BUMP: ${{ github.event.inputs.bump }} + MESSAGE: ${{ github.event.inputs.message }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git fetch --tags origin + sha="$(git rev-parse HEAD)" + tag="$(git tag | python3 scripts/next_release_tag.py --environment "${ENVIRONMENT}" --bump "${BUMP}")" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "${tag}" -m "${MESSAGE}" "${sha}" + git push origin "refs/tags/${tag}" + gh release create "${tag}" --target "${sha}" --notes "${MESSAGE}" --title "${tag}" + { + echo "tag=${tag}" + echo "sha=${sha}" + } >> "${GITHUB_OUTPUT}" + echo "Created ${tag} at ${sha}" + + deploy: + name: Deploy release + needs: cut + uses: ./.github/workflows/deploy.yaml + with: + environment: ${{ needs.cut.outputs.environment }} + ref: ${{ needs.cut.outputs.tag }} + secrets: inherit diff --git a/QUALITY_GATES.md b/QUALITY_GATES.md index 9dd5b0a..9d6609e 100644 --- a/QUALITY_GATES.md +++ b/QUALITY_GATES.md @@ -25,8 +25,8 @@ | G8 | Error disclosure | §5 | `SanitizedErrorsTests` (part of G5) | `ci` | | G9 | Board-backed regression | review framework | `REVIEW_AND_PR_FRAMEWORK.md` inventory | review-enforced | | G10 | Terraform import plan safety | live infrastructure adoption | `python scripts/test-terraform-import-plan-check.py` | `architecture-quality` → `governance-check.sh` | -| G11 | Terraform static validation | import configuration integrity | commands below | `ci` on the matching PR base | -| G12 | Terraform release plan safety | dev application CD version_label | `python scripts/test-terraform-release-plan-check.py` | `architecture-quality` → `governance-check.sh` | +| G11 | Terraform static validation | import configuration integrity | commands below | `ci-terraform` on PRs to `main` or `dev` | +| G13 | App/Terraform isolation | separate delivery lanes | `python3 scripts/check_app_terraform_isolation.py` | `architecture-quality` → `governance-check.sh` | ## How to run locally @@ -36,44 +36,49 @@ bash scripts/governance-check.sh # By default it compares against the default branch for changed-file formatting. # Override the comparison point: -BASE_REF=origin/dev bash scripts/governance-check.sh +BASE_REF=origin/main bash scripts/governance-check.sh BASE_REF=main bash scripts/governance-check.sh ``` The script: 1. `dotnet restore` (G1) 2. runs the `ArchitectureTests` filter with `--no-restore` (G2) -3. computes changed `.cs` files vs `BASE_REF` (default `origin/dev`) and runs +3. computes changed `.cs` files vs `BASE_REF` (default `origin/main`) and runs `dotnet format --verify-no-changes --include ...` (G3). When there are no changed C# files it skips G3 with an explicit "skipped: no changed C#" line. 4. builds the complete solution in Release with no restore (G4). 5. runs the complete solution test suite in Release with no rebuild (G5). 6. verifies that the Terraform plan guard rejects create, delete, replacement, unmanaged resource types, and updates not allowlisted by exact address (G10). -7. verifies that the release plan guard accepts only a version-only update of - `module.environment.aws_elastic_beanstalk_environment.this` (G12). +7. runs the release-tag, commit-check, and isolation unit tests. +8. rejects a diff that contains both `terraform/` and deployable application + files (G13). G10 permits only exact approved resource address/type pairs for the environment-owned boundary: Elastic Beanstalk environment, IAM role/inline policy/managed-policy attachment/ instance profile, Secrets Manager secret metadata, and Route 53 record. -Initial mode permits no update. Controlled mode requires one +SSM `/shoc-backend//deploy/*` parameters are created after adoption and +are not part of the import allowlist. Initial mode permits no update. +Controlled mode requires one `--allow-update-address` argument per reviewed in-place update. Every invocation also requires `--environment dev` or `--environment staging`; an empty or incomplete environment plan fails. -G11 runs `terraform fmt -check -recursive`, `terraform init -backend=false`, -and `terraform validate`. PRs to `dev` validate `live/dev`. -PRs to `staging` validate only `live/staging`. Org-baseline CloudFormation owns -the HCP role substrate, and Terraform owns the dev deploy role, so no backend -CDK or bootstrap root remains in the matrix. +G11 runs `terraform fmt -check -recursive terraform`, `terraform init -backend=false`, +and `terraform validate` for both `live/dev` and `live/staging` on every PR to +`main` or `dev`. Org-baseline CloudFormation owns the HCP role substrate, and Terraform +owns the environment GitHub deploy roles, so no backend CDK or bootstrap root +remains in the matrix. HCP plan/apply roles stay in org-baseline; this +repository never manages `hcptf-*` roles. -G12 accepts only a local or downloaded plan JSON whose sole managed update is -`module.environment.aws_elastic_beanstalk_environment.this` with -`version_label` as the only changed attribute. Counts of `0` add / `1` change / -`0` destroy are not a substitute. The optional download uses -`GET /api/v2/plans/:id/json-output` on `app.terraform.io` with one redirect to -`archivist.terraform.io` and does not create, apply, discard, or poll runs. +The former G12 version-only HCP apply guard is not part of the repository gate. +`scripts/check-terraform-release-plan.py` remains only while +`.github/workflows/deploy.yml` is still present. + +G13 fails when the same diff contains both `terraform/` and deployable +application files. Workflow, documentation, and gate-script changes may share +a PR with either side. ## Migration gates (G6) diff --git a/REVIEW_AND_PR_FRAMEWORK.md b/REVIEW_AND_PR_FRAMEWORK.md index 3b32672..d0ea942 100644 --- a/REVIEW_AND_PR_FRAMEWORK.md +++ b/REVIEW_AND_PR_FRAMEWORK.md @@ -109,9 +109,12 @@ Suppressions are single-diagnostic and cite the ADR — **no wildcard suppressions** (no global `[SuppressMessage]`, no `.editorconfig` severity sweeps, no `#pragma` swaths). See architecture §10. -Do not mix deployable application changes with Terraform or CDK changes. The -first Terraform-owned application-CD change is the allowed exception because it -introduces `release_version_label`. Later PRs must keep those diffs separate. +Infra and application **PRs** stay separate. GitHub Actions owns Elastic +Beanstalk application versions. HCP Terraform owns infrastructure and ignores +`version_label`. A change set that includes both `terraform/` and deployable +application files (`.cs`, `.csproj`, `.razor`, `.ebextensions/`, or the +Elastic Beanstalk package/smoke scripts) fails the isolation check. Workflow, +docs, and gate-script changes may travel with either side. ## 8. PR description contract (minimal) diff --git a/scripts/check_app_terraform_isolation.py b/scripts/check_app_terraform_isolation.py new file mode 100644 index 0000000..9f87212 --- /dev/null +++ b/scripts/check_app_terraform_isolation.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Fail when a change set mixes Terraform with deployable application files. + +Workflow, docs, and gate-script changes may travel with either side. +""" + +from __future__ import annotations + +import argparse +import sys + +APP_SCRIPT_NAMES = { + "scripts/package-elastic-beanstalk.sh", + "scripts/validate-elastic-beanstalk-bundle.sh", + "scripts/smoke-elastic-beanstalk.sh", +} + +APP_SUFFIXES = (".cs", ".csproj", ".razor") + + +def is_terraform_path(path: str) -> bool: + return path == "terraform" or path.startswith("terraform/") + + +def is_app_path(path: str) -> bool: + normalized = path.replace("\\", "/") + if normalized in APP_SCRIPT_NAMES: + return True + if normalized.startswith(".ebextensions/"): + return True + return normalized.endswith(APP_SUFFIXES) + + +def isolation_violation(paths: list[str]) -> tuple[list[str], list[str]] | None: + terraform_files = sorted({path for path in paths if is_terraform_path(path)}) + app_files = sorted({path for path in paths if is_app_path(path)}) + if terraform_files and app_files: + return terraform_files, app_files + return None + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "paths", + nargs="*", + help="Changed paths. Omit and pass newline-separated paths on stdin.", + ) + args = parser.parse_args() + paths = list(args.paths) + if not paths and not sys.stdin.isatty(): + paths = [line.strip() for line in sys.stdin if line.strip()] + violation = isolation_violation(paths) + if violation is None: + print("PASS: application and Terraform changes are isolated") + return 0 + terraform_files, app_files = violation + print("FAIL: do not mix deployable application files with terraform/", file=sys.stderr) + print("terraform:", file=sys.stderr) + for path in terraform_files: + print(f" {path}", file=sys.stderr) + print("application:", file=sys.stderr) + for path in app_files: + print(f" {path}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/governance-check.sh b/scripts/governance-check.sh index a3b8416..b0a9dfb 100755 --- a/scripts/governance-check.sh +++ b/scripts/governance-check.sh @@ -8,7 +8,7 @@ # # Usage: # bash scripts/governance-check.sh -# BASE_REF=origin/dev bash scripts/governance-check.sh +# BASE_REF=origin/main bash scripts/governance-check.sh # BASE_REF= HEAD_REF= bash scripts/governance-check.sh set -euo pipefail @@ -30,9 +30,9 @@ else exit 1 fi -# Comparison point for changed-file formatting. Default to the dev integration -# branch locally; CI overrides BASE_REF/HEAD_REF with the PR base/head SHAs. -BASE_REF="${BASE_REF:-origin/dev}" +# Comparison point for changed-file formatting. Default to main locally; CI +# overrides BASE_REF/HEAD_REF with the PR base/head SHAs. +BASE_REF="${BASE_REF:-origin/main}" HEAD_REF="${HEAD_REF:-HEAD}" # Resolve the base ref before using it for a diff. @@ -83,8 +83,16 @@ log "G10: Terraform import plan safety" python scripts/test-terraform-import-plan-check.py ok "G10: Terraform import plan safety" -log "G12: Terraform release plan safety" -python scripts/test-terraform-release-plan-check.py -ok "G12: Terraform release plan safety" +log "Release promotion scripts" +python3 scripts/test_next_release_tag.py +python3 scripts/test_require_commit_checks.py +python3 scripts/test_check_app_terraform_isolation.py +ok "Release promotion scripts" + +log "G13: application and Terraform isolation (${BASE_REF}..${HEAD_REF})" +python3 scripts/check_app_terraform_isolation.py < <( + git diff --name-only --diff-filter=ACMR "${BASE_REF}" "${HEAD_REF}" +) +ok "G13: application and Terraform isolation" log "governance-check: all required repository gates passed" diff --git a/scripts/next_release_tag.py b/scripts/next_release_tag.py new file mode 100644 index 0000000..df203e0 --- /dev/null +++ b/scripts/next_release_tag.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Compute the next SemVer git tag for a staging or prod cut. + +Base version is the highest existing core prod tag vX.Y.Z (not -staging). +Missing tags start at 0.0.0. Staging gets v{next}-staging; prod gets v{next}. +""" + +from __future__ import annotations + +import argparse +import re +import sys + +PROD_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") +STAGING_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-staging$") + + +def parse_environment_from_tag(tag: str) -> str: + if STAGING_TAG.fullmatch(tag): + return "staging" + if PROD_TAG.fullmatch(tag): + return "prod" + raise ValueError( + f"tag {tag!r} is not vX.Y.Z or vX.Y.Z-staging" + ) + + +def highest_prod_core(tags: list[str]) -> tuple[int, int, int]: + cores: list[tuple[int, int, int]] = [] + for tag in tags: + match = PROD_TAG.fullmatch(tag) + if match: + cores.append(tuple(int(part) for part in match.groups())) # type: ignore[arg-type] + if not cores: + return (0, 0, 0) + return max(cores) + + +def bump_core(core: tuple[int, int, int], bump: str) -> tuple[int, int, int]: + major, minor, patch = core + if bump == "major": + return (major + 1, 0, 0) + if bump == "minor": + return (major, minor + 1, 0) + if bump == "patch": + return (major, minor, patch + 1) + raise ValueError(f"bump must be major, minor, or patch, got {bump!r}") + + +def format_tag(core: tuple[int, int, int], environment: str) -> str: + name = f"v{core[0]}.{core[1]}.{core[2]}" + if environment == "staging": + return f"{name}-staging" + if environment == "prod": + return name + raise ValueError(f"environment must be staging or prod, got {environment!r}") + + +def next_release_tag( + tags: list[str], environment: str, bump: str +) -> str: + if environment not in {"staging", "prod"}: + raise ValueError(f"environment must be staging or prod, got {environment!r}") + nxt = bump_core(highest_prod_core(tags), bump) + tag = format_tag(nxt, environment) + if tag in tags: + raise ValueError(f"tag {tag} already exists") + return tag + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--environment", required=True, choices=("staging", "prod")) + parser.add_argument("--bump", required=True, choices=("major", "minor", "patch")) + parser.add_argument( + "--tag", + action="append", + default=[], + dest="tags", + help="Existing git tag. Repeat, or omit and pass tags on stdin.", + ) + args = parser.parse_args() + tags = list(args.tags) + if not tags and not sys.stdin.isatty(): + tags = [line.strip() for line in sys.stdin if line.strip()] + try: + print(next_release_tag(tags, args.environment, args.bump)) + except ValueError as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/require_commit_checks.py b/scripts/require_commit_checks.py new file mode 100644 index 0000000..c2d6bdd --- /dev/null +++ b/scripts/require_commit_checks.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Fail unless required GitHub check runs succeeded on a commit SHA.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +REQUIRED_CHECK_NAMES = ( + "Build and test", + "architecture", +) + + +def _run_recency(run: dict) -> tuple: + """Order check runs so a later rerun wins over an earlier result.""" + started = run.get("started_at") or "" + completed = run.get("completed_at") or "" + run_id = run.get("id") or 0 + return (started, completed, run_id) + + +def classify_checks( + check_runs: list[dict], required_names: tuple[str, ...] = REQUIRED_CHECK_NAMES +) -> tuple[str, list[str]]: + """Return ('success'|'pending'|'failure', detail lines).""" + by_name: dict[str, dict] = {} + for run in check_runs: + name = run.get("name") + if name not in required_names: + continue + current = by_name.get(name) + if current is None or _run_recency(run) > _run_recency(current): + by_name[name] = run + + missing = [name for name in required_names if name not in by_name] + if missing: + return "pending", [f"missing: {name}" for name in missing] + + details: list[str] = [] + pending = False + failed = False + for name in required_names: + run = by_name[name] + status = run.get("status") + conclusion = run.get("conclusion") + details.append(f"{name} status={status} conclusion={conclusion}") + if status != "completed": + pending = True + elif conclusion != "success": + failed = True + if failed: + return "failure", details + if pending: + return "pending", details + return "success", details + + +def fetch_check_runs(repo: str, sha: str, token: str) -> list[dict]: + url = ( + f"https://api.github.com/repos/{repo}/commits/{urllib.parse.quote(sha)}" + "/check-runs?per_page=100" + ) + request = urllib.request.Request( + url, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request) as response: + payload = json.load(response) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", "replace") + raise SystemExit(f"GitHub check-runs HTTP {exc.code}: {body}") from exc + return payload.get("check_runs") or [] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--sha", required=True) + parser.add_argument("--timeout-seconds", type=int, default=1200) + parser.add_argument("--poll-seconds", type=int, default=15) + args = parser.parse_args() + token = __import__("os").environ.get("GITHUB_TOKEN") or __import__("os").environ.get( + "GH_TOKEN" + ) + if not token: + print("FAIL: GITHUB_TOKEN or GH_TOKEN is required", file=sys.stderr) + return 1 + + deadline = time.time() + args.timeout_seconds + while True: + runs = fetch_check_runs(args.repo, args.sha, token) + state, details = classify_checks(runs) + for line in details: + print(line) + if state == "success": + print(f"PASS: required checks succeeded on {args.sha}") + return 0 + if state == "failure": + print(f"FAIL: required checks did not succeed on {args.sha}", file=sys.stderr) + return 1 + if time.time() >= deadline: + print( + f"FAIL: timed out waiting for required checks on {args.sha}", + file=sys.stderr, + ) + return 1 + print(f"waiting {args.poll_seconds}s for checks...") + time.sleep(args.poll_seconds) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_check_app_terraform_isolation.py b/scripts/test_check_app_terraform_isolation.py new file mode 100644 index 0000000..94899a6 --- /dev/null +++ b/scripts/test_check_app_terraform_isolation.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Tests for check_app_terraform_isolation.isolation_violation.""" + +from __future__ import annotations + +import unittest + +from check_app_terraform_isolation import isolation_violation + + +class IsolationTests(unittest.TestCase): + def test_terraform_only(self) -> None: + self.assertIsNone( + isolation_violation( + [ + "terraform/live/dev/main.tf", + "terraform/live/README.md", + ] + ) + ) + + def test_app_only(self) -> None: + self.assertIsNone( + isolation_violation( + [ + "Api.SeaHavenIndustries/Controllers/WorkOrderController.cs", + ".ebextensions/01_migrations.config", + "scripts/package-elastic-beanstalk.sh", + ] + ) + ) + + def test_docs_and_workflows_with_terraform(self) -> None: + self.assertIsNone( + isolation_violation( + [ + "terraform/live/modules/environment-owned/main.tf", + ".github/workflows/deploy.yaml", + "QUALITY_GATES.md", + "scripts/governance-check.sh", + ] + ) + ) + + def test_mixed_app_and_terraform_fails(self) -> None: + violation = isolation_violation( + [ + "terraform/live/dev/main.tf", + "SeaHaven.Services/Implementation/WorkOrderService.cs", + ] + ) + self.assertIsNotNone(violation) + terraform_files, app_files = violation or ([], []) + self.assertEqual(terraform_files, ["terraform/live/dev/main.tf"]) + self.assertTrue(any(path.endswith(".cs") for path in app_files)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_next_release_tag.py b/scripts/test_next_release_tag.py new file mode 100644 index 0000000..88a65f6 --- /dev/null +++ b/scripts/test_next_release_tag.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Tests for next_release_tag.py.""" + +from __future__ import annotations + +import unittest + +from next_release_tag import ( + next_release_tag, + parse_environment_from_tag, +) + + +class NextReleaseTagTests(unittest.TestCase): + def test_first_patch_staging(self) -> None: + self.assertEqual(next_release_tag([], "staging", "patch"), "v0.0.1-staging") + + def test_first_minor_prod(self) -> None: + self.assertEqual(next_release_tag([], "prod", "minor"), "v0.1.0") + + def test_first_major_prod(self) -> None: + self.assertEqual(next_release_tag([], "prod", "major"), "v1.0.0") + + def test_staging_then_prod_same_core(self) -> None: + tags = ["v1.2.3"] + staging = next_release_tag(tags, "staging", "patch") + self.assertEqual(staging, "v1.2.4-staging") + prod = next_release_tag(tags + [staging], "prod", "patch") + self.assertEqual(prod, "v1.2.4") + + def test_staging_prerelease_does_not_raise_prod_base(self) -> None: + tags = ["v1.2.3", "v9.9.9-staging"] + self.assertEqual(next_release_tag(tags, "staging", "patch"), "v1.2.4-staging") + + def test_duplicate_staging_fails(self) -> None: + tags = ["v1.2.3", "v1.2.4-staging"] + with self.assertRaises(ValueError): + next_release_tag(tags, "staging", "patch") + + def test_prod_after_staging_uses_same_core(self) -> None: + tags = ["v1.2.3", "v9.9.9-staging"] + self.assertEqual(next_release_tag(tags, "prod", "patch"), "v1.2.4") + + def test_parse_environment(self) -> None: + self.assertEqual(parse_environment_from_tag("v1.2.3-staging"), "staging") + self.assertEqual(parse_environment_from_tag("v1.2.3"), "prod") + + def test_reject_prefixed_and_prod_prerelease(self) -> None: + for tag in ( + "staging-v1.2.3", + "prod-v1.2.3", + "v1.2.3-prod", + "v1.2.3-staging.1", + "v1.2", + "1.2.3", + ): + with self.assertRaises(ValueError): + parse_environment_from_tag(tag) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_require_commit_checks.py b/scripts/test_require_commit_checks.py new file mode 100644 index 0000000..e3089d2 --- /dev/null +++ b/scripts/test_require_commit_checks.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Tests for require_commit_checks.classify_checks.""" + +from __future__ import annotations + +import unittest + +from require_commit_checks import classify_checks + + +class ClassifyChecksTests(unittest.TestCase): + def test_success(self) -> None: + state, _ = classify_checks( + [ + {"name": "Build and test", "status": "completed", "conclusion": "success"}, + {"name": "architecture", "status": "completed", "conclusion": "success"}, + ] + ) + self.assertEqual(state, "success") + + def test_pending_missing(self) -> None: + state, details = classify_checks( + [ + {"name": "Build and test", "status": "completed", "conclusion": "success"}, + ] + ) + self.assertEqual(state, "pending") + self.assertTrue(any("architecture" in line for line in details)) + + def test_pending_in_progress(self) -> None: + state, _ = classify_checks( + [ + {"name": "Build and test", "status": "in_progress", "conclusion": None}, + {"name": "architecture", "status": "completed", "conclusion": "success"}, + ] + ) + self.assertEqual(state, "pending") + + def test_failure(self) -> None: + state, _ = classify_checks( + [ + {"name": "Build and test", "status": "completed", "conclusion": "failure"}, + {"name": "architecture", "status": "completed", "conclusion": "success"}, + ] + ) + self.assertEqual(state, "failure") + + def test_latest_rerun_success_wins_over_older_failure(self) -> None: + state, _ = classify_checks( + [ + { + "name": "Build and test", + "id": 1, + "started_at": "2026-09-16T12:00:00Z", + "completed_at": "2026-09-16T12:05:00Z", + "status": "completed", + "conclusion": "failure", + }, + { + "name": "Build and test", + "id": 3, + "started_at": "2026-09-16T12:10:00Z", + "completed_at": "2026-09-16T12:12:00Z", + "status": "completed", + "conclusion": "success", + }, + { + "name": "architecture", + "id": 2, + "started_at": "2026-09-16T12:00:00Z", + "completed_at": "2026-09-16T12:04:00Z", + "status": "completed", + "conclusion": "success", + }, + ] + ) + self.assertEqual(state, "success") + + def test_latest_rerun_failure_wins_over_older_success(self) -> None: + state, _ = classify_checks( + [ + { + "name": "architecture", + "id": 9, + "started_at": "2026-09-16T13:00:00Z", + "completed_at": "2026-09-16T13:02:00Z", + "status": "completed", + "conclusion": "failure", + }, + { + "name": "architecture", + "id": 4, + "started_at": "2026-09-16T12:00:00Z", + "completed_at": "2026-09-16T12:01:00Z", + "status": "completed", + "conclusion": "success", + }, + { + "name": "Build and test", + "id": 5, + "started_at": "2026-09-16T12:00:00Z", + "completed_at": "2026-09-16T12:03:00Z", + "status": "completed", + "conclusion": "success", + }, + ] + ) + self.assertEqual(state, "failure") + + +if __name__ == "__main__": + unittest.main() diff --git a/terraform/README.md b/terraform/README.md index 9821ba7..d4d2023 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -36,17 +36,20 @@ update, delete, or replacement actions. The second reviewed phase may update only explicitly allowlisted ownership metadata and the narrowed dev deploy S3 policy. -The GitHub Environment secret `AWS_DEPLOY_ROLE_ARN` retains the existing role -ARN throughout adoption. +The GitHub Environment **variable** `DEPLOY_ROLE_ARN` is the OIDC role used +by application CD after cutover. Adoption may still have the older +`AWS_DEPLOY_ROLE_ARN` secret until that cutover. ## Local validation ```bash -terraform -chdir=terraform fmt -check -recursive +terraform fmt -check -recursive terraform terraform -chdir=terraform/live/dev init -backend=false terraform -chdir=terraform/live/dev validate terraform -chdir=terraform/live/staging init -backend=false terraform -chdir=terraform/live/staging validate python scripts/test-terraform-import-plan-check.py -python scripts/test-terraform-release-plan-check.py +python3 scripts/test_next_release_tag.py +python3 scripts/test_require_commit_checks.py +python3 scripts/test_check_app_terraform_isolation.py ``` diff --git a/terraform/live/README.md b/terraform/live/README.md index 07a6f25..ae77fcf 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -127,59 +127,45 @@ The measured self-contained .NET/EF bundle is approximately 199.5 MB and separate builds are not byte-identical. Each deploy job therefore validates the exact bundle it uploads; bundle bytes never enter Terraform plans or state. -## Dev application CD +## Application CD -GitHub compiles, validates, and uploads the bundle, then creates the immutable -Elastic Beanstalk application version. HCP Terraform is the only caller of -`UpdateEnvironment`, by setting `version_label` on -`module.environment.aws_elastic_beanstalk_environment.this`. GitHub then -health-checks, smokes, and requests one guarded Terraform rollback. Terraform -does not manage `aws_elastic_beanstalk_application_version`; retained versions -are the rollback inventory. +GitHub Actions owns application versions. It compiles the bundle, uploads it, +creates the Elastic Beanstalk application version, and calls +`UpdateEnvironment`. Terraform ignores `version_label` so those deploys are not +drift. If health, smoke, or the webhook probe fails after that update, the job +restores the previous Elastic Beanstalk version label. Database migrations +already applied by the failed bundle are not reverted. Deploy parameters are read from `/shoc-backend//deploy/*` SSM +parameters this module writes. -`release_version_label` is a nullable root and module variable. Null VCS plans -leave the live version unchanged. Application-CD runs pass the immutable -`--` label only as a run-specific -`TF_VAR_release_version_label` HCL string. Do not set this variable on the -workspace, in a variable set, or in `terraform.tfvars`. Do not upload a new -configuration version on application releases; `create-run` reuses the -workspace's last applied VCS config. Global auto-apply stays off. GitHub -`apply-run` treats an already-applied run as success so a mis-set auto-apply -cannot start a false-failure rollback. The workspace stays branch-based on -`dev` with Automatic Speculative Plans enabled and trigger patterns -`terraform/live/dev/**` and `terraform/live/modules/**`. GitHub discards a -leftover non-speculative VCS run before `create-run`, so a merge to `dev` -cannot lock the workspace out from under GitHub CD. GitHub applies only after -`plan-output` counts are `0/1/0` and -`scripts/check-terraform-release-plan.py` accepts a version-only plan JSON. +Merge to `main` deploys **dev** unless the push is terraform-only. Staging and +prod are cut from **Actions → Release** (`environment`, `bump`, `message`). +That workflow waits for CI, tags `vX.Y.Z-staging` or `vX.Y.Z` from main HEAD +with `GITHUB_TOKEN`, then calls deploy. Prod waits on GitHub Environment +reviewers; AWS steps stay skipped until `PROD_APP_CD_ENABLED` is true. +Staging remains `adoption_complete=false` with a pinned API CNAME until its +import apply is proven. -Staging application CD uses the same guarded lane against workspace -`shoc-backend-staging`. The workspace stays branch-based on `staging` with -trigger patterns `terraform/live/staging/**` and `terraform/live/modules/**`, -and GitHub deploys on pushes to `staging` and on manual `workflow_dispatch`. +HCP workspaces stay VCS-driven with auto-apply on after cutover. Speculative +plans on every PR are the infra gate. Do not point `TFC_AWS_*` at +`hcptf-bootstrap`. Org-baseline CloudFormation owns the HCP plan/apply roles. +GitHub Actions does not create, wait on, or apply HCP runs. Application and +Terraform changes stay in separate PRs so a merge cannot race an HCP apply +against an app deploy. Terraform-only merges skip `deploy.yaml`. App-only +tags skip HCP when trigger patterns do not match. -### Credentials and enablement +Until cutover, `.github/workflows/deploy.yml` still uses `TF_API_TOKEN` and +`TERRAFORM_APP_CD_ENABLED`. Keep those secrets and the version-only plan guard +on that leftover path only. -Store dedicated HCP team tokens as the GitHub environment secret `TF_API_TOKEN`: +### Credentials -- `dev`: use a token scoped only to workspace `shoc-backend-dev`. -- `staging`: use a separate token scoped only to workspace - `shoc-backend-staging`. - -Plan JSON download requires workspace admin on the corresponding workspace. Do -not grant project admin or workspace create/move/delete permissions. Do not rely -on a repository-level token or reuse the dev-scoped token for staging. Rotate -each token at least every 90 days. - -Repository variable `TERRAFORM_APP_CD_ENABLED` starts unset/false so pushes to -`dev` do not deploy. `workflow_dispatch` on `dev` still runs a release for the -first manual proof. Set the variable to `true` only after that proof confirms -the exact version, a version-only plan, apply, `efbundle`, Ready/Green, smokes, -and a retained previous version. - -This change is the allowed exception that mixes deployable application CD with -the Terraform variable that application CD needs. Later PRs must not mix -deployable application changes with Terraform or CDK changes. +Store `DEPLOY_ROLE_ARN` as a GitHub Environment **variable** (`dev`, +`staging`, later `prod`). OIDC trust is +`repo:Sea-Haven-Industries/shoc-backend:environment:` plus +`job_workflow_ref` for `.github/workflows/deploy.yaml` at `refs/heads/main` +and `refs/tags/v*`. Adding another deploy workflow is a cross-family IAM +change. After cutover, drop `TF_API_TOKEN` from GitHub Environments. The new +CD path does not use it. ## Pinned live identities @@ -194,11 +180,12 @@ identifiers make accidental cross-environment reuse fail review and planning. ## Safety invariants -- Auto-apply remains off. -- VCS stays branch-based on `dev` for `shoc-backend-dev` and on `staging` for - `shoc-backend-staging`, with speculative PR plans enabled and trigger - patterns `terraform/live/dev/**` (dev) and `terraform/live/staging/**` - (staging), each alongside `terraform/live/modules/**`. Do not switch - Automatic Run Triggering to tag-based. -- Org baseline owns final HCP plan/apply permissions and manager tags. +- After cutover, auto-apply is on. Speculative PR plans stay on. +- `shoc-backend-dev` is branch-based on `main` with trigger patterns + `terraform/live/dev/**` and `terraform/live/modules/**`. +- `shoc-backend-staging` is tag-based on `^v\d+\.\d+\.\d+-staging$` with + trigger patterns `terraform/live/staging/**` and `terraform/live/modules/**`. +- Org baseline owns HCP plan/apply permissions and manager tags. Never manage + `hcptf-*` in this repository. - Every imported Terraform resource has `prevent_destroy`. +- Elastic Beanstalk `version_label` is ignored so GitHub deploys are not drift. diff --git a/terraform/live/dev/main.tf b/terraform/live/dev/main.tf index 8d1ce47..33e1416 100644 --- a/terraform/live/dev/main.tf +++ b/terraform/live/dev/main.tf @@ -67,7 +67,7 @@ module "environment" { github_deploy_role_name = "githubdeploy-shoc-backend-dev" github_deploy_policy_name = "GithubDeployRoleDefaultPolicyE8F540D1" legacy_dev_s3_policy = true - release_version_label = var.release_version_label + smoke_url = "https://api.dev.seahaven.com" hosted_zone_id = "Z07671212N75U4YLPWZR8" api_domain = local.api_domain api_record_type = "A" diff --git a/terraform/live/dev/variables.tf b/terraform/live/dev/variables.tf deleted file mode 100644 index 3f21d9b..0000000 --- a/terraform/live/dev/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "release_version_label" { - type = string - default = null - nullable = true - - description = "Immutable Elastic Beanstalk application version. Null VCS plans leave the live version unchanged." - - validation { - condition = ( - var.release_version_label == null || - can(regex("^[0-9a-f]{40}-[0-9]+-[0-9]+$", var.release_version_label)) - ) - error_message = "release_version_label must be --." - } -} diff --git a/terraform/live/modules/environment-owned/main.tf b/terraform/live/modules/environment-owned/main.tf index 29b42f2..cc01dc4 100644 --- a/terraform/live/modules/environment-owned/main.tf +++ b/terraform/live/modules/environment-owned/main.tf @@ -11,6 +11,7 @@ locals { eb_bucket_name = "elasticbeanstalk-${var.aws_region}-${var.aws_account_id}" use_legacy_s3_policy = var.legacy_dev_s3_policy app_config_secret_pattern = "arn:aws:secretsmanager:${var.aws_region}:${var.aws_account_id}:secret:${var.app_config_secret_name}-*" + deploy_ssm_prefix = "/shoc-backend/${var.environment}/deploy" } data "aws_iam_policy_document" "runtime_assume" { @@ -177,6 +178,18 @@ data "aws_iam_policy_document" "deploy_assume" { variable = "token.actions.githubusercontent.com:sub" values = ["repo:${var.github_repo}:environment:${var.github_environment}"] } + + # StringLike: a tag-loaded reusable workflow uses @refs/tags/v*, while + # push and workflow_dispatch use @refs/heads/main. Adding a deploy + # workflow means adding its ref here (cross-family IAM). + condition { + test = "StringLike" + variable = "token.actions.githubusercontent.com:job_workflow_ref" + values = [ + "${var.github_repo}/.github/workflows/deploy.yaml@refs/heads/main", + "${var.github_repo}/.github/workflows/deploy.yaml@refs/tags/v*", + ] + } } } @@ -293,6 +306,18 @@ data "aws_iam_policy_document" "deploy" { resources = ["arn:aws:s3:::${local.eb_bucket_name}"] } } + + statement { + sid = "ReadDeployParameters" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = [ + "arn:aws:ssm:${var.aws_region}:${var.aws_account_id}:parameter/shoc-backend/${var.environment}/deploy/*", + ] + } } resource "aws_iam_role_policy" "github_deploy" { @@ -305,6 +330,34 @@ resource "aws_iam_role_policy" "github_deploy" { } } +resource "aws_ssm_parameter" "deploy_application_name" { + name = "${local.deploy_ssm_prefix}/application-name" + type = "String" + value = var.eb_application_name + description = "Elastic Beanstalk application name; GitHub CD creates application versions here" +} + +resource "aws_ssm_parameter" "deploy_environment_name" { + name = "${local.deploy_ssm_prefix}/environment-name" + type = "String" + value = var.eb_environment_name + description = "Elastic Beanstalk environment name; GitHub CD calls UpdateEnvironment" +} + +resource "aws_ssm_parameter" "deploy_artifacts_bucket" { + name = "${local.deploy_ssm_prefix}/artifacts-bucket" + type = "String" + value = local.eb_bucket_name + description = "Bucket for release zips; GitHub CD uploads site.zip here" +} + +resource "aws_ssm_parameter" "deploy_smoke_url" { + name = "${local.deploy_ssm_prefix}/smoke-url" + type = "String" + value = var.smoke_url + description = "HTTPS origin for post-deploy smoke checks" +} + locals { managed_eb_settings = concat( [ @@ -444,16 +497,13 @@ locals { } resource "aws_elastic_beanstalk_environment" "this" { - # Null VCS plans omit this Optional+Computed argument, so the provider - # refreshes the live label without reverting releases. Application-CD runs - # pass an immutable -- value as a run-specific - # TF_VAR_release_version_label. - name = var.eb_environment_name - application = var.eb_application_name - platform_arn = var.platform_arn - version_label = var.release_version_label - tier = "WebServer" - cname_prefix = var.eb_environment_name + # GitHub Actions owns application versions via UpdateEnvironment. + # version_label is ignored so app deploys are not Terraform drift. + name = var.eb_environment_name + application = var.eb_application_name + platform_arn = var.platform_arn + tier = "WebServer" + cname_prefix = var.eb_environment_name dynamic "setting" { for_each = var.manage_eb_settings ? local.managed_eb_settings : [] @@ -471,6 +521,7 @@ resource "aws_elastic_beanstalk_environment" "this" { prevent_destroy = true ignore_changes = [ wait_for_ready_timeout, + version_label, ] } } diff --git a/terraform/live/modules/environment-owned/variables.tf b/terraform/live/modules/environment-owned/variables.tf index 71a97cd..d72d630 100644 --- a/terraform/live/modules/environment-owned/variables.tf +++ b/terraform/live/modules/environment-owned/variables.tf @@ -200,20 +200,9 @@ variable "legacy_dev_s3_policy" { default = false } -variable "release_version_label" { - type = string - default = null - nullable = true - - description = "Immutable Elastic Beanstalk application version. Null VCS plans leave the live version unchanged." - - validation { - condition = ( - var.release_version_label == null || - can(regex("^[0-9a-f]{40}-[0-9]+-[0-9]+$", var.release_version_label)) - ) - error_message = "release_version_label must be --." - } +variable "smoke_url" { + type = string + description = "HTTPS origin used by post-deploy smoke checks. Written to SSM for GitHub Actions." } variable "hosted_zone_id" { diff --git a/terraform/live/staging/main.tf b/terraform/live/staging/main.tf index 01391d5..0368da2 100644 --- a/terraform/live/staging/main.tf +++ b/terraform/live/staging/main.tf @@ -58,11 +58,11 @@ module "environment" { github_deploy_role_name = "githubdeploy-shoc-backend-staging" github_deploy_policy_name = "GithubDeployRoleDefaultPolicyE8F540D1" legacy_dev_s3_policy = false + smoke_url = "https://api.staging.seahaven.com" hosted_zone_id = "Z02602739VQWBWCAGXP4" api_domain = local.api_domain api_record_type = "CNAME" api_cname_target = "awseb--AWSEB-pPXqiRgNnZe8-16996010.us-east-1.elb.amazonaws.com" - release_version_label = var.release_version_label metadata_before_adoption = { runtime_role_description = "SHOC backend staging compute role (EB instance profile)" runtime_role_tags = { diff --git a/terraform/live/staging/variables.tf b/terraform/live/staging/variables.tf deleted file mode 100644 index 3f21d9b..0000000 --- a/terraform/live/staging/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "release_version_label" { - type = string - default = null - nullable = true - - description = "Immutable Elastic Beanstalk application version. Null VCS plans leave the live version unchanged." - - validation { - condition = ( - var.release_version_label == null || - can(regex("^[0-9a-f]{40}-[0-9]+-[0-9]+$", var.release_version_label)) - ) - error_message = "release_version_label must be --." - } -} From 7eaa7fb3f93021c8b70c0181499d3839561a1987 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Fri, 18 Sep 2026 11:22:50 -0400 Subject: [PATCH 2/8] docs(cd): document GitHub Environment branch and tag policies --- terraform/live/README.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/terraform/live/README.md b/terraform/live/README.md index ae77fcf..0c78f8f 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -137,11 +137,11 @@ restores the previous Elastic Beanstalk version label. Database migrations already applied by the failed bundle are not reverted. Deploy parameters are read from `/shoc-backend//deploy/*` SSM parameters this module writes. -Merge to `main` deploys **dev** unless the push is terraform-only. Staging and -prod are cut from **Actions → Release** (`environment`, `bump`, `message`). -That workflow waits for CI, tags `vX.Y.Z-staging` or `vX.Y.Z` from main HEAD -with `GITHUB_TOKEN`, then calls deploy. Prod waits on GitHub Environment -reviewers; AWS steps stay skipped until `PROD_APP_CD_ENABLED` is true. +Merge to `main` deploys **dev** unless the push is terraform-only. Staging is +cut from **Actions → Release** (`environment`, `bump`, `message`). That +workflow waits for CI, tags `vX.Y.Z-staging` from main HEAD with +`GITHUB_TOKEN`, then calls deploy. Do not cut prod yet; leave +`PROD_APP_CD_ENABLED` unset and do not create the `prod` GitHub Environment. Staging remains `adoption_complete=false` with a pinned API CNAME until its import apply is proven. @@ -160,13 +160,32 @@ on that leftover path only. ### Credentials Store `DEPLOY_ROLE_ARN` as a GitHub Environment **variable** (`dev`, -`staging`, later `prod`). OIDC trust is +`staging`). OIDC trust is `repo:Sea-Haven-Industries/shoc-backend:environment:` plus `job_workflow_ref` for `.github/workflows/deploy.yaml` at `refs/heads/main` and `refs/tags/v*`. Adding another deploy workflow is a cross-family IAM change. After cutover, drop `TF_API_TOKEN` from GitHub Environments. The new CD path does not use it. +GitHub Environment deployment branch and tag policies are repository +settings, not this diff. Update them before the first merge to `main` and +the first staging cut. The policy matches `GITHUB_REF` of the workflow run. +Branch patterns never match tag refs; adding `v*` as a branch pattern fails +the same way as an empty allowlist. + +1. `dev` — allow branch `main`. Keep `dev` allowed while leftover + `.github/workflows/deploy.yml` still deploys from that branch. +2. `staging` — add a **tag-type** policy matching `v*.*.*-staging` for + `deploy-tag.yaml`. Allow branch `main` because Actions → Release is + `workflow_dispatch` on `main` and then calls `deploy.yaml` + (`GITHUB_TOKEN` tag pushes do not start `deploy-tag.yaml`). Keep + `staging` allowed while leftover `deploy.yml` still deploys from that + branch. + +Do not create the `prod` environment yet. Leave `PROD_APP_CD_ENABLED` +unset. Do not run Actions → Release with `environment=prod`; the first +prod dispatch would auto-create an unprotected environment. + ## Pinned live identities - Dev: workspace `shoc-backend-dev`; EB environment `shoc-backend-dev` From 396b1c690bfbc776ece47d5a609e48ffccbc860d Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Fri, 18 Sep 2026 11:33:34 -0400 Subject: [PATCH 3/8] fix(cd): honor reusable workflow inputs when resolving deploy target --- .github/workflows/deploy.yaml | 40 +++++++++++++++++++---------------- terraform/live/README.md | 7 ++++-- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 59791dc..7ff766e 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -58,24 +58,28 @@ jobs: GITHUB_SHA_IN: ${{ github.sha }} run: | set -euo pipefail - case "${EVENT_NAME}" in - workflow_call) - environment="${CALL_ENVIRONMENT}" - ref="${CALL_REF}" - ;; - workflow_dispatch) - environment="${INPUT_ENVIRONMENT}" - ref="${INPUT_REF:-${GITHUB_SHA_IN}}" - ;; - push) - environment=dev - ref="${GITHUB_SHA_IN}" - ;; - *) - echo "unsupported event ${EVENT_NAME}" >&2 - exit 1 - ;; - esac + # A called reusable workflow keeps the caller's github.event_name + # (push or workflow_dispatch), not workflow_call. Prefer the call + # inputs whenever they are set. + if [ -n "${CALL_ENVIRONMENT}" ]; then + environment="${CALL_ENVIRONMENT}" + ref="${CALL_REF:-${GITHUB_SHA_IN}}" + else + case "${EVENT_NAME}" in + workflow_dispatch) + environment="${INPUT_ENVIRONMENT}" + ref="${INPUT_REF:-${GITHUB_SHA_IN}}" + ;; + push) + environment=dev + ref="${GITHUB_SHA_IN}" + ;; + *) + echo "unsupported event ${EVENT_NAME}" >&2 + exit 1 + ;; + esac + fi case "${environment}" in dev|staging|prod) ;; *) diff --git a/terraform/live/README.md b/terraform/live/README.md index 0c78f8f..a1bbdf4 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -183,8 +183,11 @@ the same way as an empty allowlist. branch. Do not create the `prod` environment yet. Leave `PROD_APP_CD_ENABLED` -unset. Do not run Actions → Release with `environment=prod`; the first -prod dispatch would auto-create an unprotected environment. +unset. Until the `prod` environment exists with reviewers, do not run +Actions → Release with `environment=prod`, do not push a bare `vX.Y.Z` +tag, and do not `workflow_dispatch` deploy with `environment=prod`. Any +of those declares `environment: prod` and would auto-create an +unprotected environment. ## Pinned live identities From f42576e2db40b3a0115f2ee58e27300349f6090e Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Fri, 18 Sep 2026 12:38:15 -0400 Subject: [PATCH 4/8] chore(cd): retire leftover Terraform app CD path --- .github/workflows/deploy.yml | 1238 ----------------- QUALITY_GATES.md | 2 - scripts/check-terraform-release-plan.py | 328 ----- scripts/test-terraform-release-plan-check.py | 295 ---- .../terraform-release-plans/create.json | 16 - .../terraform-release-plans/delete.json | 16 - .../terraform-release-plans/dns-update.json | 28 - .../eb-setting-change.json | 34 - .../terraform-release-plans/empty.json | 3 - .../terraform-release-plans/iam-update.json | 18 - .../multiple-updates.json | 32 - .../nested-unknown-tags.json | 39 - .../terraform-release-plans/replace.json | 20 - .../unknown-only-description.json | 39 - .../terraform-release-plans/version-only.json | 48 - .../terraform-release-plans/wrong-label.json | 22 - terraform/live/README.md | 21 +- 17 files changed, 6 insertions(+), 2193 deletions(-) delete mode 100644 .github/workflows/deploy.yml delete mode 100644 scripts/check-terraform-release-plan.py delete mode 100644 scripts/test-terraform-release-plan-check.py delete mode 100644 scripts/testdata/terraform-release-plans/create.json delete mode 100644 scripts/testdata/terraform-release-plans/delete.json delete mode 100644 scripts/testdata/terraform-release-plans/dns-update.json delete mode 100644 scripts/testdata/terraform-release-plans/eb-setting-change.json delete mode 100644 scripts/testdata/terraform-release-plans/empty.json delete mode 100644 scripts/testdata/terraform-release-plans/iam-update.json delete mode 100644 scripts/testdata/terraform-release-plans/multiple-updates.json delete mode 100644 scripts/testdata/terraform-release-plans/nested-unknown-tags.json delete mode 100644 scripts/testdata/terraform-release-plans/replace.json delete mode 100644 scripts/testdata/terraform-release-plans/unknown-only-description.json delete mode 100644 scripts/testdata/terraform-release-plans/version-only.json delete mode 100644 scripts/testdata/terraform-release-plans/wrong-label.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index ea2d3ef..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,1238 +0,0 @@ -name: Validate and deploy - -on: - pull_request: - branches: [dev, staging, main] - push: - branches: [dev, staging] - workflow_dispatch: - -permissions: - contents: read - -jobs: - validate: - name: Validate deployable source bundle - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Set up .NET - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 - with: - dotnet-version: "8.0.x" - - - name: Repository quality gate - env: - BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/dev' }} - HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - run: bash scripts/governance-check.sh - - - name: Build Elastic Beanstalk source bundle - run: bash scripts/package-elastic-beanstalk.sh - - - name: Inspect source bundle contract - run: bash scripts/validate-elastic-beanstalk-bundle.sh - - deploy-dev: - name: Deploy shoc-backend-dev through Terraform - if: > - (github.event_name == 'push' && github.ref == 'refs/heads/dev' && - vars.TERRAFORM_APP_CD_ENABLED == 'true') || - (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/dev') - needs: validate - runs-on: ubuntu-latest - timeout-minutes: 180 - permissions: - contents: read - id-token: write - environment: - name: dev - concurrency: - group: deploy-dev - cancel-in-progress: false - env: - TF_CLOUD_ORGANIZATION: seahaven - TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }} - EB_APPLICATION_NAME: shoc-backend - EB_ENVIRONMENT_NAME: shoc-backend-dev - SMOKE_URL: https://api.dev.seahaven.com - EB_BUCKET: elasticbeanstalk-us-east-1-396287094661 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up .NET - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 - with: - dotnet-version: "8.0.x" - - - name: Build Elastic Beanstalk source bundle - run: bash scripts/package-elastic-beanstalk.sh - - - name: Validate exact release bundle - run: bash scripts/validate-elastic-beanstalk-bundle.sh - - - name: Configure AWS credentials (OIDC) - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} - aws-region: us-east-1 - audience: sts.amazonaws.com - - - name: Capture current environment version - run: | - set -euo pipefail - prev="$(aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].VersionLabel' \ - --output text)" - echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt - echo "Previous version label: $prev" - - - name: Assign immutable release identity - id: release - run: | - set -euo pipefail - version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - s3_key="shoc-backend/releases/dev/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip" - { - echo "version_label=${version_label}" - echo "s3_key=${s3_key}" - } >> "${GITHUB_OUTPUT}" - - - name: Upload immutable bundle - run: | - set -euo pipefail - aws s3 cp .artifacts/elastic-beanstalk/site.zip \ - "s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \ - --region us-east-1 - - - name: Create Elastic Beanstalk application version - run: | - set -euo pipefail - aws elasticbeanstalk create-application-version \ - --application-name "${EB_APPLICATION_NAME}" \ - --version-label "${{ steps.release.outputs.version_label }}" \ - --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \ - --source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \ - --process \ - --region us-east-1 - - status="UNPROCESSED" - for _ in $(seq 1 36); do - status="$(aws elasticbeanstalk describe-application-versions \ - --application-name "${EB_APPLICATION_NAME}" \ - --version-labels "${{ steps.release.outputs.version_label }}" \ - --region us-east-1 \ - --query 'ApplicationVersions[0].Status' \ - --output text)" - echo "application version status: $status" - if [ "$status" = "PROCESSED" ]; then - exit 0 - fi - if [ "$status" = "FAILED" ]; then - echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2 - exit 1 - fi - sleep 5 - done - echo "Application version did not become PROCESSED." >&2 - exit 1 - - name: Discard blocking VCS run before GitHub CD - run: | - set -euo pipefail - python3 << 'PY' - import json, os, urllib.error, urllib.request - - token = os.environ["TF_API_TOKEN"] - workspace = "shoc-backend-dev" - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - } - - def get(url): - req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req) as resp: - return json.load(resp) - - def post(url, payload): - data = json.dumps(payload).encode() - req = urllib.request.Request( - url, data=data, method="POST", headers=headers - ) - try: - with urllib.request.urlopen(req) as resp: - return resp.status - except urllib.error.HTTPError as exc: - if exc.code in (409, 404): - body = exc.read().decode("utf-8", "replace") - print(f"discard returned HTTP {exc.code}: {body}") - return exc.code - raise - - ws = get( - f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" - )["data"] - attrs = ws["attributes"] - if attrs.get("auto-apply") is True: - raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue") - if not attrs.get("speculative-enabled"): - raise SystemExit("speculative plans are off; refuse to continue") - if (attrs.get("vcs-repo") or {}).get("tags-regex"): - raise SystemExit("tag-based VCS triggering is set; refuse to continue") - expected_patterns = [ - "terraform/live/dev/**", - "terraform/live/modules/**", - ] - if attrs.get("trigger-patterns") != expected_patterns: - raise SystemExit( - "trigger-patterns must be " - f"{expected_patterns}; got {attrs.get('trigger-patterns')}" - ) - if not attrs.get("locked"): - print("workspace is unlocked") - raise SystemExit(0) - - current = ( - ws.get("relationships", {}) - .get("current-run", {}) - .get("data") - ) - if not current: - raise SystemExit("workspace is locked without a current run") - run_id = current["id"] - run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] - run_attrs = run["attributes"] - status = run_attrs.get("status") - plan_only = run_attrs.get("plan-only") - print(f"current run {run_id} status={status} plan-only={plan_only}") - if plan_only: - print("speculative run does not block GitHub CD") - raise SystemExit(0) - if status in {"applying", "apply_queued"}: - raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") - discardable = { - "pending", "planned", "cost_estimated", "policy_checked", "policy_override" - } - if status not in discardable: - raise SystemExit(f"{run_id} status {status} is not discardable") - code = post( - f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", - {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, - ) - print(f"discarded {run_id} http={code}") - PY - - - name: Create Terraform release run - id: release-run - uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - env: - TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"' - with: - workspace: shoc-backend-dev - message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions" - - - name: Read Terraform release plan counts - id: release-plan - uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - plan: ${{ steps.release-run.outputs.plan_id }} - - - name: Reject non-version-only resource counts - env: - PLAN_ADD: ${{ steps.release-plan.outputs.add }} - PLAN_CHANGE: ${{ steps.release-plan.outputs.change }} - PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }} - run: | - set -euo pipefail - if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then - echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2 - exit 1 - fi - - - name: Guard version-only Terraform plan - run: | - set -euo pipefail - python scripts/check-terraform-release-plan.py \ - --plan-id "${{ steps.release-run.outputs.plan_id }}" \ - --expected-version-label "${{ steps.release.outputs.version_label }}" - - - name: Discard release run when the guard fails - if: failure() && steps.release-run.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.release-run.outputs.run_id }} - comment: Rejected by the version-only plan guard from GitHub Actions - - - name: Apply Terraform release run - id: release-apply - continue-on-error: true - uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.release-run.outputs.run_id }} - comment: Apply version-only release from GitHub Actions ${{ github.sha }} - - - name: Treat already-applied release run as success - env: - APPLY_OUTCOME: ${{ steps.release-apply.outcome }} - RUN_ID: ${{ steps.release-run.outputs.run_id }} - run: | - set -euo pipefail - if [ "$APPLY_OUTCOME" = "success" ]; then - echo "Apply succeeded." - exit 0 - fi - python3 << 'PY' - import json, os, urllib.request - run_id = os.environ["RUN_ID"] - token = os.environ["TF_API_TOKEN"] - req = urllib.request.Request( - f"https://app.terraform.io/api/v2/runs/{run_id}", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - }, - ) - with urllib.request.urlopen(req) as resp: - status = json.load(resp)["data"]["attributes"]["status"] - print(f"HCP run {run_id} status={status}") - if status == "applied": - raise SystemExit(0) - raise SystemExit( - f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " - f"HCP status={status}" - ) - PY - - - name: Verify exact application version is active - run: | - set -euo pipefail - expected="${{ steps.release.outputs.version_label }}" - status="Unknown" - current="Unknown" - health="Unknown" - - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - - if [ "$status" = "Ready" ]; then - if [ "$current" != "$expected" ]; then - echo "Environment became Ready on version $current, not the expected $expected." >&2 - exit 1 - fi - if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then - echo "Expected application version is Ready and healthy." - exit 0 - fi - # The expected version IS active. Elastic Beanstalk reports Ready as - # soon as the rollout finishes, before enhanced health has converged, - # so deciding on the first Ready poll fails a good release on a health - # value that was always going to change. Keep polling; an environment - # that is genuinely unhealthy still fails when the window runs out. - echo "Expected version is active; waiting for health to leave $health." - fi - sleep 15 - done - - echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2 - exit 1 - - - name: Post-deploy smoke - run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}" - - - name: Verify webhook secret source is operational - run: | - set -euo pipefail - response_file="$(mktemp)" - trap 'rm -f "$response_file"' EXIT - status="$(curl --silent --show-error \ - --output "$response_file" \ - --write-out '%{http_code}' \ - --request POST \ - --header 'Content-Type: application/json' \ - --header "X-SH-Timestamp: $(date +%s)" \ - --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \ - --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \ - --data '{}' \ - "${SMOKE_URL}/api/webhooks/work-orders")" - if [ "$status" != "401" ]; then - echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2 - sed -n '1,20p' "$response_file" >&2 - exit 1 - fi - - - name: Restore previous application version on failure (schema is not reverted) - if: failure() - run: | - set -euo pipefail - prev_file=".artifacts/elastic-beanstalk/previous-version.txt" - if [ ! -f "$prev_file" ]; then - echo "No previous version captured; nothing to roll back." >&2 - exit 0 - fi - prev="$(cat "$prev_file")" - if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then - echo "No previous version recorded; nothing to roll back." >&2 - exit 0 - fi - - echo "Waiting for any in-flight environment update to settle..." - status="Unknown" - current="Unknown" - health="Unknown" - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - if [ "$status" = "Ready" ]; then - break - fi - sleep 15 - done - - if [ "$status" != "Ready" ]; then - echo "Environment did not settle before rollback." >&2 - exit 1 - fi - if [ "$current" = "$prev" ]; then - echo "Environment is already on previous version $prev." - exit 0 - fi - if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then - echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2 - exit 1 - fi - echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}" - id: rollback-prepare - - - name: Discard blocking VCS run before GitHub rollback - id: rollback-discard-vcs - if: failure() && steps.rollback-prepare.outputs.rollback_label != '' - run: | - set -euo pipefail - python3 << 'PY' - import json, os, urllib.error, urllib.request - - token = os.environ["TF_API_TOKEN"] - workspace = "shoc-backend-dev" - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - } - - def get(url): - req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req) as resp: - return json.load(resp) - - def post(url, payload): - data = json.dumps(payload).encode() - req = urllib.request.Request( - url, data=data, method="POST", headers=headers - ) - try: - with urllib.request.urlopen(req) as resp: - return resp.status - except urllib.error.HTTPError as exc: - if exc.code in (409, 404): - body = exc.read().decode("utf-8", "replace") - print(f"discard returned HTTP {exc.code}: {body}") - return exc.code - raise - - ws = get( - f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" - )["data"] - attrs = ws["attributes"] - if attrs.get("auto-apply") is True: - raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue") - if not attrs.get("speculative-enabled"): - raise SystemExit("speculative plans are off; refuse to continue") - if (attrs.get("vcs-repo") or {}).get("tags-regex"): - raise SystemExit("tag-based VCS triggering is set; refuse to continue") - expected_patterns = [ - "terraform/live/dev/**", - "terraform/live/modules/**", - ] - if attrs.get("trigger-patterns") != expected_patterns: - raise SystemExit( - "trigger-patterns must be " - f"{expected_patterns}; got {attrs.get('trigger-patterns')}" - ) - if not attrs.get("locked"): - print("workspace is unlocked") - raise SystemExit(0) - - current = ( - ws.get("relationships", {}) - .get("current-run", {}) - .get("data") - ) - if not current: - raise SystemExit("workspace is locked without a current run") - run_id = current["id"] - run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] - run_attrs = run["attributes"] - status = run_attrs.get("status") - plan_only = run_attrs.get("plan-only") - print(f"current run {run_id} status={status} plan-only={plan_only}") - if plan_only: - print("speculative run does not block GitHub CD") - raise SystemExit(0) - if status in {"applying", "apply_queued"}: - raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") - discardable = { - "pending", "planned", "cost_estimated", "policy_checked", "policy_override" - } - if status not in discardable: - raise SystemExit(f"{run_id} status {status} is not discardable") - code = post( - f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", - {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, - ) - print(f"discarded {run_id} http={code}") - PY - - - name: Create Terraform rollback run - id: rollback-run - if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - env: - TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"' - with: - workspace: shoc-backend-dev - message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions" - - name: Read Terraform rollback plan counts - id: rollback-plan - if: failure() && steps.rollback-run.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - plan: ${{ steps.rollback-run.outputs.plan_id }} - - - name: Reject non-version-only rollback counts - id: rollback-count-guard - if: failure() && steps.rollback-plan.outcome == 'success' - env: - PLAN_ADD: ${{ steps.rollback-plan.outputs.add }} - PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }} - PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }} - run: | - set -euo pipefail - if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then - echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2 - exit 1 - fi - - - name: Guard version-only Terraform rollback plan - id: rollback-json-guard - if: failure() && steps.rollback-count-guard.outcome == 'success' - run: | - set -euo pipefail - python scripts/check-terraform-release-plan.py \ - --plan-id "${{ steps.rollback-run.outputs.plan_id }}" \ - --expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}" - - - name: Discard rollback run when the guard fails - if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success' - uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.rollback-run.outputs.run_id }} - comment: Rejected by the version-only rollback plan guard from GitHub Actions - - - name: Apply Terraform rollback run - id: rollback-apply - if: failure() && steps.rollback-json-guard.outcome == 'success' - continue-on-error: true - uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.rollback-run.outputs.run_id }} - comment: Apply version-only rollback from GitHub Actions ${{ github.sha }} - - - name: Treat already-applied rollback run as success - id: rollback-apply-result - if: failure() && steps.rollback-apply.outcome != 'skipped' - env: - APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }} - RUN_ID: ${{ steps.rollback-run.outputs.run_id }} - run: | - set -euo pipefail - if [ "$APPLY_OUTCOME" = "success" ]; then - echo "Apply succeeded." - exit 0 - fi - python3 << 'PY' - import json, os, urllib.request - run_id = os.environ["RUN_ID"] - token = os.environ["TF_API_TOKEN"] - req = urllib.request.Request( - f"https://app.terraform.io/api/v2/runs/{run_id}", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - }, - ) - with urllib.request.urlopen(req) as resp: - status = json.load(resp)["data"]["attributes"]["status"] - print(f"HCP run {run_id} status={status}") - if status == "applied": - raise SystemExit(0) - raise SystemExit( - f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " - f"HCP status={status}" - ) - PY - - - name: Verify previous application version is active - if: failure() && steps.rollback-apply-result.outcome == 'success' - run: | - set -euo pipefail - prev="${{ steps.rollback-prepare.outputs.rollback_label }}" - echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy." - status="Unknown" - current="Unknown" - health="Unknown" - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - if [ "$status" = "Ready" ]; then - if [ "$current" != "$prev" ]; then - echo "Rollback reached Ready on version $current, not the previous $prev." >&2 - exit 1 - fi - if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then - echo "Application version restore complete; previous code is Ready and healthy." - exit 0 - fi - # Same convergence gap as the release check above: the previous version - # is back, health has not settled yet, and reporting a failed rollback - # here hides the fact that the restore itself worked. - echo "Previous version is active; waiting for health to leave $health." - fi - sleep 15 - done - echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2 - exit 1 - - deploy-staging: - name: Deploy shoc-backend-staging through Terraform - if: > - (github.event_name == 'push' && github.ref == 'refs/heads/staging') || - (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging') - needs: validate - runs-on: ubuntu-latest - timeout-minutes: 180 - permissions: - contents: read - id-token: write - environment: - name: staging - concurrency: - group: deploy-staging - cancel-in-progress: false - env: - TF_CLOUD_ORGANIZATION: seahaven - TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }} - EB_APPLICATION_NAME: shoc-backend - EB_ENVIRONMENT_NAME: shoc-backend-staging - SMOKE_URL: https://api.staging.seahaven.com - EB_BUCKET: elasticbeanstalk-us-east-1-396287094661 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up .NET - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 - with: - dotnet-version: "8.0.x" - - - name: Build Elastic Beanstalk source bundle - run: bash scripts/package-elastic-beanstalk.sh - - - name: Validate exact release bundle - run: bash scripts/validate-elastic-beanstalk-bundle.sh - - - name: Configure AWS credentials (OIDC) - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} - aws-region: us-east-1 - audience: sts.amazonaws.com - - - name: Capture current environment version - run: | - set -euo pipefail - prev="$(aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].VersionLabel' \ - --output text)" - echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt - echo "Previous version label: $prev" - - - name: Assign immutable release identity - id: release - run: | - set -euo pipefail - version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - s3_key="shoc-backend/releases/staging/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip" - { - echo "version_label=${version_label}" - echo "s3_key=${s3_key}" - } >> "${GITHUB_OUTPUT}" - - - name: Upload immutable bundle - run: | - set -euo pipefail - aws s3 cp .artifacts/elastic-beanstalk/site.zip \ - "s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \ - --region us-east-1 - - - name: Create Elastic Beanstalk application version - run: | - set -euo pipefail - aws elasticbeanstalk create-application-version \ - --application-name "${EB_APPLICATION_NAME}" \ - --version-label "${{ steps.release.outputs.version_label }}" \ - --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \ - --source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \ - --process \ - --region us-east-1 - - status="UNPROCESSED" - for _ in $(seq 1 36); do - status="$(aws elasticbeanstalk describe-application-versions \ - --application-name "${EB_APPLICATION_NAME}" \ - --version-labels "${{ steps.release.outputs.version_label }}" \ - --region us-east-1 \ - --query 'ApplicationVersions[0].Status' \ - --output text)" - echo "application version status: $status" - if [ "$status" = "PROCESSED" ]; then - exit 0 - fi - if [ "$status" = "FAILED" ]; then - echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2 - exit 1 - fi - sleep 5 - done - echo "Application version did not become PROCESSED." >&2 - exit 1 - - - name: Discard blocking VCS run before GitHub CD - run: | - set -euo pipefail - python3 << 'PY' - import json, os, urllib.error, urllib.request - - token = os.environ["TF_API_TOKEN"] - workspace = "shoc-backend-staging" - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - } - - def get(url): - req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req) as resp: - return json.load(resp) - - def post(url, payload): - data = json.dumps(payload).encode() - req = urllib.request.Request( - url, data=data, method="POST", headers=headers - ) - try: - with urllib.request.urlopen(req) as resp: - return resp.status - except urllib.error.HTTPError as exc: - if exc.code in (409, 404): - body = exc.read().decode("utf-8", "replace") - print(f"discard returned HTTP {exc.code}: {body}") - return exc.code - raise - - ws = get( - f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" - )["data"] - attrs = ws["attributes"] - if attrs.get("auto-apply") is True: - raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue") - if not attrs.get("speculative-enabled"): - raise SystemExit("speculative plans are off; refuse to continue") - if (attrs.get("vcs-repo") or {}).get("tags-regex"): - raise SystemExit("tag-based VCS triggering is set; refuse to continue") - expected_patterns = [ - "terraform/live/staging/**", - "terraform/live/modules/**", - ] - if attrs.get("trigger-patterns") != expected_patterns: - raise SystemExit( - "trigger-patterns must be " - f"{expected_patterns}; got {attrs.get('trigger-patterns')}" - ) - if not attrs.get("locked"): - print("workspace is unlocked") - raise SystemExit(0) - - current = ( - ws.get("relationships", {}) - .get("current-run", {}) - .get("data") - ) - if not current: - raise SystemExit("workspace is locked without a current run") - run_id = current["id"] - run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] - run_attrs = run["attributes"] - status = run_attrs.get("status") - plan_only = run_attrs.get("plan-only") - print(f"current run {run_id} status={status} plan-only={plan_only}") - if plan_only: - print("speculative run does not block GitHub CD") - raise SystemExit(0) - if status in {"applying", "apply_queued"}: - raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") - discardable = { - "pending", "planned", "cost_estimated", "policy_checked", "policy_override" - } - if status not in discardable: - raise SystemExit(f"{run_id} status {status} is not discardable") - code = post( - f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", - {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, - ) - print(f"discarded {run_id} http={code}") - PY - - - name: Create Terraform release run - id: release-run - uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - env: - TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"' - with: - workspace: shoc-backend-staging - message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions" - - - name: Read Terraform release plan counts - id: release-plan - uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - plan: ${{ steps.release-run.outputs.plan_id }} - - - name: Reject non-version-only resource counts - env: - PLAN_ADD: ${{ steps.release-plan.outputs.add }} - PLAN_CHANGE: ${{ steps.release-plan.outputs.change }} - PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }} - run: | - set -euo pipefail - if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then - echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2 - exit 1 - fi - - - name: Guard version-only Terraform plan - run: | - set -euo pipefail - python scripts/check-terraform-release-plan.py \ - --plan-id "${{ steps.release-run.outputs.plan_id }}" \ - --expected-version-label "${{ steps.release.outputs.version_label }}" - - - name: Discard release run when the guard fails - if: failure() && steps.release-run.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.release-run.outputs.run_id }} - comment: Rejected by the version-only plan guard from GitHub Actions - - - name: Apply Terraform release run - id: release-apply - continue-on-error: true - uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.release-run.outputs.run_id }} - comment: Apply version-only release from GitHub Actions ${{ github.sha }} - - - name: Treat already-applied release run as success - env: - APPLY_OUTCOME: ${{ steps.release-apply.outcome }} - RUN_ID: ${{ steps.release-run.outputs.run_id }} - run: | - set -euo pipefail - if [ "$APPLY_OUTCOME" = "success" ]; then - echo "Apply succeeded." - exit 0 - fi - python3 << 'PY' - import json, os, urllib.request - run_id = os.environ["RUN_ID"] - token = os.environ["TF_API_TOKEN"] - req = urllib.request.Request( - f"https://app.terraform.io/api/v2/runs/{run_id}", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - }, - ) - with urllib.request.urlopen(req) as resp: - status = json.load(resp)["data"]["attributes"]["status"] - print(f"HCP run {run_id} status={status}") - if status == "applied": - raise SystemExit(0) - raise SystemExit( - f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " - f"HCP status={status}" - ) - PY - - - name: Verify exact application version is active - run: | - set -euo pipefail - expected="${{ steps.release.outputs.version_label }}" - status="Unknown" - current="Unknown" - health="Unknown" - - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - - if [ "$status" = "Ready" ]; then - if [ "$current" != "$expected" ]; then - echo "Environment became Ready on version $current, not the expected $expected." >&2 - exit 1 - fi - if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then - echo "Expected application version is Ready and healthy." - exit 0 - fi - # The expected version IS active. Elastic Beanstalk reports Ready as - # soon as the rollout finishes, before enhanced health has converged, - # so deciding on the first Ready poll fails a good release on a health - # value that was always going to change. Keep polling; an environment - # that is genuinely unhealthy still fails when the window runs out. - echo "Expected version is active; waiting for health to leave $health." - fi - sleep 15 - done - - echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2 - exit 1 - - - name: Post-deploy smoke - run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}" - - - name: Verify webhook secret source is operational - run: | - set -euo pipefail - response_file="$(mktemp)" - trap 'rm -f "$response_file"' EXIT - status="$(curl --silent --show-error \ - --output "$response_file" \ - --write-out '%{http_code}' \ - --request POST \ - --header 'Content-Type: application/json' \ - --header "X-SH-Timestamp: $(date +%s)" \ - --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \ - --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \ - --data '{}' \ - "${SMOKE_URL}/api/webhooks/work-orders")" - if [ "$status" != "401" ]; then - echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2 - sed -n '1,20p' "$response_file" >&2 - exit 1 - fi - - - name: Restore previous application version on failure (schema is not reverted) - if: failure() - run: | - set -euo pipefail - prev_file=".artifacts/elastic-beanstalk/previous-version.txt" - if [ ! -f "$prev_file" ]; then - echo "No previous version captured; nothing to roll back." >&2 - exit 0 - fi - prev="$(cat "$prev_file")" - if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then - echo "No previous version recorded; nothing to roll back." >&2 - exit 0 - fi - - echo "Waiting for any in-flight environment update to settle..." - status="Unknown" - current="Unknown" - health="Unknown" - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - if [ "$status" = "Ready" ]; then - break - fi - sleep 15 - done - - if [ "$status" != "Ready" ]; then - echo "Environment did not settle before rollback." >&2 - exit 1 - fi - if [ "$current" = "$prev" ]; then - echo "Environment is already on previous version $prev." - exit 0 - fi - if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then - echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2 - exit 1 - fi - echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}" - id: rollback-prepare - - - name: Discard blocking VCS run before GitHub rollback - id: rollback-discard-vcs - if: failure() && steps.rollback-prepare.outputs.rollback_label != '' - run: | - set -euo pipefail - python3 << 'PY' - import json, os, urllib.error, urllib.request - - token = os.environ["TF_API_TOKEN"] - workspace = "shoc-backend-staging" - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - } - - def get(url): - req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req) as resp: - return json.load(resp) - - def post(url, payload): - data = json.dumps(payload).encode() - req = urllib.request.Request( - url, data=data, method="POST", headers=headers - ) - try: - with urllib.request.urlopen(req) as resp: - return resp.status - except urllib.error.HTTPError as exc: - if exc.code in (409, 404): - body = exc.read().decode("utf-8", "replace") - print(f"discard returned HTTP {exc.code}: {body}") - return exc.code - raise - - ws = get( - f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" - )["data"] - attrs = ws["attributes"] - if attrs.get("auto-apply") is True: - raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue") - if not attrs.get("speculative-enabled"): - raise SystemExit("speculative plans are off; refuse to continue") - if (attrs.get("vcs-repo") or {}).get("tags-regex"): - raise SystemExit("tag-based VCS triggering is set; refuse to continue") - expected_patterns = [ - "terraform/live/staging/**", - "terraform/live/modules/**", - ] - if attrs.get("trigger-patterns") != expected_patterns: - raise SystemExit( - "trigger-patterns must be " - f"{expected_patterns}; got {attrs.get('trigger-patterns')}" - ) - if not attrs.get("locked"): - print("workspace is unlocked") - raise SystemExit(0) - - current = ( - ws.get("relationships", {}) - .get("current-run", {}) - .get("data") - ) - if not current: - raise SystemExit("workspace is locked without a current run") - run_id = current["id"] - run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] - run_attrs = run["attributes"] - status = run_attrs.get("status") - plan_only = run_attrs.get("plan-only") - print(f"current run {run_id} status={status} plan-only={plan_only}") - if plan_only: - print("speculative run does not block GitHub CD") - raise SystemExit(0) - if status in {"applying", "apply_queued"}: - raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") - discardable = { - "pending", "planned", "cost_estimated", "policy_checked", "policy_override" - } - if status not in discardable: - raise SystemExit(f"{run_id} status {status} is not discardable") - code = post( - f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", - {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, - ) - print(f"discarded {run_id} http={code}") - PY - - - name: Create Terraform rollback run - id: rollback-run - if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - env: - TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"' - with: - workspace: shoc-backend-staging - message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions" - - name: Read Terraform rollback plan counts - id: rollback-plan - if: failure() && steps.rollback-run.outcome == 'success' - uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - plan: ${{ steps.rollback-run.outputs.plan_id }} - - - name: Reject non-version-only rollback counts - id: rollback-count-guard - if: failure() && steps.rollback-plan.outcome == 'success' - env: - PLAN_ADD: ${{ steps.rollback-plan.outputs.add }} - PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }} - PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }} - run: | - set -euo pipefail - if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then - echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2 - exit 1 - fi - - - name: Guard version-only Terraform rollback plan - id: rollback-json-guard - if: failure() && steps.rollback-count-guard.outcome == 'success' - run: | - set -euo pipefail - python scripts/check-terraform-release-plan.py \ - --plan-id "${{ steps.rollback-run.outputs.plan_id }}" \ - --expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}" - - - name: Discard rollback run when the guard fails - if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success' - uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.rollback-run.outputs.run_id }} - comment: Rejected by the version-only rollback plan guard from GitHub Actions - - - name: Apply Terraform rollback run - id: rollback-apply - if: failure() && steps.rollback-json-guard.outcome == 'success' - continue-on-error: true - uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 - with: - run: ${{ steps.rollback-run.outputs.run_id }} - comment: Apply version-only rollback from GitHub Actions ${{ github.sha }} - - - name: Treat already-applied rollback run as success - id: rollback-apply-result - if: failure() && steps.rollback-apply.outcome != 'skipped' - env: - APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }} - RUN_ID: ${{ steps.rollback-run.outputs.run_id }} - run: | - set -euo pipefail - if [ "$APPLY_OUTCOME" = "success" ]; then - echo "Apply succeeded." - exit 0 - fi - python3 << 'PY' - import json, os, urllib.request - run_id = os.environ["RUN_ID"] - token = os.environ["TF_API_TOKEN"] - req = urllib.request.Request( - f"https://app.terraform.io/api/v2/runs/{run_id}", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - }, - ) - with urllib.request.urlopen(req) as resp: - status = json.load(resp)["data"]["attributes"]["status"] - print(f"HCP run {run_id} status={status}") - if status == "applied": - raise SystemExit(0) - raise SystemExit( - f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " - f"HCP status={status}" - ) - PY - - - name: Verify previous application version is active - if: failure() && steps.rollback-apply-result.outcome == 'success' - run: | - set -euo pipefail - prev="${{ steps.rollback-prepare.outputs.rollback_label }}" - echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy." - status="Unknown" - current="Unknown" - health="Unknown" - for _ in $(seq 1 80); do - read -r status current health < <( - aws elasticbeanstalk describe-environments \ - --environment-names "${EB_ENVIRONMENT_NAME}" \ - --region us-east-1 \ - --query 'Environments[0].[Status,VersionLabel,Health]' \ - --output text - ) - echo "environment status: $status; version: $current; health: $health" - if [ "$status" = "Ready" ]; then - if [ "$current" != "$prev" ]; then - echo "Rollback reached Ready on version $current, not the previous $prev." >&2 - exit 1 - fi - if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then - echo "Application version restore complete; previous code is Ready and healthy." - exit 0 - fi - # Same convergence gap as the release check above: the previous version - # is back, health has not settled yet, and reporting a failed rollback - # here hides the fact that the restore itself worked. - echo "Previous version is active; waiting for health to leave $health." - fi - sleep 15 - done - echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2 - exit 1 diff --git a/QUALITY_GATES.md b/QUALITY_GATES.md index 9d6609e..c1a83a6 100644 --- a/QUALITY_GATES.md +++ b/QUALITY_GATES.md @@ -73,8 +73,6 @@ remains in the matrix. HCP plan/apply roles stay in org-baseline; this repository never manages `hcptf-*` roles. The former G12 version-only HCP apply guard is not part of the repository gate. -`scripts/check-terraform-release-plan.py` remains only while -`.github/workflows/deploy.yml` is still present. G13 fails when the same diff contains both `terraform/` and deployable application files. Workflow, documentation, and gate-script changes may share diff --git a/scripts/check-terraform-release-plan.py b/scripts/check-terraform-release-plan.py deleted file mode 100644 index e0f8a88..0000000 --- a/scripts/check-terraform-release-plan.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python3 -"""Reject HCP Terraform plans that are not a version-only Elastic Beanstalk update. - -This script may read a local plan JSON file or download plan JSON from the -documented HashiCorp endpoint: - - GET https://app.terraform.io/api/v2/plans/:id/json-output - -The download follows exactly one redirect, and only to archivist.terraform.io. -It does not create, apply, discard, or poll runs. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import ssl -import sys -import urllib.error -import urllib.request -from pathlib import Path -from typing import Any, Callable -from urllib.parse import urlparse - - -RELEASE_ADDRESS = "module.environment.aws_elastic_beanstalk_environment.this" -API_HOST = "app.terraform.io" -ARCHIVE_HOST = "archivist.terraform.io" -PLAN_ID_RE = re.compile(r"^plan-[A-Za-z0-9]+$") -VERSION_LABEL_RE = re.compile(r"^[0-9a-f]{40}-[0-9]+-[0-9]+$") -IGNORED_ACTIONS = {"no-op", "read"} -UNSAFE_ACTIONS = {"create", "delete"} -# Wholly unknown computed attributes may be ignored. Nested unknowns on any -# other attribute are treated as changes so the version-only guard fails closed. -COMPUTED_UNKNOWN_ATTRIBUTES = frozenset({"instances", "load_balancers"}) -REDIRECT_STATUSES = {301, 302, 303, 307, 308} - -UrlOpen = Callable[..., Any] - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Return the redirect response instead of following it.""" - - def http_error_301(self, req, fp, code, msg, headers): - return self._capture(req, fp, code, headers) - - http_error_302 = http_error_303 = http_error_307 = http_error_308 = http_error_301 - - @staticmethod - def _capture(req, fp, code, headers): - response = urllib.response.addinfourl(fp, headers, req.full_url, code=code) - response.msg = "Redirect" - return response - - -def _urlopen_without_redirects( - *handlers: urllib.request.BaseHandler, -) -> UrlOpen: - context = ssl.create_default_context() - opener = urllib.request.build_opener( - urllib.request.HTTPSHandler(context=context), - _NoRedirectHandler, - *handlers, - ) - return opener.open - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - source = parser.add_mutually_exclusive_group(required=True) - source.add_argument( - "plan_json", - type=Path, - nargs="?", - help="Local Terraform plan JSON. Mutually exclusive with --plan-id.", - ) - source.add_argument( - "--plan-id", - help="HCP Terraform plan ID. Downloads JSON from app.terraform.io.", - ) - parser.add_argument( - "--expected-version-label", - required=True, - help="Immutable application version the plan must apply.", - ) - parser.add_argument( - "--evidence-out", - type=Path, - help="Write machine-readable proof after every assertion passes.", - ) - return parser.parse_args() - - -def download_plan_json( - plan_id: str, - token: str, - *, - urlopen: UrlOpen | None = None, - handlers: tuple[urllib.request.BaseHandler, ...] = (), -) -> dict[str, Any]: - if not PLAN_ID_RE.fullmatch(plan_id): - raise ValueError(f"plan id {plan_id!r} is not a valid HCP plan id") - if not token: - raise ValueError("TF_API_TOKEN is required to download plan JSON") - - opener = urlopen or _urlopen_without_redirects(*handlers) - api_url = f"https://{API_HOST}/api/v2/plans/{plan_id}/json-output" - request = urllib.request.Request( - api_url, - method="GET", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/vnd.api+json", - "Accept": "application/json", - }, - ) - first = _open_pinned(opener, request, allowed_host=API_HOST) - try: - if first.status == 204: - raise ValueError( - "plan JSON is not ready; refusing to poll the plans endpoint" - ) - if first.status not in REDIRECT_STATUSES: - raise ValueError( - f"expected a redirect from {API_HOST}, got HTTP {first.status}" - ) - location = first.headers.get("Location") - if not location: - raise ValueError(f"{API_HOST} redirect is missing a Location header") - archive = urlparse(location) - if archive.scheme != "https" or archive.hostname != ARCHIVE_HOST: - raise ValueError( - "refusing redirect that is not https://" - f"{ARCHIVE_HOST}/" - ) - archive_request = urllib.request.Request(location, method="GET") - second = _open_pinned(opener, archive_request, allowed_host=ARCHIVE_HOST) - try: - if second.status in REDIRECT_STATUSES: - raise ValueError( - f"refusing a second redirect from {ARCHIVE_HOST}" - ) - if second.status != 200: - raise ValueError( - f"plan JSON download from {ARCHIVE_HOST} returned " - f"HTTP {second.status}" - ) - payload = second.read() - finally: - second.close() - finally: - first.close() - - plan = json.loads(payload.decode("utf-8")) - if not isinstance(plan, dict): - raise ValueError("plan JSON must be an object") - return plan - - -def _open_pinned(urlopen: UrlOpen, request: urllib.request.Request, *, allowed_host: str): - parsed = urlparse(request.full_url) - if parsed.scheme != "https" or parsed.hostname != allowed_host: - raise ValueError( - f"refusing to contact {parsed.scheme}://{parsed.hostname} " - f"(pinned host is {allowed_host})" - ) - context = ssl.create_default_context() - try: - return urlopen(request, context=context, timeout=30) - except TypeError: - return urlopen(request, timeout=30) - - -def _is_nested_unknown(value: Any) -> bool: - if isinstance(value, dict): - return any(item is True or _is_nested_unknown(item) for item in value.values()) - if isinstance(value, list): - return any(item is True or _is_nested_unknown(item) for item in value) - return False - - -def changed_attributes(change: dict[str, Any]) -> set[str]: - before = change.get("before") or {} - after = change.get("after") or {} - unknown = change.get("after_unknown") or {} - keys = set(before) | set(after) | set(unknown) - changed: set[str] = set() - for key in keys: - unknown_value = unknown.get(key) - if unknown_value is True: - if key in COMPUTED_UNKNOWN_ATTRIBUTES: - continue - changed.add(key) - continue - if _is_nested_unknown(unknown_value): - changed.add(key) - continue - if before.get(key) != after.get(key): - changed.add(key) - return changed - - -def validate_plan(plan: dict[str, Any], expected_label: str) -> list[str]: - violations: list[str] = [] - if not VERSION_LABEL_RE.fullmatch(expected_label): - violations.append( - "expected version label must be --" - ) - return violations - - updates: list[dict[str, Any]] = [] - for resource in plan.get("resource_changes", []): - if resource.get("mode", "managed") != "managed": - continue - address = resource.get("address", "") - change = resource.get("change") or {} - actions = list(change.get("actions") or []) - action_set = set(actions) - if action_set <= IGNORED_ACTIONS: - continue - - if change.get("importing"): - violations.append(f"{address}: import actions are not allowed") - - unsafe = sorted(action_set & UNSAFE_ACTIONS) - if unsafe: - violations.append(f"{address}: unsafe actions {unsafe}") - if "replace" in action_set or actions in ( - ["delete", "create"], - ["create", "delete"], - ): - violations.append(f"{address}: replacement is not allowed") - - if "update" in action_set: - updates.append(resource) - if action_set != {"update"}: - violations.append( - f"{address}: update must be the only action, got {actions}" - ) - - if address != RELEASE_ADDRESS and action_set - IGNORED_ACTIONS: - violations.append( - f"{address}: managed address is outside the version-only release" - ) - - if len(updates) != 1: - violations.append( - f"expected exactly one managed update, found {len(updates)}" - ) - return violations - - resource = updates[0] - address = resource.get("address", "") - if address != RELEASE_ADDRESS: - violations.append( - f"{address}: expected update address {RELEASE_ADDRESS}" - ) - return violations - - change = resource.get("change") or {} - changed = changed_attributes(change) - if changed != {"version_label"}: - violations.append( - f"{address}: expected only version_label to change, found " - f"{sorted(changed) if changed else 'no attribute changes'}" - ) - - after = change.get("after") or {} - actual = after.get("version_label") - if actual != expected_label: - violations.append( - f"{address}: after version_label {actual!r} does not match " - f"{expected_label!r}" - ) - - unknown = change.get("after_unknown") or {} - if unknown.get("version_label") is True: - violations.append(f"{address}: version_label after value is unknown") - - return violations - - -def main() -> int: - args = parse_args() - if args.plan_id: - try: - plan = download_plan_json(args.plan_id, os.environ.get("TF_API_TOKEN", "")) - except (OSError, ValueError, json.JSONDecodeError, urllib.error.URLError) as exc: - print(f"FAIL: could not download plan JSON: {exc}", file=sys.stderr) - return 1 - else: - if args.plan_json is None: - print("FAIL: plan JSON path or --plan-id is required", file=sys.stderr) - return 1 - plan = json.loads(args.plan_json.read_text(encoding="utf-8")) - - violations = validate_plan(plan, args.expected_version_label) - if violations: - print("FAIL: Terraform plan is not a version-only release", file=sys.stderr) - for violation in violations: - print(f" - {violation}", file=sys.stderr) - return 1 - - if args.evidence_out: - evidence = { - "address": RELEASE_ADDRESS, - "expected_version_label": args.expected_version_label, - "managed_updates": 1, - "changed_attributes": ["version_label"], - "creates": 0, - "deletes": 0, - "replacements": 0, - } - args.evidence_out.write_text( - json.dumps(evidence, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - print( - "PASS: version-only plan updates " - f"{RELEASE_ADDRESS} version_label to {args.expected_version_label}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-terraform-release-plan-check.py b/scripts/test-terraform-release-plan-check.py deleted file mode 100644 index 5967d6f..0000000 --- a/scripts/test-terraform-release-plan-check.py +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic tests for check-terraform-release-plan.py.""" - -from __future__ import annotations - -import importlib.util -import io -import subprocess -import sys -import urllib.request -from email.message import EmailMessage -from pathlib import Path -from urllib.request import Request - -SCRIPT = Path(__file__).with_name("check-terraform-release-plan.py") -FIXTURES = Path(__file__).with_name("testdata") / "terraform-release-plans" -EXPECTED_LABEL = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1" -PLAN_ID = "plan-8F5JFydVYAmtTjET" - - -def run_case( - fixture_name: str, - *, - expected_label: str = EXPECTED_LABEL, -) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ - sys.executable, - str(SCRIPT), - str(FIXTURES / fixture_name), - "--expected-version-label", - expected_label, - ], - check=False, - capture_output=True, - text=True, - ) - - -class FakeResponse: - def __init__( - self, - *, - url: str, - status: int, - headers: dict[str, str] | None = None, - body: bytes = b"", - ) -> None: - self.url = url - self.status = status - self.headers = headers or {} - self._body = body - - def read(self) -> bytes: - return self._body - - def close(self) -> None: - return None - - -def load_check_module(): - spec = importlib.util.spec_from_file_location("check_terraform_release_plan", SCRIPT) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -def test_download_pinning() -> list[str]: - module = load_check_module() - fixture = (FIXTURES / "version-only.json").read_bytes() - archive_url = "https://archivist.terraform.io/v1/object/example" - calls: list[str] = [] - - def fake_urlopen(request: Request, **_kwargs): - url = request.full_url - calls.append(url) - host = request.host if hasattr(request, "host") else "" - if url.startswith("https://app.terraform.io/api/v2/plans/"): - if request.get_header("Authorization") != "Bearer test-token": - raise AssertionError("API request is missing the bearer token") - if "/runs" in url or "/apply" in url or "/discard" in url: - raise AssertionError(f"download contacted a run-control path: {url}") - return FakeResponse( - url=url, - status=307, - headers={"Location": archive_url}, - ) - if url == archive_url: - if request.get_header("Authorization"): - raise AssertionError("archivist request must not send TF_API_TOKEN") - return FakeResponse(url=url, status=200, body=fixture) - raise AssertionError(f"unexpected URL {url} host={host}") - - plan = module.download_plan_json(PLAN_ID, "test-token", urlopen=fake_urlopen) - failures: list[str] = [] - if plan["resource_changes"][1]["address"] != ( - "module.environment.aws_elastic_beanstalk_environment.this" - ): - failures.append("download did not return the version-only fixture") - if calls != [ - f"https://app.terraform.io/api/v2/plans/{PLAN_ID}/json-output", - archive_url, - ]: - failures.append(f"download URLs were {calls}") - - try: - module.download_plan_json("run-not-a-plan", "test-token", urlopen=fake_urlopen) - failures.append("invalid plan id was accepted") - except ValueError: - pass - - def redirect_elsewhere(request: Request, **_kwargs): - return FakeResponse( - url=request.full_url, - status=307, - headers={"Location": "https://evil.example/plan.json"}, - ) - - try: - module.download_plan_json(PLAN_ID, "test-token", urlopen=redirect_elsewhere) - failures.append("redirect to a non-archivist host was accepted") - except ValueError: - pass - - def double_redirect(request: Request, **_kwargs): - if request.full_url.startswith("https://app.terraform.io/"): - return FakeResponse( - url=request.full_url, - status=307, - headers={"Location": archive_url}, - ) - return FakeResponse( - url=request.full_url, - status=307, - headers={"Location": "https://archivist.terraform.io/v1/object/other"}, - ) - - try: - module.download_plan_json(PLAN_ID, "test-token", urlopen=double_redirect) - failures.append("second archivist redirect was accepted") - except ValueError: - pass - - def not_ready(request: Request, **_kwargs): - return FakeResponse(url=request.full_url, status=204) - - try: - module.download_plan_json(PLAN_ID, "test-token", urlopen=not_ready) - failures.append("HTTP 204 was polled or accepted") - except ValueError as exc: - if "poll" not in str(exc): - failures.append(f"HTTP 204 error was {exc}") - - source = SCRIPT.read_text(encoding="utf-8") - for banned in ("/apply", "/discard", "/runs"): - if banned in source: - failures.append(f"download client contains run-control path {banned}") - - return failures - - -def _scripted_https_handler(fixture: bytes, archive_url: str): - calls: list[str] = [] - api_prefix = "https://app.terraform.io/api/v2/plans/" - - class ScriptedHTTPSHandler(urllib.request.BaseHandler): - handler_order = 100 - - def https_open(self, req: Request): - url = req.full_url - calls.append(url) - headers = EmailMessage() - if url.startswith(api_prefix): - headers["Location"] = archive_url - body = b"" - status = 307 - msg = "Temporary Redirect" - elif url == archive_url: - body = fixture - status = 200 - msg = "OK" - else: - raise AssertionError(f"unexpected URL {url}") - response = urllib.response.addinfourl( - io.BytesIO(body), - headers, - url, - code=status, - ) - response.msg = msg - return response - - return ScriptedHTTPSHandler(), calls - - -def test_download_standard_opener_redirect() -> list[str]: - """urllib follows the HCP 307; the guard must still inspect that first hop.""" - module = load_check_module() - fixture = (FIXTURES / "version-only.json").read_bytes() - archive_url = "https://archivist.terraform.io/v1/object/example" - api_url = f"https://app.terraform.io/api/v2/plans/{PLAN_ID}/json-output" - failures: list[str] = [] - - following_handler, following_calls = _scripted_https_handler(fixture, archive_url) - followed = urllib.request.build_opener(following_handler).open(api_url) - try: - if followed.status != 200: - failures.append( - f"standard opener first status was {followed.status}, not 200" - ) - if following_calls != [api_url, archive_url]: - failures.append(f"standard opener URLs were {following_calls}") - finally: - followed.close() - - guard_handler, guard_calls = _scripted_https_handler(fixture, archive_url) - try: - plan = module.download_plan_json( - PLAN_ID, - "test-token", - handlers=(guard_handler,), - ) - except ValueError as exc: - failures.append(f"no-redirect download failed: {exc}") - return failures - - if plan["resource_changes"][1]["address"] != ( - "module.environment.aws_elastic_beanstalk_environment.this" - ): - failures.append("no-redirect download did not return the version-only fixture") - if guard_calls != [api_url, archive_url]: - failures.append(f"no-redirect download URLs were {guard_calls}") - - following_urlopen_handler, _ = _scripted_https_handler(fixture, archive_url) - following_urlopen = urllib.request.build_opener(following_urlopen_handler).open - try: - module.download_plan_json( - PLAN_ID, - "test-token", - urlopen=following_urlopen, - ) - failures.append("redirect-following urlopen was accepted as the first hop") - except ValueError as exc: - if "expected a redirect" not in str(exc): - failures.append(f"following urlopen error was {exc}") - - return failures - - -def main() -> int: - cases = [ - ("version-only", run_case("version-only.json"), 0), - ("wrong-label", run_case("wrong-label.json"), 1), - ("eb-setting-change", run_case("eb-setting-change.json"), 1), - ("nested-unknown-tags", run_case("nested-unknown-tags.json"), 1), - ("unknown-only-description", run_case("unknown-only-description.json"), 1), - ("iam-update", run_case("iam-update.json"), 1), - ("dns-update", run_case("dns-update.json"), 1), - ("create", run_case("create.json"), 1), - ("delete", run_case("delete.json"), 1), - ("replace", run_case("replace.json"), 1), - ("multiple-updates", run_case("multiple-updates.json"), 1), - ("empty", run_case("empty.json"), 1), - ] - failures = [ - (name, result, expected) - for name, result, expected in cases - if result.returncode != expected - ] - download_failures = test_download_pinning() - redirect_failures = test_download_standard_opener_redirect() - download_failures.extend(redirect_failures) - if failures or download_failures: - if failures: - print( - "FAIL: release plan-check cases failed: " - + ", ".join(name for name, _, _ in failures), - file=sys.stderr, - ) - for name, result, expected in failures: - print( - f"{name}: expected {expected}, got {result.returncode}\n" - f"{result.stdout}{result.stderr}", - file=sys.stderr, - ) - for item in download_failures: - print(f"FAIL: {item}", file=sys.stderr) - return 1 - print("PASS: Terraform release plan safety checks") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/testdata/terraform-release-plans/create.json b/scripts/testdata/terraform-release-plans/create.json deleted file mode 100644 index 8ea3979..0000000 --- a/scripts/testdata/terraform-release-plans/create.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["create"], - "before": null, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1" - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/delete.json b/scripts/testdata/terraform-release-plans/delete.json deleted file mode 100644 index 958c2f6..0000000 --- a/scripts/testdata/terraform-release-plans/delete.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["delete"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1" - }, - "after": null - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/dns-update.json b/scripts/testdata/terraform-release-plans/dns-update.json deleted file mode 100644 index 5b2f27c..0000000 --- a/scripts/testdata/terraform-release-plans/dns-update.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_route53_record.api_alias[0]", - "mode": "managed", - "type": "aws_route53_record", - "change": { - "actions": ["update"], - "before": { - "alias": [ - { - "name": "awseb--awseb-cmpb3ypfib53-1654918745.us-east-1.elb.amazonaws.com", - "zone_id": "Z35SXDOTRQ7X7K" - } - ] - }, - "after": { - "alias": [ - { - "name": "shoc-backend-dev.us-east-1.elasticbeanstalk.com", - "zone_id": "Z117KPS5GTRQ2G" - } - ] - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/eb-setting-change.json b/scripts/testdata/terraform-release-plans/eb-setting-change.json deleted file mode 100644 index a0a2ecf..0000000 --- a/scripts/testdata/terraform-release-plans/eb-setting-change.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:application:environment", - "name": "ASPNETCORE_ENVIRONMENT", - "value": "Production" - } - ], - "tags": { "env": "dev" } - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:application:environment", - "name": "ASPNETCORE_ENVIRONMENT", - "value": "Development" - } - ], - "tags": { "env": "dev" } - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/empty.json b/scripts/testdata/terraform-release-plans/empty.json deleted file mode 100644 index 360110a..0000000 --- a/scripts/testdata/terraform-release-plans/empty.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "resource_changes": [] -} diff --git a/scripts/testdata/terraform-release-plans/iam-update.json b/scripts/testdata/terraform-release-plans/iam-update.json deleted file mode 100644 index aee8aec..0000000 --- a/scripts/testdata/terraform-release-plans/iam-update.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_iam_role.github_deploy", - "mode": "managed", - "type": "aws_iam_role", - "change": { - "actions": ["update"], - "before": { - "permissions_boundary": "arn:aws:iam::396287094661:policy/shoc-backend-dev-deploy-boundary" - }, - "after": { - "permissions_boundary": null - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/multiple-updates.json b/scripts/testdata/terraform-release-plans/multiple-updates.json deleted file mode 100644 index a4c4e0c..0000000 --- a/scripts/testdata/terraform-release-plans/multiple-updates.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [], - "tags": { "env": "dev" } - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "setting": [], - "tags": { "env": "dev" } - } - } - }, - { - "address": "module.environment.aws_iam_role.github_deploy", - "mode": "managed", - "type": "aws_iam_role", - "change": { - "actions": ["update"], - "before": { "description": "old" }, - "after": { "description": "new" } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/nested-unknown-tags.json b/scripts/testdata/terraform-release-plans/nested-unknown-tags.json deleted file mode 100644 index a9f2f43..0000000 --- a/scripts/testdata/terraform-release-plans/nested-unknown-tags.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "dev", "project": "shoc" } - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "prod", "project": "shoc" } - }, - "after_unknown": { - "instances": true, - "load_balancers": true, - "tags": { "env": true } - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/replace.json b/scripts/testdata/terraform-release-plans/replace.json deleted file mode 100644 index 73b8030..0000000 --- a/scripts/testdata/terraform-release-plans/replace.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["delete", "create"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "name": "shoc-backend-dev" - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "name": "shoc-backend-dev" - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/unknown-only-description.json b/scripts/testdata/terraform-release-plans/unknown-only-description.json deleted file mode 100644 index e1dd3bc..0000000 --- a/scripts/testdata/terraform-release-plans/unknown-only-description.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "dev", "project": "shoc" } - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "dev", "project": "shoc" } - }, - "after_unknown": { - "instances": true, - "load_balancers": true, - "description": true - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/version-only.json b/scripts/testdata/terraform-release-plans/version-only.json deleted file mode 100644 index 143a924..0000000 --- a/scripts/testdata/terraform-release-plans/version-only.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_iam_role.runtime", - "mode": "managed", - "type": "aws_iam_role", - "change": { - "actions": ["no-op"], - "before": { "name": "shoc-backend-dev" }, - "after": { "name": "shoc-backend-dev" } - } - }, - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "dev", "project": "shoc" } - }, - "after": { - "version_label": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2-1", - "setting": [ - { - "namespace": "aws:elasticbeanstalk:environment", - "name": "EnvironmentType", - "value": "LoadBalanced" - } - ], - "tags": { "env": "dev", "project": "shoc" } - }, - "after_unknown": { - "instances": true, - "load_balancers": true - } - } - } - ] -} diff --git a/scripts/testdata/terraform-release-plans/wrong-label.json b/scripts/testdata/terraform-release-plans/wrong-label.json deleted file mode 100644 index bd1c671..0000000 --- a/scripts/testdata/terraform-release-plans/wrong-label.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "resource_changes": [ - { - "address": "module.environment.aws_elastic_beanstalk_environment.this", - "mode": "managed", - "type": "aws_elastic_beanstalk_environment", - "change": { - "actions": ["update"], - "before": { - "version_label": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1-1", - "setting": [], - "tags": { "env": "dev" } - }, - "after": { - "version_label": "cccccccccccccccccccccccccccccccccccccccc-9-9", - "setting": [], - "tags": { "env": "dev" } - } - } - } - ] -} diff --git a/terraform/live/README.md b/terraform/live/README.md index a1bbdf4..e7630cd 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -143,7 +143,7 @@ workflow waits for CI, tags `vX.Y.Z-staging` from main HEAD with `GITHUB_TOKEN`, then calls deploy. Do not cut prod yet; leave `PROD_APP_CD_ENABLED` unset and do not create the `prod` GitHub Environment. Staging remains `adoption_complete=false` with a pinned API CNAME until its -import apply is proven. +import apply is proven after the first `vX.Y.Z-staging` GitHub-owned zip. HCP workspaces stay VCS-driven with auto-apply on after cutover. Speculative plans on every PR are the infra gate. Do not point `TFC_AWS_*` at @@ -153,10 +153,6 @@ Terraform changes stay in separate PRs so a merge cannot race an HCP apply against an app deploy. Terraform-only merges skip `deploy.yaml`. App-only tags skip HCP when trigger patterns do not match. -Until cutover, `.github/workflows/deploy.yml` still uses `TF_API_TOKEN` and -`TERRAFORM_APP_CD_ENABLED`. Keep those secrets and the version-only plan guard -on that leftover path only. - ### Credentials Store `DEPLOY_ROLE_ARN` as a GitHub Environment **variable** (`dev`, @@ -164,23 +160,18 @@ Store `DEPLOY_ROLE_ARN` as a GitHub Environment **variable** (`dev`, `repo:Sea-Haven-Industries/shoc-backend:environment:` plus `job_workflow_ref` for `.github/workflows/deploy.yaml` at `refs/heads/main` and `refs/tags/v*`. Adding another deploy workflow is a cross-family IAM -change. After cutover, drop `TF_API_TOKEN` from GitHub Environments. The new -CD path does not use it. +change. The new CD path does not use `TF_API_TOKEN`. GitHub Environment deployment branch and tag policies are repository -settings, not this diff. Update them before the first merge to `main` and -the first staging cut. The policy matches `GITHUB_REF` of the workflow run. +settings, not this diff. The policy matches `GITHUB_REF` of the workflow run. Branch patterns never match tag refs; adding `v*` as a branch pattern fails the same way as an empty allowlist. -1. `dev` — allow branch `main`. Keep `dev` allowed while leftover - `.github/workflows/deploy.yml` still deploys from that branch. -2. `staging` — add a **tag-type** policy matching `v*.*.*-staging` for +1. `dev` — allow branch `main`. +2. `staging` — **tag-type** policy matching `v*.*.*-staging` for `deploy-tag.yaml`. Allow branch `main` because Actions → Release is `workflow_dispatch` on `main` and then calls `deploy.yaml` - (`GITHUB_TOKEN` tag pushes do not start `deploy-tag.yaml`). Keep - `staging` allowed while leftover `deploy.yml` still deploys from that - branch. + (`GITHUB_TOKEN` tag pushes do not start `deploy-tag.yaml`). Do not create the `prod` environment yet. Leave `PROD_APP_CD_ENABLED` unset. Until the `prod` environment exists with reviewers, do not run From b696a414a5152560cdaa84bb7927455fa82c4d44 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Fri, 18 Sep 2026 12:42:42 -0400 Subject: [PATCH 5/8] fix(cd): grant caller id-token write for reusable deploy workflows --- .github/workflows/deploy-tag.yaml | 2 ++ .github/workflows/release.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/deploy-tag.yaml b/.github/workflows/deploy-tag.yaml index 581fe97..00f9389 100644 --- a/.github/workflows/deploy-tag.yaml +++ b/.github/workflows/deploy-tag.yaml @@ -10,6 +10,8 @@ on: permissions: contents: read + checks: read + id-token: write jobs: target: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 777a11c..e5830a5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,6 +24,7 @@ on: permissions: contents: write checks: read + id-token: write jobs: cut: From facc6ef71e6aa2abd253268ab1872b0d7626c9a6 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:29:15 -0400 Subject: [PATCH 6/8] fix(iam): let staging githubdeploy GetObject the release zip (#154) * fix(iam): let staging githubdeploy GetObject the release zip * fix(iam): allow staging githubdeploy to cache EB processed extensions * fix(iam): allow staging githubdeploy GetObjectAcl for EB updates * fix(iam): grant staging githubdeploy named S3 reads on EB resources prefix * fix(iam): allow staging githubdeploy to delete EB version cache objects * fix(iam): scope staging githubdeploy S3 object access to the EB bucket * fix(iam): allow staging githubdeploy PutObjectVersionAcl on EB artifacts * fix(iam): allow staging githubdeploy GetBucketPolicy on the EB bucket * fix(iam): scope staging githubdeploy S3 objects to SHOC and staging EB prefixes --- .../live/modules/environment-owned/main.tf | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/terraform/live/modules/environment-owned/main.tf b/terraform/live/modules/environment-owned/main.tf index cc01dc4..8bf364b 100644 --- a/terraform/live/modules/environment-owned/main.tf +++ b/terraform/live/modules/environment-owned/main.tf @@ -292,17 +292,37 @@ data "aws_iam_policy_document" "deploy" { dynamic "statement" { for_each = local.use_legacy_s3_policy ? [] : [1] content { - effect = "Allow" - actions = ["s3:PutObject"] - resources = ["arn:aws:s3:::${local.eb_bucket_name}/${var.eb_application_name}/*"] + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:PutObjectVersionAcl", + "s3:GetObject", + "s3:GetObjectAcl", + "s3:GetObjectVersion", + "s3:GetObjectVersionAcl", + "s3:DeleteObject", + ] + resources = [ + "arn:aws:s3:::${local.eb_bucket_name}/${var.eb_application_name}/*", + "arn:aws:s3:::${local.eb_bucket_name}/resources/_runtime/_embedded_extensions/${var.eb_application_name}/*", + "arn:aws:s3:::${local.eb_bucket_name}/resources/environments/${var.eb_environment_id}/*", + ] } } dynamic "statement" { for_each = local.use_legacy_s3_policy ? [] : [1] content { - effect = "Allow" - actions = ["s3:GetBucketLocation", "s3:ListBucket"] + effect = "Allow" + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:GetBucketPolicy", + "s3:GetBucketAcl", + "s3:GetBucketVersioning", + "s3:GetBucketOwnershipControls", + ] resources = ["arn:aws:s3:::${local.eb_bucket_name}"] } } From 9eb51b528f9a41aafbd85a9dda79c260c9fbc15c Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:13:18 -0400 Subject: [PATCH 7/8] chore(terraform): complete staging environment adoption (#156) * chore(terraform): complete staging environment adoption * test(terraform): include deploy SSM parameters in the import ownership boundary --- scripts/terraform_import_plan_resources.py | 28 ++++++++++++++++++++++ terraform/README.md | 4 ++-- terraform/live/README.md | 5 ++-- terraform/live/staging/main.tf | 4 ++-- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/scripts/terraform_import_plan_resources.py b/scripts/terraform_import_plan_resources.py index dc66365..2feb00d 100644 --- a/scripts/terraform_import_plan_resources.py +++ b/scripts/terraform_import_plan_resources.py @@ -9,6 +9,10 @@ COMMON_RESOURCES = { "module.environment.aws_iam_role_policy.runtime_app_config": "aws_iam_role_policy", "module.environment.aws_iam_role_policy_attachment.web_tier": "aws_iam_role_policy_attachment", "module.environment.aws_secretsmanager_secret.app_config": "aws_secretsmanager_secret", + "module.environment.aws_ssm_parameter.deploy_application_name": "aws_ssm_parameter", + "module.environment.aws_ssm_parameter.deploy_artifacts_bucket": "aws_ssm_parameter", + "module.environment.aws_ssm_parameter.deploy_environment_name": "aws_ssm_parameter", + "module.environment.aws_ssm_parameter.deploy_smoke_url": "aws_ssm_parameter", } REQUIRED_RESOURCES = { @@ -52,6 +56,18 @@ DEV_IMPORT_IDS = { "module.environment.aws_route53_record.api_alias[0]": ( "Z07671212N75U4YLPWZR8_api.dev.seahaven.com_A" ), + "module.environment.aws_ssm_parameter.deploy_application_name": ( + "/shoc-backend/dev/deploy/application-name" + ), + "module.environment.aws_ssm_parameter.deploy_artifacts_bucket": ( + "/shoc-backend/dev/deploy/artifacts-bucket" + ), + "module.environment.aws_ssm_parameter.deploy_environment_name": ( + "/shoc-backend/dev/deploy/environment-name" + ), + "module.environment.aws_ssm_parameter.deploy_smoke_url": ( + "/shoc-backend/dev/deploy/smoke-url" + ), } DEV_IMPORT_BASELINE = { @@ -92,6 +108,18 @@ STAGING_IMPORT_IDS = { "module.environment.aws_route53_record.api_cname[0]": ( "Z02602739VQWBWCAGXP4_api.staging.seahaven.com_CNAME" ), + "module.environment.aws_ssm_parameter.deploy_application_name": ( + "/shoc-backend/staging/deploy/application-name" + ), + "module.environment.aws_ssm_parameter.deploy_artifacts_bucket": ( + "/shoc-backend/staging/deploy/artifacts-bucket" + ), + "module.environment.aws_ssm_parameter.deploy_environment_name": ( + "/shoc-backend/staging/deploy/environment-name" + ), + "module.environment.aws_ssm_parameter.deploy_smoke_url": ( + "/shoc-backend/staging/deploy/smoke-url" + ), } STAGING_IMPORT_BASELINE = { diff --git a/terraform/README.md b/terraform/README.md index d4d2023..d11938b 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -37,8 +37,8 @@ only explicitly allowlisted ownership metadata and the narrowed dev deploy S3 policy. The GitHub Environment **variable** `DEPLOY_ROLE_ARN` is the OIDC role used -by application CD after cutover. Adoption may still have the older -`AWS_DEPLOY_ROLE_ARN` secret until that cutover. +by application CD. Environment secrets `TF_API_TOKEN` and +`AWS_DEPLOY_ROLE_ARN` were removed at cutover. ## Local validation diff --git a/terraform/live/README.md b/terraform/live/README.md index e7630cd..8f3850b 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -142,8 +142,9 @@ cut from **Actions → Release** (`environment`, `bump`, `message`). That workflow waits for CI, tags `vX.Y.Z-staging` from main HEAD with `GITHUB_TOKEN`, then calls deploy. Do not cut prod yet; leave `PROD_APP_CD_ENABLED` unset and do not create the `prod` GitHub Environment. -Staging remains `adoption_complete=false` with a pinned API CNAME until its -import apply is proven after the first `vX.Y.Z-staging` GitHub-owned zip. +Staging import is proven after the first GitHub-owned zip +(`v0.0.1-staging`). Terraform now manages the declared Elastic Beanstalk +settings. The API CNAME stays pinned to the imported ALB target. HCP workspaces stay VCS-driven with auto-apply on after cutover. Speculative plans on every PR are the infra gate. Do not point `TFC_AWS_*` at diff --git a/terraform/live/staging/main.tf b/terraform/live/staging/main.tf index 0368da2..cd64267 100644 --- a/terraform/live/staging/main.tf +++ b/terraform/live/staging/main.tf @@ -26,8 +26,8 @@ module "environment" { aws_account_id = local.aws_account_id aws_region = local.aws_region environment = "staging" - adoption_complete = false - manage_eb_settings = false + adoption_complete = true + manage_eb_settings = true eb_application_name = local.eb_application_name eb_environment_name = local.eb_environment_name eb_environment_id = local.eb_environment_id From 4fb4159df2463b41f0eeba26c55d4696c52ae1d0 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:10:13 -0400 Subject: [PATCH 8/8] fix(iam): scope staging githubdeploy uploads to the staging release prefix (#158) --- terraform/live/README.md | 10 ++++++-- .../live/modules/environment-owned/main.tf | 24 ++++++++++++++++++- terraform/live/staging/main.tf | 2 +- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/terraform/live/README.md b/terraform/live/README.md index 8f3850b..e4af673 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -15,7 +15,10 @@ shared or Elastic Beanstalk-generated infrastructure. The shared `shoc-backend` Elastic Beanstalk application and `shoc-sqlserver-shared` RDS instance, VPC, subnets, EB service role, shared certificate, shared RDS security group, and EB-generated SG/ALB/ASG/CloudFormation -resources must never enter an environment state. +resources must never enter an environment state. Both roots leave +`instance_security_group_id` null: the AWS provider reports the EB-generated +`awseb-*-AWSEBSecurityGroup` as an empty `SecurityGroups` setting, so pinning it +produces a permanent update diff. Secret values are not Terraform resources, variables, outputs, or managed EB settings. Terraform manages the app-config secret shell and maps approved JSON @@ -132,7 +135,10 @@ exact bundle it uploads; bundle bytes never enter Terraform plans or state. GitHub Actions owns application versions. It compiles the bundle, uploads it, creates the Elastic Beanstalk application version, and calls `UpdateEnvironment`. Terraform ignores `version_label` so those deploys are not -drift. If health, smoke, or the webhook probe fails after that update, the job +drift. The non-legacy deploy policy writes bundles only under +`shoc-backend/releases//*`; the Elastic Beanstalk staging +prefixes (`resources/_runtime/_embedded_extensions/shoc-backend/*` and +`resources/environments//*`) keep the full object and ACL action set. If health, smoke, or the webhook probe fails after that update, the job restores the previous Elastic Beanstalk version label. Database migrations already applied by the failed bundle are not reverted. Deploy parameters are read from `/shoc-backend//deploy/*` SSM parameters this module writes. diff --git a/terraform/live/modules/environment-owned/main.tf b/terraform/live/modules/environment-owned/main.tf index 8bf364b..713f7fa 100644 --- a/terraform/live/modules/environment-owned/main.tf +++ b/terraform/live/modules/environment-owned/main.tf @@ -289,9 +289,32 @@ data "aws_iam_policy_document" "deploy" { } } + # deploy.yaml uploads each bundle to + # /releases////site.zip and Elastic Beanstalk + # reads it back from there. The deploy role never writes another + # environment's release prefix. dynamic "statement" { for_each = local.use_legacy_s3_policy ? [] : [1] content { + sid = "UploadReleaseBundle" + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject", + ] + resources = [ + "arn:aws:s3:::${local.eb_bucket_name}/${var.eb_application_name}/releases/${var.environment}/*", + ] + } + } + + # Elastic Beanstalk stages the processed version, embedded extensions, + # and manifests under environment-scoped prefixes and requires object ACLs + # on this BucketOwnerPreferred bucket. + dynamic "statement" { + for_each = local.use_legacy_s3_policy ? [] : [1] + content { + sid = "ManageEnvironmentArtifacts" effect = "Allow" actions = [ "s3:PutObject", @@ -304,7 +327,6 @@ data "aws_iam_policy_document" "deploy" { "s3:DeleteObject", ] resources = [ - "arn:aws:s3:::${local.eb_bucket_name}/${var.eb_application_name}/*", "arn:aws:s3:::${local.eb_bucket_name}/resources/_runtime/_embedded_extensions/${var.eb_application_name}/*", "arn:aws:s3:::${local.eb_bucket_name}/resources/environments/${var.eb_environment_id}/*", ] diff --git a/terraform/live/staging/main.tf b/terraform/live/staging/main.tf index cd64267..1964ad5 100644 --- a/terraform/live/staging/main.tf +++ b/terraform/live/staging/main.tf @@ -35,7 +35,7 @@ module "environment" { vpc_id = "vpc-0d16336143f3da25e" instance_subnet_ids = ["subnet-02946ccd6735742e9", "subnet-0bedaa6e0c750be4f", "subnet-09eaf2bfa468d206f"] load_balancer_subnet_ids = ["subnet-09eaf2bfa468d206f", "subnet-02946ccd6735742e9", "subnet-0bedaa6e0c750be4f"] - instance_security_group_id = "sg-02ea36a6719217fa2" + instance_security_group_id = null eb_service_role_name = "shoc-eb-service-role" shared_certificate_arn = "arn:aws:acm:us-east-1:396287094661:certificate/2b78e74f-7b65-4b82-a413-7a498b102f00" runtime_role_name = "shoc-backend-staging"