diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4a7091e..d327271 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,6 +26,9 @@ jobs: dotnet-version: "8.0.x" - name: Repository quality gate + env: + BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/dev' }} + HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} run: bash scripts/governance-check.sh - name: Build Elastic Beanstalk source bundle @@ -141,6 +144,92 @@ jobs: echo "Application version did not become PROCESSED." >&2 exit 1 + - name: Discard blocking VCS run before GitHub CD + run: | + set -euo pipefail + python3 << 'PY' + import json, os, urllib.error, urllib.request + + token = os.environ["TF_API_TOKEN"] + workspace = "shoc-backend-dev" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/vnd.api+json", + } + + def get(url): + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as resp: + return json.load(resp) + + def post(url, payload): + data = json.dumps(payload).encode() + req = urllib.request.Request( + url, data=data, method="POST", headers=headers + ) + try: + with urllib.request.urlopen(req) as resp: + return resp.status + except urllib.error.HTTPError as exc: + if exc.code in (409, 404): + body = exc.read().decode("utf-8", "replace") + print(f"discard returned HTTP {exc.code}: {body}") + return exc.code + raise + + ws = get( + f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" + )["data"] + attrs = ws["attributes"] + if attrs.get("auto-apply") is True: + raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue") + if not attrs.get("speculative-enabled"): + raise SystemExit("speculative plans are off; refuse to continue") + if (attrs.get("vcs-repo") or {}).get("tags-regex"): + raise SystemExit("tag-based VCS triggering is set; refuse to continue") + expected_patterns = [ + "terraform/live/dev/**", + "terraform/live/modules/**", + ] + if attrs.get("trigger-patterns") != expected_patterns: + raise SystemExit( + "trigger-patterns must be " + f"{expected_patterns}; got {attrs.get('trigger-patterns')}" + ) + if not attrs.get("locked"): + print("workspace is unlocked") + raise SystemExit(0) + + current = ( + ws.get("relationships", {}) + .get("current-run", {}) + .get("data") + ) + if not current: + raise SystemExit("workspace is locked without a current run") + run_id = current["id"] + run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] + run_attrs = run["attributes"] + status = run_attrs.get("status") + plan_only = run_attrs.get("plan-only") + print(f"current run {run_id} status={status} plan-only={plan_only}") + if plan_only: + print("speculative run does not block GitHub CD") + raise SystemExit(0) + if status in {"applying", "apply_queued"}: + raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") + discardable = { + "pending", "planned", "cost_estimated", "policy_checked", "policy_override" + } + if status not in discardable: + raise SystemExit(f"{run_id} status {status} is not discardable") + code = post( + f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", + {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, + ) + print(f"discarded {run_id} http={code}") + PY + - name: Create Terraform release run id: release-run uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 @@ -184,11 +273,44 @@ jobs: - name: Apply Terraform release run id: release-apply + continue-on-error: true uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 with: run: ${{ steps.release-run.outputs.run_id }} comment: Apply version-only release from GitHub Actions ${{ github.sha }} + - name: Treat already-applied release run as success + env: + APPLY_OUTCOME: ${{ steps.release-apply.outcome }} + RUN_ID: ${{ steps.release-run.outputs.run_id }} + run: | + set -euo pipefail + if [ "$APPLY_OUTCOME" = "success" ]; then + echo "Apply succeeded." + exit 0 + fi + python3 << 'PY' + import json, os, urllib.request + run_id = os.environ["RUN_ID"] + token = os.environ["TF_API_TOKEN"] + req = urllib.request.Request( + f"https://app.terraform.io/api/v2/runs/{run_id}", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/vnd.api+json", + }, + ) + with urllib.request.urlopen(req) as resp: + status = json.load(resp)["data"]["attributes"]["status"] + print(f"HCP run {run_id} status={status}") + if status == "applied": + raise SystemExit(0) + raise SystemExit( + f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " + f"HCP status={status}" + ) + PY + - name: Verify exact application version is active run: | set -euo pipefail @@ -294,16 +416,103 @@ jobs: echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}" id: rollback-prepare + - name: Discard blocking VCS run before GitHub rollback + id: rollback-discard-vcs + if: failure() && steps.rollback-prepare.outputs.rollback_label != '' + run: | + set -euo pipefail + python3 << 'PY' + import json, os, urllib.error, urllib.request + + token = os.environ["TF_API_TOKEN"] + workspace = "shoc-backend-dev" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/vnd.api+json", + } + + def get(url): + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as resp: + return json.load(resp) + + def post(url, payload): + data = json.dumps(payload).encode() + req = urllib.request.Request( + url, data=data, method="POST", headers=headers + ) + try: + with urllib.request.urlopen(req) as resp: + return resp.status + except urllib.error.HTTPError as exc: + if exc.code in (409, 404): + body = exc.read().decode("utf-8", "replace") + print(f"discard returned HTTP {exc.code}: {body}") + return exc.code + raise + + ws = get( + f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}" + )["data"] + attrs = ws["attributes"] + if attrs.get("auto-apply") is True: + raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue") + if not attrs.get("speculative-enabled"): + raise SystemExit("speculative plans are off; refuse to continue") + if (attrs.get("vcs-repo") or {}).get("tags-regex"): + raise SystemExit("tag-based VCS triggering is set; refuse to continue") + expected_patterns = [ + "terraform/live/dev/**", + "terraform/live/modules/**", + ] + if attrs.get("trigger-patterns") != expected_patterns: + raise SystemExit( + "trigger-patterns must be " + f"{expected_patterns}; got {attrs.get('trigger-patterns')}" + ) + if not attrs.get("locked"): + print("workspace is unlocked") + raise SystemExit(0) + + current = ( + ws.get("relationships", {}) + .get("current-run", {}) + .get("data") + ) + if not current: + raise SystemExit("workspace is locked without a current run") + run_id = current["id"] + run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"] + run_attrs = run["attributes"] + status = run_attrs.get("status") + plan_only = run_attrs.get("plan-only") + print(f"current run {run_id} status={status} plan-only={plan_only}") + if plan_only: + print("speculative run does not block GitHub CD") + raise SystemExit(0) + if status in {"applying", "apply_queued"}: + raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply") + discardable = { + "pending", "planned", "cost_estimated", "policy_checked", "policy_override" + } + if status not in discardable: + raise SystemExit(f"{run_id} status {status} is not discardable") + code = post( + f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard", + {"comment": "Discarded so GitHub CD can create the version-only applyable run"}, + ) + print(f"discarded {run_id} http={code}") + PY + - name: Create Terraform rollback run id: rollback-run - if: failure() && steps.rollback-prepare.outputs.rollback_label != '' + if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success' uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 env: TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"' with: workspace: shoc-backend-dev message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions" - - name: Read Terraform rollback plan counts id: rollback-plan if: failure() && steps.rollback-run.outcome == 'success' @@ -344,13 +553,48 @@ jobs: - name: Apply Terraform rollback run id: rollback-apply if: failure() && steps.rollback-json-guard.outcome == 'success' + continue-on-error: true uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2 with: run: ${{ steps.rollback-run.outputs.run_id }} comment: Apply version-only rollback from GitHub Actions ${{ github.sha }} + - name: Treat already-applied rollback run as success + id: rollback-apply-result + if: failure() && steps.rollback-apply.outcome != 'skipped' + env: + APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }} + RUN_ID: ${{ steps.rollback-run.outputs.run_id }} + run: | + set -euo pipefail + if [ "$APPLY_OUTCOME" = "success" ]; then + echo "Apply succeeded." + exit 0 + fi + python3 << 'PY' + import json, os, urllib.request + run_id = os.environ["RUN_ID"] + token = os.environ["TF_API_TOKEN"] + req = urllib.request.Request( + f"https://app.terraform.io/api/v2/runs/{run_id}", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/vnd.api+json", + }, + ) + with urllib.request.urlopen(req) as resp: + status = json.load(resp)["data"]["attributes"]["status"] + print(f"HCP run {run_id} status={status}") + if status == "applied": + raise SystemExit(0) + raise SystemExit( + f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} " + f"HCP status={status}" + ) + PY + - name: Verify previous application version is active - if: failure() && steps.rollback-apply.outcome == 'success' + if: failure() && steps.rollback-apply-result.outcome == 'success' run: | set -euo pipefail prev="${{ steps.rollback-prepare.outputs.rollback_label }}" diff --git a/terraform/live/README.md b/terraform/live/README.md index fe486cd..f9869cd 100644 --- a/terraform/live/README.md +++ b/terraform/live/README.md @@ -144,7 +144,13 @@ leave the live version unchanged. Application-CD runs pass the immutable workspace, in a variable set, or in `terraform.tfvars`. Do not upload a new configuration version on application releases; `create-run` reuses the workspace's last applied VCS config. Global auto-apply stays off. GitHub -applies only after `plan-output` counts are `0/1/0` and +`apply-run` treats an already-applied run as success so a mis-set auto-apply +cannot start a false-failure rollback. The workspace stays branch-based on +`dev` with Automatic Speculative Plans enabled and trigger patterns +`terraform/live/dev/**` and `terraform/live/modules/**`. GitHub discards a +leftover non-speculative VCS run before `create-run`, so a merge to `dev` +cannot lock the workspace out from under GitHub CD. GitHub applies only after +`plan-output` counts are `0/1/0` and `scripts/check-terraform-release-plan.py` accepts a version-only plan JSON. Staging keeps today's direct Elastic Beanstalk deploy path until staging @@ -181,5 +187,8 @@ identifiers make accidental cross-environment reuse fail review and planning. ## Safety invariants - Auto-apply remains off. +- VCS stays branch-based on `dev` with speculative PR plans enabled and + trigger patterns `terraform/live/dev/**` and `terraform/live/modules/**`. + Do not switch Automatic Run Triggering to tag-based. - Org baseline owns final HCP plan/apply permissions and manager tags. - Every imported Terraform resource has `prevent_destroy`.