fix(deploy): treat applied HCP runs as success (#108)

* fix(deploy): treat applied HCP runs as success

* fix(deploy): wait on rollback apply result

* fix(deploy): compare G3 formatting against the PR merge base

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(deploy): fence rollback create-run on unlock success

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

.github/workflows/deploy.yml

@@ -26,6 +26,9 @@ jobs:
           dotnet-version: "8.0.x"
 
       - name: Repository quality gate
+        env:
+          BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/dev' }}
+          HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
         run: bash scripts/governance-check.sh
 
       - name: Build Elastic Beanstalk source bundle
@@ -141,6 +144,92 @@ jobs:
           echo "Application version did not become PROCESSED." >&2
           exit 1
 
+      - name: Discard blocking VCS run before GitHub CD
+        run: |
+          set -euo pipefail
+          python3 << 'PY'
+          import json, os, urllib.error, urllib.request
+
+          token = os.environ["TF_API_TOKEN"]
+          workspace = "shoc-backend-dev"
+          headers = {
+              "Authorization": f"Bearer {token}",
+              "Content-Type": "application/vnd.api+json",
+          }
+
+          def get(url):
+              req = urllib.request.Request(url, headers=headers)
+              with urllib.request.urlopen(req) as resp:
+                  return json.load(resp)
+
+          def post(url, payload):
+              data = json.dumps(payload).encode()
+              req = urllib.request.Request(
+                  url, data=data, method="POST", headers=headers
+              )
+              try:
+                  with urllib.request.urlopen(req) as resp:
+                      return resp.status
+              except urllib.error.HTTPError as exc:
+                  if exc.code in (409, 404):
+                      body = exc.read().decode("utf-8", "replace")
+                      print(f"discard returned HTTP {exc.code}: {body}")
+                      return exc.code
+                  raise
+
+          ws = get(
+              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
+          )["data"]
+          attrs = ws["attributes"]
+          if attrs.get("auto-apply") is True:
+              raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
+          if not attrs.get("speculative-enabled"):
+              raise SystemExit("speculative plans are off; refuse to continue")
+          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
+              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
+          expected_patterns = [
+              "terraform/live/dev/**",
+              "terraform/live/modules/**",
+          ]
+          if attrs.get("trigger-patterns") != expected_patterns:
+              raise SystemExit(
+                  "trigger-patterns must be "
+                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
+              )
+          if not attrs.get("locked"):
+              print("workspace is unlocked")
+              raise SystemExit(0)
+
+          current = (
+              ws.get("relationships", {})
+              .get("current-run", {})
+              .get("data")
+          )
+          if not current:
+              raise SystemExit("workspace is locked without a current run")
+          run_id = current["id"]
+          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
+          run_attrs = run["attributes"]
+          status = run_attrs.get("status")
+          plan_only = run_attrs.get("plan-only")
+          print(f"current run {run_id} status={status} plan-only={plan_only}")
+          if plan_only:
+              print("speculative run does not block GitHub CD")
+              raise SystemExit(0)
+          if status in {"applying", "apply_queued"}:
+              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
+          discardable = {
+              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
+          }
+          if status not in discardable:
+              raise SystemExit(f"{run_id} status {status} is not discardable")
+          code = post(
+              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
+              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
+          )
+          print(f"discarded {run_id} http={code}")
+          PY
+
       - name: Create Terraform release run
         id: release-run
         uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
@@ -184,11 +273,44 @@ jobs:
 
       - name: Apply Terraform release run
         id: release-apply
+        continue-on-error: true
         uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
         with:
           run: ${{ steps.release-run.outputs.run_id }}
           comment: Apply version-only release from GitHub Actions ${{ github.sha }}
 
+      - name: Treat already-applied release run as success
+        env:
+          APPLY_OUTCOME: ${{ steps.release-apply.outcome }}
+          RUN_ID: ${{ steps.release-run.outputs.run_id }}
+        run: |
+          set -euo pipefail
+          if [ "$APPLY_OUTCOME" = "success" ]; then
+            echo "Apply succeeded."
+            exit 0
+          fi
+          python3 << 'PY'
+          import json, os, urllib.request
+          run_id = os.environ["RUN_ID"]
+          token = os.environ["TF_API_TOKEN"]
+          req = urllib.request.Request(
+              f"https://app.terraform.io/api/v2/runs/{run_id}",
+              headers={
+                  "Authorization": f"Bearer {token}",
+                  "Content-Type": "application/vnd.api+json",
+              },
+          )
+          with urllib.request.urlopen(req) as resp:
+              status = json.load(resp)["data"]["attributes"]["status"]
+          print(f"HCP run {run_id} status={status}")
+          if status == "applied":
+              raise SystemExit(0)
+          raise SystemExit(
+              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
+              f"HCP status={status}"
+          )
+          PY
+
       - name: Verify exact application version is active
         run: |
           set -euo pipefail
@@ -294,16 +416,103 @@ jobs:
           echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}"
         id: rollback-prepare
 
+      - name: Discard blocking VCS run before GitHub rollback
+        id: rollback-discard-vcs
+        if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
+        run: |
+          set -euo pipefail
+          python3 << 'PY'
+          import json, os, urllib.error, urllib.request
+
+          token = os.environ["TF_API_TOKEN"]
+          workspace = "shoc-backend-dev"
+          headers = {
+              "Authorization": f"Bearer {token}",
+              "Content-Type": "application/vnd.api+json",
+          }
+
+          def get(url):
+              req = urllib.request.Request(url, headers=headers)
+              with urllib.request.urlopen(req) as resp:
+                  return json.load(resp)
+
+          def post(url, payload):
+              data = json.dumps(payload).encode()
+              req = urllib.request.Request(
+                  url, data=data, method="POST", headers=headers
+              )
+              try:
+                  with urllib.request.urlopen(req) as resp:
+                      return resp.status
+              except urllib.error.HTTPError as exc:
+                  if exc.code in (409, 404):
+                      body = exc.read().decode("utf-8", "replace")
+                      print(f"discard returned HTTP {exc.code}: {body}")
+                      return exc.code
+                  raise
+
+          ws = get(
+              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
+          )["data"]
+          attrs = ws["attributes"]
+          if attrs.get("auto-apply") is True:
+              raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
+          if not attrs.get("speculative-enabled"):
+              raise SystemExit("speculative plans are off; refuse to continue")
+          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
+              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
+          expected_patterns = [
+              "terraform/live/dev/**",
+              "terraform/live/modules/**",
+          ]
+          if attrs.get("trigger-patterns") != expected_patterns:
+              raise SystemExit(
+                  "trigger-patterns must be "
+                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
+              )
+          if not attrs.get("locked"):
+              print("workspace is unlocked")
+              raise SystemExit(0)
+
+          current = (
+              ws.get("relationships", {})
+              .get("current-run", {})
+              .get("data")
+          )
+          if not current:
+              raise SystemExit("workspace is locked without a current run")
+          run_id = current["id"]
+          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
+          run_attrs = run["attributes"]
+          status = run_attrs.get("status")
+          plan_only = run_attrs.get("plan-only")
+          print(f"current run {run_id} status={status} plan-only={plan_only}")
+          if plan_only:
+              print("speculative run does not block GitHub CD")
+              raise SystemExit(0)
+          if status in {"applying", "apply_queued"}:
+              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
+          discardable = {
+              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
+          }
+          if status not in discardable:
+              raise SystemExit(f"{run_id} status {status} is not discardable")
+          code = post(
+              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
+              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
+          )
+          print(f"discarded {run_id} http={code}")
+          PY
+
       - name: Create Terraform rollback run
         id: rollback-run
-        if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
+        if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success'
         uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
         env:
           TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"'
         with:
           workspace: shoc-backend-dev
           message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions"
-
       - name: Read Terraform rollback plan counts
         id: rollback-plan
         if: failure() && steps.rollback-run.outcome == 'success'
@@ -344,13 +553,48 @@ jobs:
       - name: Apply Terraform rollback run
         id: rollback-apply
         if: failure() && steps.rollback-json-guard.outcome == 'success'
+        continue-on-error: true
         uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
         with:
           run: ${{ steps.rollback-run.outputs.run_id }}
           comment: Apply version-only rollback from GitHub Actions ${{ github.sha }}
 
+      - name: Treat already-applied rollback run as success
+        id: rollback-apply-result
+        if: failure() && steps.rollback-apply.outcome != 'skipped'
+        env:
+          APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }}
+          RUN_ID: ${{ steps.rollback-run.outputs.run_id }}
+        run: |
+          set -euo pipefail
+          if [ "$APPLY_OUTCOME" = "success" ]; then
+            echo "Apply succeeded."
+            exit 0
+          fi
+          python3 << 'PY'
+          import json, os, urllib.request
+          run_id = os.environ["RUN_ID"]
+          token = os.environ["TF_API_TOKEN"]
+          req = urllib.request.Request(
+              f"https://app.terraform.io/api/v2/runs/{run_id}",
+              headers={
+                  "Authorization": f"Bearer {token}",
+                  "Content-Type": "application/vnd.api+json",
+              },
+          )
+          with urllib.request.urlopen(req) as resp:
+              status = json.load(resp)["data"]["attributes"]["status"]
+          print(f"HCP run {run_id} status={status}")
+          if status == "applied":
+              raise SystemExit(0)
+          raise SystemExit(
+              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
+              f"HCP status={status}"
+          )
+          PY
+
       - name: Verify previous application version is active
-        if: failure() && steps.rollback-apply.outcome == 'success'
+        if: failure() && steps.rollback-apply-result.outcome == 'success'
         run: |
           set -euo pipefail
           prev="${{ steps.rollback-prepare.outputs.rollback_label }}"

terraform/live/README.md

@@ -144,7 +144,13 @@ leave the live version unchanged. Application-CD runs pass the immutable
 workspace, in a variable set, or in `terraform.tfvars`. Do not upload a new
 configuration version on application releases; `create-run` reuses the
 workspace's last applied VCS config. Global auto-apply stays off. GitHub
-applies only after `plan-output` counts are `0/1/0` and
+`apply-run` treats an already-applied run as success so a mis-set auto-apply
+cannot start a false-failure rollback. The workspace stays branch-based on
+`dev` with Automatic Speculative Plans enabled and trigger patterns
+`terraform/live/dev/**` and `terraform/live/modules/**`. GitHub discards a
+leftover non-speculative VCS run before `create-run`, so a merge to `dev`
+cannot lock the workspace out from under GitHub CD. GitHub applies only after
+`plan-output` counts are `0/1/0` and
 `scripts/check-terraform-release-plan.py` accepts a version-only plan JSON.
 
 Staging keeps today's direct Elastic Beanstalk deploy path until staging
@@ -181,5 +187,8 @@ identifiers make accidental cross-environment reuse fail review and planning.
 ## Safety invariants
 
 - Auto-apply remains off.
+- VCS stays branch-based on `dev` with speculative PR plans enabled and
+  trigger patterns `terraform/live/dev/**` and `terraform/live/modules/**`.
+  Do not switch Automatic Run Triggering to tag-based.
 - Org baseline owns final HCP plan/apply permissions and manager tags.
 - Every imported Terraform resource has `prevent_destroy`.