name: Validate and deploy

on:
  pull_request:
    branches: [dev, staging, main]
  push:
    branches: [dev, staging]
  workflow_dispatch:

permissions:
  contents: read

jobs:
  validate:
    name: Validate deployable source bundle
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - name: Set up .NET
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
        with:
          dotnet-version: "8.0.x"

      - name: Repository quality gate
        env:
          BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/dev' }}
          HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
        run: bash scripts/governance-check.sh

      - name: Build Elastic Beanstalk source bundle
        run: bash scripts/package-elastic-beanstalk.sh

      - name: Inspect source bundle contract
        run: bash scripts/validate-elastic-beanstalk-bundle.sh

  deploy-dev:
    name: Deploy shoc-backend-dev through Terraform
    if: >
      (github.event_name == 'push' && github.ref == 'refs/heads/dev' &&
       vars.TERRAFORM_APP_CD_ENABLED == 'true') ||
      (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/dev')
    needs: validate
    runs-on: ubuntu-latest
    timeout-minutes: 180
    permissions:
      contents: read
      id-token: write
    environment:
      name: dev
    concurrency:
      group: deploy-dev
      cancel-in-progress: false
    env:
      TF_CLOUD_ORGANIZATION: seahaven
      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}
      EB_APPLICATION_NAME: shoc-backend
      EB_ENVIRONMENT_NAME: shoc-backend-dev
      SMOKE_URL: https://api.dev.seahaven.com
      EB_BUCKET: elasticbeanstalk-us-east-1-396287094661
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up .NET
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
        with:
          dotnet-version: "8.0.x"

      - name: Build Elastic Beanstalk source bundle
        run: bash scripts/package-elastic-beanstalk.sh

      - name: Validate exact release bundle
        run: bash scripts/validate-elastic-beanstalk-bundle.sh

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
        with:
          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
          aws-region: us-east-1
          audience: sts.amazonaws.com

      - name: Capture current environment version
        run: |
          set -euo pipefail
          prev="$(aws elasticbeanstalk describe-environments \
            --environment-names "${EB_ENVIRONMENT_NAME}" \
            --region us-east-1 \
            --query 'Environments[0].VersionLabel' \
            --output text)"
          echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
          echo "Previous version label: $prev"

      - name: Assign immutable release identity
        id: release
        run: |
          set -euo pipefail
          version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          s3_key="shoc-backend/releases/dev/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip"
          {
            echo "version_label=${version_label}"
            echo "s3_key=${s3_key}"
          } >> "${GITHUB_OUTPUT}"

      - name: Upload immutable bundle
        run: |
          set -euo pipefail
          aws s3 cp .artifacts/elastic-beanstalk/site.zip \
            "s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \
            --region us-east-1

      - name: Create Elastic Beanstalk application version
        run: |
          set -euo pipefail
          aws elasticbeanstalk create-application-version \
            --application-name "${EB_APPLICATION_NAME}" \
            --version-label "${{ steps.release.outputs.version_label }}" \
            --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \
            --source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \
            --process \
            --region us-east-1

          status="UNPROCESSED"
          for _ in $(seq 1 36); do
            status="$(aws elasticbeanstalk describe-application-versions \
              --application-name "${EB_APPLICATION_NAME}" \
              --version-labels "${{ steps.release.outputs.version_label }}" \
              --region us-east-1 \
              --query 'ApplicationVersions[0].Status' \
              --output text)"
            echo "application version status: $status"
            if [ "$status" = "PROCESSED" ]; then
              exit 0
            fi
            if [ "$status" = "FAILED" ]; then
              echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2
              exit 1
            fi
            sleep 5
          done
          echo "Application version did not become PROCESSED." >&2
          exit 1
      - name: Discard blocking VCS run before GitHub CD
        run: |
          set -euo pipefail
          python3 << 'PY'
          import json, os, urllib.error, urllib.request

          token = os.environ["TF_API_TOKEN"]
          workspace = "shoc-backend-dev"
          headers = {
              "Authorization": f"Bearer {token}",
              "Content-Type": "application/vnd.api+json",
          }

          def get(url):
              req = urllib.request.Request(url, headers=headers)
              with urllib.request.urlopen(req) as resp:
                  return json.load(resp)

          def post(url, payload):
              data = json.dumps(payload).encode()
              req = urllib.request.Request(
                  url, data=data, method="POST", headers=headers
              )
              try:
                  with urllib.request.urlopen(req) as resp:
                      return resp.status
              except urllib.error.HTTPError as exc:
                  if exc.code in (409, 404):
                      body = exc.read().decode("utf-8", "replace")
                      print(f"discard returned HTTP {exc.code}: {body}")
                      return exc.code
                  raise

          ws = get(
              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
          )["data"]
          attrs = ws["attributes"]
          if attrs.get("auto-apply") is True:
              raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
          if not attrs.get("speculative-enabled"):
              raise SystemExit("speculative plans are off; refuse to continue")
          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
          expected_patterns = [
              "terraform/live/dev/**",
              "terraform/live/modules/**",
          ]
          if attrs.get("trigger-patterns") != expected_patterns:
              raise SystemExit(
                  "trigger-patterns must be "
                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
              )
          if not attrs.get("locked"):
              print("workspace is unlocked")
              raise SystemExit(0)

          current = (
              ws.get("relationships", {})
              .get("current-run", {})
              .get("data")
          )
          if not current:
              raise SystemExit("workspace is locked without a current run")
          run_id = current["id"]
          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
          run_attrs = run["attributes"]
          status = run_attrs.get("status")
          plan_only = run_attrs.get("plan-only")
          print(f"current run {run_id} status={status} plan-only={plan_only}")
          if plan_only:
              print("speculative run does not block GitHub CD")
              raise SystemExit(0)
          if status in {"applying", "apply_queued"}:
              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
          discardable = {
              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
          }
          if status not in discardable:
              raise SystemExit(f"{run_id} status {status} is not discardable")
          code = post(
              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
          )
          print(f"discarded {run_id} http={code}")
          PY

      - name: Create Terraform release run
        id: release-run
        uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        env:
          TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"'
        with:
          workspace: shoc-backend-dev
          message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions"

      - name: Read Terraform release plan counts
        id: release-plan
        uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          plan: ${{ steps.release-run.outputs.plan_id }}

      - name: Reject non-version-only resource counts
        env:
          PLAN_ADD: ${{ steps.release-plan.outputs.add }}
          PLAN_CHANGE: ${{ steps.release-plan.outputs.change }}
          PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }}
        run: |
          set -euo pipefail
          if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
            echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
            exit 1
          fi

      - name: Guard version-only Terraform plan
        run: |
          set -euo pipefail
          python scripts/check-terraform-release-plan.py \
            --plan-id "${{ steps.release-run.outputs.plan_id }}" \
            --expected-version-label "${{ steps.release.outputs.version_label }}"

      - name: Discard release run when the guard fails
        if: failure() && steps.release-run.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.release-run.outputs.run_id }}
          comment: Rejected by the version-only plan guard from GitHub Actions

      - name: Apply Terraform release run
        id: release-apply
        continue-on-error: true
        uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.release-run.outputs.run_id }}
          comment: Apply version-only release from GitHub Actions ${{ github.sha }}

      - name: Treat already-applied release run as success
        env:
          APPLY_OUTCOME: ${{ steps.release-apply.outcome }}
          RUN_ID: ${{ steps.release-run.outputs.run_id }}
        run: |
          set -euo pipefail
          if [ "$APPLY_OUTCOME" = "success" ]; then
            echo "Apply succeeded."
            exit 0
          fi
          python3 << 'PY'
          import json, os, urllib.request
          run_id = os.environ["RUN_ID"]
          token = os.environ["TF_API_TOKEN"]
          req = urllib.request.Request(
              f"https://app.terraform.io/api/v2/runs/{run_id}",
              headers={
                  "Authorization": f"Bearer {token}",
                  "Content-Type": "application/vnd.api+json",
              },
          )
          with urllib.request.urlopen(req) as resp:
              status = json.load(resp)["data"]["attributes"]["status"]
          print(f"HCP run {run_id} status={status}")
          if status == "applied":
              raise SystemExit(0)
          raise SystemExit(
              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
              f"HCP status={status}"
          )
          PY

      - name: Verify exact application version is active
        run: |
          set -euo pipefail
          expected="${{ steps.release.outputs.version_label }}"
          status="Unknown"
          current="Unknown"
          health="Unknown"

          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"

            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$expected" ]; then
                echo "Environment became Ready on version $current, not the expected $expected." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Expected application version is Ready and healthy."
                exit 0
              fi
              # The expected version IS active. Elastic Beanstalk reports Ready as
              # soon as the rollout finishes, before enhanced health has converged,
              # so deciding on the first Ready poll fails a good release on a health
              # value that was always going to change. Keep polling; an environment
              # that is genuinely unhealthy still fails when the window runs out.
              echo "Expected version is active; waiting for health to leave $health."
            fi
            sleep 15
          done

          echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2
          exit 1

      - name: Post-deploy smoke
        run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}"

      - name: Verify webhook secret source is operational
        run: |
          set -euo pipefail
          response_file="$(mktemp)"
          trap 'rm -f "$response_file"' EXIT
          status="$(curl --silent --show-error \
            --output "$response_file" \
            --write-out '%{http_code}' \
            --request POST \
            --header 'Content-Type: application/json' \
            --header "X-SH-Timestamp: $(date +%s)" \
            --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
            --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
            --data '{}' \
            "${SMOKE_URL}/api/webhooks/work-orders")"
          if [ "$status" != "401" ]; then
            echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2
            sed -n '1,20p' "$response_file" >&2
            exit 1
          fi

      - name: Restore previous application version on failure (schema is not reverted)
        if: failure()
        run: |
          set -euo pipefail
          prev_file=".artifacts/elastic-beanstalk/previous-version.txt"
          if [ ! -f "$prev_file" ]; then
            echo "No previous version captured; nothing to roll back." >&2
            exit 0
          fi
          prev="$(cat "$prev_file")"
          if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
            echo "No previous version recorded; nothing to roll back." >&2
            exit 0
          fi

          echo "Waiting for any in-flight environment update to settle..."
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              break
            fi
            sleep 15
          done

          if [ "$status" != "Ready" ]; then
            echo "Environment did not settle before rollback." >&2
            exit 1
          fi
          if [ "$current" = "$prev" ]; then
            echo "Environment is already on previous version $prev."
            exit 0
          fi
          if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then
            echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2
            exit 1
          fi
          echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}"
        id: rollback-prepare

      - name: Discard blocking VCS run before GitHub rollback
        id: rollback-discard-vcs
        if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
        run: |
          set -euo pipefail
          python3 << 'PY'
          import json, os, urllib.error, urllib.request

          token = os.environ["TF_API_TOKEN"]
          workspace = "shoc-backend-dev"
          headers = {
              "Authorization": f"Bearer {token}",
              "Content-Type": "application/vnd.api+json",
          }

          def get(url):
              req = urllib.request.Request(url, headers=headers)
              with urllib.request.urlopen(req) as resp:
                  return json.load(resp)

          def post(url, payload):
              data = json.dumps(payload).encode()
              req = urllib.request.Request(
                  url, data=data, method="POST", headers=headers
              )
              try:
                  with urllib.request.urlopen(req) as resp:
                      return resp.status
              except urllib.error.HTTPError as exc:
                  if exc.code in (409, 404):
                      body = exc.read().decode("utf-8", "replace")
                      print(f"discard returned HTTP {exc.code}: {body}")
                      return exc.code
                  raise

          ws = get(
              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
          )["data"]
          attrs = ws["attributes"]
          if attrs.get("auto-apply") is True:
              raise SystemExit("shoc-backend-dev auto-apply is on; refuse to continue")
          if not attrs.get("speculative-enabled"):
              raise SystemExit("speculative plans are off; refuse to continue")
          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
          expected_patterns = [
              "terraform/live/dev/**",
              "terraform/live/modules/**",
          ]
          if attrs.get("trigger-patterns") != expected_patterns:
              raise SystemExit(
                  "trigger-patterns must be "
                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
              )
          if not attrs.get("locked"):
              print("workspace is unlocked")
              raise SystemExit(0)

          current = (
              ws.get("relationships", {})
              .get("current-run", {})
              .get("data")
          )
          if not current:
              raise SystemExit("workspace is locked without a current run")
          run_id = current["id"]
          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
          run_attrs = run["attributes"]
          status = run_attrs.get("status")
          plan_only = run_attrs.get("plan-only")
          print(f"current run {run_id} status={status} plan-only={plan_only}")
          if plan_only:
              print("speculative run does not block GitHub CD")
              raise SystemExit(0)
          if status in {"applying", "apply_queued"}:
              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
          discardable = {
              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
          }
          if status not in discardable:
              raise SystemExit(f"{run_id} status {status} is not discardable")
          code = post(
              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
          )
          print(f"discarded {run_id} http={code}")
          PY

      - name: Create Terraform rollback run
        id: rollback-run
        if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        env:
          TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"'
        with:
          workspace: shoc-backend-dev
          message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions"
      - name: Read Terraform rollback plan counts
        id: rollback-plan
        if: failure() && steps.rollback-run.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          plan: ${{ steps.rollback-run.outputs.plan_id }}

      - name: Reject non-version-only rollback counts
        id: rollback-count-guard
        if: failure() && steps.rollback-plan.outcome == 'success'
        env:
          PLAN_ADD: ${{ steps.rollback-plan.outputs.add }}
          PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }}
          PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }}
        run: |
          set -euo pipefail
          if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
            echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
            exit 1
          fi

      - name: Guard version-only Terraform rollback plan
        id: rollback-json-guard
        if: failure() && steps.rollback-count-guard.outcome == 'success'
        run: |
          set -euo pipefail
          python scripts/check-terraform-release-plan.py \
            --plan-id "${{ steps.rollback-run.outputs.plan_id }}" \
            --expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}"

      - name: Discard rollback run when the guard fails
        if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success'
        uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.rollback-run.outputs.run_id }}
          comment: Rejected by the version-only rollback plan guard from GitHub Actions

      - name: Apply Terraform rollback run
        id: rollback-apply
        if: failure() && steps.rollback-json-guard.outcome == 'success'
        continue-on-error: true
        uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.rollback-run.outputs.run_id }}
          comment: Apply version-only rollback from GitHub Actions ${{ github.sha }}

      - name: Treat already-applied rollback run as success
        id: rollback-apply-result
        if: failure() && steps.rollback-apply.outcome != 'skipped'
        env:
          APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }}
          RUN_ID: ${{ steps.rollback-run.outputs.run_id }}
        run: |
          set -euo pipefail
          if [ "$APPLY_OUTCOME" = "success" ]; then
            echo "Apply succeeded."
            exit 0
          fi
          python3 << 'PY'
          import json, os, urllib.request
          run_id = os.environ["RUN_ID"]
          token = os.environ["TF_API_TOKEN"]
          req = urllib.request.Request(
              f"https://app.terraform.io/api/v2/runs/{run_id}",
              headers={
                  "Authorization": f"Bearer {token}",
                  "Content-Type": "application/vnd.api+json",
              },
          )
          with urllib.request.urlopen(req) as resp:
              status = json.load(resp)["data"]["attributes"]["status"]
          print(f"HCP run {run_id} status={status}")
          if status == "applied":
              raise SystemExit(0)
          raise SystemExit(
              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
              f"HCP status={status}"
          )
          PY

      - name: Verify previous application version is active
        if: failure() && steps.rollback-apply-result.outcome == 'success'
        run: |
          set -euo pipefail
          prev="${{ steps.rollback-prepare.outputs.rollback_label }}"
          echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$prev" ]; then
                echo "Rollback reached Ready on version $current, not the previous $prev." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Application version restore complete; previous code is Ready and healthy."
                exit 0
              fi
              # Same convergence gap as the release check above: the previous version
              # is back, health has not settled yet, and reporting a failed rollback
              # here hides the fact that the restore itself worked.
              echo "Previous version is active; waiting for health to leave $health."
            fi
            sleep 15
          done
          echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2
          exit 1

  deploy-staging:
    name: Deploy shoc-backend-staging through Terraform
    if: >
      (github.event_name == 'push' && github.ref == 'refs/heads/staging') ||
      (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging')
    needs: validate
    runs-on: ubuntu-latest
    timeout-minutes: 180
    permissions:
      contents: read
      id-token: write
    environment:
      name: staging
    concurrency:
      group: deploy-staging
      cancel-in-progress: false
    env:
      TF_CLOUD_ORGANIZATION: seahaven
      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}
      EB_APPLICATION_NAME: shoc-backend
      EB_ENVIRONMENT_NAME: shoc-backend-staging
      SMOKE_URL: https://api.staging.seahaven.com
      EB_BUCKET: elasticbeanstalk-us-east-1-396287094661
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up .NET
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
        with:
          dotnet-version: "8.0.x"

      - name: Build Elastic Beanstalk source bundle
        run: bash scripts/package-elastic-beanstalk.sh

      - name: Validate exact release bundle
        run: bash scripts/validate-elastic-beanstalk-bundle.sh

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
        with:
          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
          aws-region: us-east-1
          audience: sts.amazonaws.com

      - name: Capture current environment version
        run: |
          set -euo pipefail
          prev="$(aws elasticbeanstalk describe-environments \
            --environment-names "${EB_ENVIRONMENT_NAME}" \
            --region us-east-1 \
            --query 'Environments[0].VersionLabel' \
            --output text)"
          echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
          echo "Previous version label: $prev"

      - name: Assign immutable release identity
        id: release
        run: |
          set -euo pipefail
          version_label="${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          s3_key="shoc-backend/releases/staging/${GITHUB_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip"
          {
            echo "version_label=${version_label}"
            echo "s3_key=${s3_key}"
          } >> "${GITHUB_OUTPUT}"

      - name: Upload immutable bundle
        run: |
          set -euo pipefail
          aws s3 cp .artifacts/elastic-beanstalk/site.zip \
            "s3://${EB_BUCKET}/${{ steps.release.outputs.s3_key }}" \
            --region us-east-1

      - name: Create Elastic Beanstalk application version
        run: |
          set -euo pipefail
          aws elasticbeanstalk create-application-version \
            --application-name "${EB_APPLICATION_NAME}" \
            --version-label "${{ steps.release.outputs.version_label }}" \
            --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}" \
            --source-bundle "S3Bucket=${EB_BUCKET},S3Key=${{ steps.release.outputs.s3_key }}" \
            --process \
            --region us-east-1

          status="UNPROCESSED"
          for _ in $(seq 1 36); do
            status="$(aws elasticbeanstalk describe-application-versions \
              --application-name "${EB_APPLICATION_NAME}" \
              --version-labels "${{ steps.release.outputs.version_label }}" \
              --region us-east-1 \
              --query 'ApplicationVersions[0].Status' \
              --output text)"
            echo "application version status: $status"
            if [ "$status" = "PROCESSED" ]; then
              exit 0
            fi
            if [ "$status" = "FAILED" ]; then
              echo "Elastic Beanstalk failed to process ${{ steps.release.outputs.version_label }}." >&2
              exit 1
            fi
            sleep 5
          done
          echo "Application version did not become PROCESSED." >&2
          exit 1

      - name: Discard blocking VCS run before GitHub CD
        run: |
          set -euo pipefail
          python3 << 'PY'
          import json, os, urllib.error, urllib.request

          token = os.environ["TF_API_TOKEN"]
          workspace = "shoc-backend-staging"
          headers = {
              "Authorization": f"Bearer {token}",
              "Content-Type": "application/vnd.api+json",
          }

          def get(url):
              req = urllib.request.Request(url, headers=headers)
              with urllib.request.urlopen(req) as resp:
                  return json.load(resp)

          def post(url, payload):
              data = json.dumps(payload).encode()
              req = urllib.request.Request(
                  url, data=data, method="POST", headers=headers
              )
              try:
                  with urllib.request.urlopen(req) as resp:
                      return resp.status
              except urllib.error.HTTPError as exc:
                  if exc.code in (409, 404):
                      body = exc.read().decode("utf-8", "replace")
                      print(f"discard returned HTTP {exc.code}: {body}")
                      return exc.code
                  raise

          ws = get(
              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
          )["data"]
          attrs = ws["attributes"]
          if attrs.get("auto-apply") is True:
              raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue")
          if not attrs.get("speculative-enabled"):
              raise SystemExit("speculative plans are off; refuse to continue")
          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
          expected_patterns = [
              "terraform/live/staging/**",
              "terraform/live/modules/**",
          ]
          if attrs.get("trigger-patterns") != expected_patterns:
              raise SystemExit(
                  "trigger-patterns must be "
                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
              )
          if not attrs.get("locked"):
              print("workspace is unlocked")
              raise SystemExit(0)

          current = (
              ws.get("relationships", {})
              .get("current-run", {})
              .get("data")
          )
          if not current:
              raise SystemExit("workspace is locked without a current run")
          run_id = current["id"]
          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
          run_attrs = run["attributes"]
          status = run_attrs.get("status")
          plan_only = run_attrs.get("plan-only")
          print(f"current run {run_id} status={status} plan-only={plan_only}")
          if plan_only:
              print("speculative run does not block GitHub CD")
              raise SystemExit(0)
          if status in {"applying", "apply_queued"}:
              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
          discardable = {
              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
          }
          if status not in discardable:
              raise SystemExit(f"{run_id} status {status} is not discardable")
          code = post(
              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
          )
          print(f"discarded {run_id} http={code}")
          PY

      - name: Create Terraform release run
        id: release-run
        uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        env:
          TF_VAR_release_version_label: '"${{ steps.release.outputs.version_label }}"'
        with:
          workspace: shoc-backend-staging
          message: "Release ${{ steps.release.outputs.version_label }} from GitHub Actions"

      - name: Read Terraform release plan counts
        id: release-plan
        uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          plan: ${{ steps.release-run.outputs.plan_id }}

      - name: Reject non-version-only resource counts
        env:
          PLAN_ADD: ${{ steps.release-plan.outputs.add }}
          PLAN_CHANGE: ${{ steps.release-plan.outputs.change }}
          PLAN_DESTROY: ${{ steps.release-plan.outputs.destroy }}
        run: |
          set -euo pipefail
          if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
            echo "HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
            exit 1
          fi

      - name: Guard version-only Terraform plan
        run: |
          set -euo pipefail
          python scripts/check-terraform-release-plan.py \
            --plan-id "${{ steps.release-run.outputs.plan_id }}" \
            --expected-version-label "${{ steps.release.outputs.version_label }}"

      - name: Discard release run when the guard fails
        if: failure() && steps.release-run.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.release-run.outputs.run_id }}
          comment: Rejected by the version-only plan guard from GitHub Actions

      - name: Apply Terraform release run
        id: release-apply
        continue-on-error: true
        uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.release-run.outputs.run_id }}
          comment: Apply version-only release from GitHub Actions ${{ github.sha }}

      - name: Treat already-applied release run as success
        env:
          APPLY_OUTCOME: ${{ steps.release-apply.outcome }}
          RUN_ID: ${{ steps.release-run.outputs.run_id }}
        run: |
          set -euo pipefail
          if [ "$APPLY_OUTCOME" = "success" ]; then
            echo "Apply succeeded."
            exit 0
          fi
          python3 << 'PY'
          import json, os, urllib.request
          run_id = os.environ["RUN_ID"]
          token = os.environ["TF_API_TOKEN"]
          req = urllib.request.Request(
              f"https://app.terraform.io/api/v2/runs/{run_id}",
              headers={
                  "Authorization": f"Bearer {token}",
                  "Content-Type": "application/vnd.api+json",
              },
          )
          with urllib.request.urlopen(req) as resp:
              status = json.load(resp)["data"]["attributes"]["status"]
          print(f"HCP run {run_id} status={status}")
          if status == "applied":
              raise SystemExit(0)
          raise SystemExit(
              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
              f"HCP status={status}"
          )
          PY

      - name: Verify exact application version is active
        run: |
          set -euo pipefail
          expected="${{ steps.release.outputs.version_label }}"
          status="Unknown"
          current="Unknown"
          health="Unknown"

          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"

            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$expected" ]; then
                echo "Environment became Ready on version $current, not the expected $expected." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Expected application version is Ready and healthy."
                exit 0
              fi
              # The expected version IS active. Elastic Beanstalk reports Ready as
              # soon as the rollout finishes, before enhanced health has converged,
              # so deciding on the first Ready poll fails a good release on a health
              # value that was always going to change. Keep polling; an environment
              # that is genuinely unhealthy still fails when the window runs out.
              echo "Expected version is active; waiting for health to leave $health."
            fi
            sleep 15
          done

          echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2
          exit 1

      - name: Post-deploy smoke
        run: bash scripts/smoke-elastic-beanstalk.sh "${SMOKE_URL}"

      - name: Verify webhook secret source is operational
        run: |
          set -euo pipefail
          response_file="$(mktemp)"
          trap 'rm -f "$response_file"' EXIT
          status="$(curl --silent --show-error \
            --output "$response_file" \
            --write-out '%{http_code}' \
            --request POST \
            --header 'Content-Type: application/json' \
            --header "X-SH-Timestamp: $(date +%s)" \
            --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
            --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
            --data '{}' \
            "${SMOKE_URL}/api/webhooks/work-orders")"
          if [ "$status" != "401" ]; then
            echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2
            sed -n '1,20p' "$response_file" >&2
            exit 1
          fi

      - name: Restore previous application version on failure (schema is not reverted)
        if: failure()
        run: |
          set -euo pipefail
          prev_file=".artifacts/elastic-beanstalk/previous-version.txt"
          if [ ! -f "$prev_file" ]; then
            echo "No previous version captured; nothing to roll back." >&2
            exit 0
          fi
          prev="$(cat "$prev_file")"
          if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
            echo "No previous version recorded; nothing to roll back." >&2
            exit 0
          fi

          echo "Waiting for any in-flight environment update to settle..."
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              break
            fi
            sleep 15
          done

          if [ "$status" != "Ready" ]; then
            echo "Environment did not settle before rollback." >&2
            exit 1
          fi
          if [ "$current" = "$prev" ]; then
            echo "Environment is already on previous version $prev."
            exit 0
          fi
          if [[ ! "$prev" =~ ^[0-9a-f]{40}-[0-9]+-[0-9]+$ ]]; then
            echo "Previous version $prev is not a Terraform-managed release label; cannot roll back through HCP." >&2
            exit 1
          fi
          echo "rollback_label=$prev" >> "${GITHUB_OUTPUT}"
        id: rollback-prepare

      - name: Discard blocking VCS run before GitHub rollback
        id: rollback-discard-vcs
        if: failure() && steps.rollback-prepare.outputs.rollback_label != ''
        run: |
          set -euo pipefail
          python3 << 'PY'
          import json, os, urllib.error, urllib.request

          token = os.environ["TF_API_TOKEN"]
          workspace = "shoc-backend-staging"
          headers = {
              "Authorization": f"Bearer {token}",
              "Content-Type": "application/vnd.api+json",
          }

          def get(url):
              req = urllib.request.Request(url, headers=headers)
              with urllib.request.urlopen(req) as resp:
                  return json.load(resp)

          def post(url, payload):
              data = json.dumps(payload).encode()
              req = urllib.request.Request(
                  url, data=data, method="POST", headers=headers
              )
              try:
                  with urllib.request.urlopen(req) as resp:
                      return resp.status
              except urllib.error.HTTPError as exc:
                  if exc.code in (409, 404):
                      body = exc.read().decode("utf-8", "replace")
                      print(f"discard returned HTTP {exc.code}: {body}")
                      return exc.code
                  raise

          ws = get(
              f"https://app.terraform.io/api/v2/organizations/seahaven/workspaces/{workspace}"
          )["data"]
          attrs = ws["attributes"]
          if attrs.get("auto-apply") is True:
              raise SystemExit("shoc-backend-staging auto-apply is on; refuse to continue")
          if not attrs.get("speculative-enabled"):
              raise SystemExit("speculative plans are off; refuse to continue")
          if (attrs.get("vcs-repo") or {}).get("tags-regex"):
              raise SystemExit("tag-based VCS triggering is set; refuse to continue")
          expected_patterns = [
              "terraform/live/staging/**",
              "terraform/live/modules/**",
          ]
          if attrs.get("trigger-patterns") != expected_patterns:
              raise SystemExit(
                  "trigger-patterns must be "
                  f"{expected_patterns}; got {attrs.get('trigger-patterns')}"
              )
          if not attrs.get("locked"):
              print("workspace is unlocked")
              raise SystemExit(0)

          current = (
              ws.get("relationships", {})
              .get("current-run", {})
              .get("data")
          )
          if not current:
              raise SystemExit("workspace is locked without a current run")
          run_id = current["id"]
          run = get(f"https://app.terraform.io/api/v2/runs/{run_id}")["data"]
          run_attrs = run["attributes"]
          status = run_attrs.get("status")
          plan_only = run_attrs.get("plan-only")
          print(f"current run {run_id} status={status} plan-only={plan_only}")
          if plan_only:
              print("speculative run does not block GitHub CD")
              raise SystemExit(0)
          if status in {"applying", "apply_queued"}:
              raise SystemExit(f"{run_id} is {status}; wait, do not discard an apply")
          discardable = {
              "pending", "planned", "cost_estimated", "policy_checked", "policy_override"
          }
          if status not in discardable:
              raise SystemExit(f"{run_id} status {status} is not discardable")
          code = post(
              f"https://app.terraform.io/api/v2/runs/{run_id}/actions/discard",
              {"comment": "Discarded so GitHub CD can create the version-only applyable run"},
          )
          print(f"discarded {run_id} http={code}")
          PY

      - name: Create Terraform rollback run
        id: rollback-run
        if: failure() && steps.rollback-prepare.outputs.rollback_label != '' && steps.rollback-discard-vcs.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/create-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        env:
          TF_VAR_release_version_label: '"${{ steps.rollback-prepare.outputs.rollback_label }}"'
        with:
          workspace: shoc-backend-staging
          message: "Rollback to ${{ steps.rollback-prepare.outputs.rollback_label }} from GitHub Actions"
      - name: Read Terraform rollback plan counts
        id: rollback-plan
        if: failure() && steps.rollback-run.outcome == 'success'
        uses: hashicorp/tfc-workflows-github/actions/plan-output@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          plan: ${{ steps.rollback-run.outputs.plan_id }}

      - name: Reject non-version-only rollback counts
        id: rollback-count-guard
        if: failure() && steps.rollback-plan.outcome == 'success'
        env:
          PLAN_ADD: ${{ steps.rollback-plan.outputs.add }}
          PLAN_CHANGE: ${{ steps.rollback-plan.outputs.change }}
          PLAN_DESTROY: ${{ steps.rollback-plan.outputs.destroy }}
        run: |
          set -euo pipefail
          if [ "$PLAN_ADD" != "0" ] || [ "$PLAN_CHANGE" != "1" ] || [ "$PLAN_DESTROY" != "0" ]; then
            echo "Rollback HCP plan counts are add=${PLAN_ADD} change=${PLAN_CHANGE} destroy=${PLAN_DESTROY}; expected 0/1/0." >&2
            exit 1
          fi

      - name: Guard version-only Terraform rollback plan
        id: rollback-json-guard
        if: failure() && steps.rollback-count-guard.outcome == 'success'
        run: |
          set -euo pipefail
          python scripts/check-terraform-release-plan.py \
            --plan-id "${{ steps.rollback-run.outputs.plan_id }}" \
            --expected-version-label "${{ steps.rollback-prepare.outputs.rollback_label }}"

      - name: Discard rollback run when the guard fails
        if: failure() && steps.rollback-run.outcome == 'success' && steps.rollback-json-guard.outcome != 'success'
        uses: hashicorp/tfc-workflows-github/actions/discard-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.rollback-run.outputs.run_id }}
          comment: Rejected by the version-only rollback plan guard from GitHub Actions

      - name: Apply Terraform rollback run
        id: rollback-apply
        if: failure() && steps.rollback-json-guard.outcome == 'success'
        continue-on-error: true
        uses: hashicorp/tfc-workflows-github/actions/apply-run@8e08d1ba957673f5fbf971a22b3219639dc45661 # v1.3.2
        with:
          run: ${{ steps.rollback-run.outputs.run_id }}
          comment: Apply version-only rollback from GitHub Actions ${{ github.sha }}

      - name: Treat already-applied rollback run as success
        id: rollback-apply-result
        if: failure() && steps.rollback-apply.outcome != 'skipped'
        env:
          APPLY_OUTCOME: ${{ steps.rollback-apply.outcome }}
          RUN_ID: ${{ steps.rollback-run.outputs.run_id }}
        run: |
          set -euo pipefail
          if [ "$APPLY_OUTCOME" = "success" ]; then
            echo "Apply succeeded."
            exit 0
          fi
          python3 << 'PY'
          import json, os, urllib.request
          run_id = os.environ["RUN_ID"]
          token = os.environ["TF_API_TOKEN"]
          req = urllib.request.Request(
              f"https://app.terraform.io/api/v2/runs/{run_id}",
              headers={
                  "Authorization": f"Bearer {token}",
                  "Content-Type": "application/vnd.api+json",
              },
          )
          with urllib.request.urlopen(req) as resp:
              status = json.load(resp)["data"]["attributes"]["status"]
          print(f"HCP run {run_id} status={status}")
          if status == "applied":
              raise SystemExit(0)
          raise SystemExit(
              f"Apply failed: GitHub outcome={os.environ['APPLY_OUTCOME']} "
              f"HCP status={status}"
          )
          PY

      - name: Verify previous application version is active
        if: failure() && steps.rollback-apply-result.outcome == 'success'
        run: |
          set -euo pipefail
          prev="${{ steps.rollback-prepare.outputs.rollback_label }}"
          echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${EB_ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$prev" ]; then
                echo "Rollback reached Ready on version $current, not the previous $prev." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Application version restore complete; previous code is Ready and healthy."
                exit 0
              fi
              # Same convergence gap as the release check above: the previous version
              # is back, health has not settled yet, and reporting a failed rollback
              # here hides the fact that the restore itself worked.
              echo "Previous version is active; waiting for health to leave $health."
            fi
            sleep 15
          done
          echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2
          exit 1