shoc-backend/.github/workflows/deploy.yaml

name: Deploy API

# GitHub owns Elastic Beanstalk application versions. Terraform ignores
# version_label. Do not put path filters on tag events; those live in
# deploy-tag.yaml.

on:
  workflow_call:
    inputs:
      environment:
        required: true
        type: string
      ref:
        required: true
        type: string
  workflow_dispatch:
    inputs:
      environment:
        description: Target Environment
        required: true
        type: choice
        options: [dev, staging, prod]
      ref:
        description: Git ref to build (tag, branch, or SHA). Empty means this run's SHA.
        required: false
        type: string
        default: ""
  push:
    branches: [main]
    paths-ignore:
      - "terraform/**"
      - "**/*.md"
      - ".github/workflows/ci.yml"
      - ".github/workflows/ci-terraform.yaml"
      - ".github/workflows/architecture-quality.yml"
      - ".github/workflows/release.yaml"
      - ".github/workflows/deploy-tag.yaml"

permissions:
  contents: read

jobs:
  target:
    name: Resolve target
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      environment: ${{ steps.resolve.outputs.environment }}
      ref: ${{ steps.resolve.outputs.ref }}
    steps:
      - id: resolve
        env:
          EVENT_NAME: ${{ github.event_name }}
          CALL_ENVIRONMENT: ${{ inputs.environment }}
          CALL_REF: ${{ inputs.ref }}
          INPUT_ENVIRONMENT: ${{ github.event.inputs.environment }}
          INPUT_REF: ${{ github.event.inputs.ref }}
          GITHUB_SHA_IN: ${{ github.sha }}
        run: |
          set -euo pipefail
          case "${EVENT_NAME}" in
            workflow_call)
              environment="${CALL_ENVIRONMENT}"
              ref="${CALL_REF}"
              ;;
            workflow_dispatch)
              environment="${INPUT_ENVIRONMENT}"
              ref="${INPUT_REF:-${GITHUB_SHA_IN}}"
              ;;
            push)
              environment=dev
              ref="${GITHUB_SHA_IN}"
              ;;
            *)
              echo "unsupported event ${EVENT_NAME}" >&2
              exit 1
              ;;
          esac
          case "${environment}" in
            dev|staging|prod) ;;
            *)
              echo "unknown environment ${environment}" >&2
              exit 1
              ;;
          esac
          {
            echo "environment=${environment}"
            echo "ref=${ref}"
          } >> "${GITHUB_OUTPUT}"
          echo "Deploying ${ref} to ${environment}"

  deploy:
    name: Deploy ${{ needs.target.outputs.environment }}
    needs: target
    runs-on: ubuntu-latest
    timeout-minutes: 180
    environment: ${{ needs.target.outputs.environment }}
    concurrency:
      group: deploy-api-${{ needs.target.outputs.environment }}
      cancel-in-progress: false
    permissions:
      contents: read
      id-token: write
      checks: read
    env:
      AWS_REGION: us-east-1
      DEPLOY_ROLE_ARN: ${{ vars.DEPLOY_ROLE_ARN }}
      TARGET_ENVIRONMENT: ${{ needs.target.outputs.environment }}
      TARGET_REF: ${{ needs.target.outputs.ref }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ needs.target.outputs.ref }}
          persist-credentials: false
          fetch-depth: 0

      - name: Resolve commit
        id: commit
        run: |
          set -euo pipefail
          sha="$(git rev-parse HEAD)"
          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
          echo "Building ${sha}"

      - name: Require tag on main
        if: needs.target.outputs.environment != 'dev'
        env:
          REPO: ${{ github.repository }}
          GH_TOKEN: ${{ github.token }}
          TAG_OR_REF: ${{ needs.target.outputs.ref }}
        run: |
          set -euo pipefail
          status="$(gh api "repos/${REPO}/compare/main...${TAG_OR_REF}" --jq .status)"
          if [ "${status}" != "behind" ] && [ "${status}" != "identical" ]; then
            echo "ref ${TAG_OR_REF} is not on main (compare status: ${status})" >&2
            exit 1
          fi

      - name: Require CI on the SHA
        if: needs.target.outputs.environment != 'dev'
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          python3 scripts/require_commit_checks.py \
            --repo "${{ github.repository }}" \
            --sha "${{ steps.commit.outputs.sha }}" \
            --timeout-seconds 60

      - name: Skip prod AWS until live/prod exists
        id: prod-gate
        if: needs.target.outputs.environment == 'prod'
        run: |
          set -euo pipefail
          if [ "${{ vars.PROD_APP_CD_ENABLED }}" = "true" ]; then
            echo "skip_aws=false" >> "${GITHUB_OUTPUT}"
          else
            echo "PROD_APP_CD_ENABLED is not true; reviewers already approved; skipping AWS."
            echo "skip_aws=true" >> "${GITHUB_OUTPUT}"
          fi

      - name: Set up .NET
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
        with:
          dotnet-version: "8.0.x"

      - name: Build Elastic Beanstalk source bundle
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        run: bash scripts/package-elastic-beanstalk.sh

      - name: Validate exact release bundle
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        run: bash scripts/validate-elastic-beanstalk-bundle.sh

      - name: Configure AWS credentials using OIDC
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
        with:
          role-to-assume: ${{ env.DEPLOY_ROLE_ARN }}
          aws-region: us-east-1
          audience: sts.amazonaws.com

      - name: Get deploy parameters
        id: deploy
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        run: |
          set -euo pipefail
          prefix="/shoc-backend/${TARGET_ENVIRONMENT}/deploy"
          APPLICATION=$(aws ssm get-parameter --name "${prefix}/application-name" --query Parameter.Value --output text)
          ENVIRONMENT_NAME=$(aws ssm get-parameter --name "${prefix}/environment-name" --query Parameter.Value --output text)
          ARTIFACTS_BUCKET=$(aws ssm get-parameter --name "${prefix}/artifacts-bucket" --query Parameter.Value --output text)
          SMOKE_URL=$(aws ssm get-parameter --name "${prefix}/smoke-url" --query Parameter.Value --output text)
          {
            echo "application=${APPLICATION}"
            echo "environment_name=${ENVIRONMENT_NAME}"
            echo "artifacts_bucket=${ARTIFACTS_BUCKET}"
            echo "smoke_url=${SMOKE_URL}"
          } >> "${GITHUB_OUTPUT}"

      - name: Capture current environment version
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        env:
          ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }}
        run: |
          set -euo pipefail
          prev="$(aws elasticbeanstalk describe-environments \
            --environment-names "${ENVIRONMENT_NAME}" \
            --region us-east-1 \
            --query 'Environments[0].VersionLabel' \
            --output text)"
          echo "previous_version_label=${prev}" >> "${GITHUB_ENV}"
          echo "Previous version label: ${prev}"

      - name: Upload bundle and update environment
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        env:
          APPLICATION: ${{ steps.deploy.outputs.application }}
          ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }}
          ARTIFACTS_BUCKET: ${{ steps.deploy.outputs.artifacts_bucket }}
          GIT_SHA: ${{ steps.commit.outputs.sha }}
        run: |
          set -euo pipefail
          version_label="${GIT_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          s3_key="shoc-backend/releases/${TARGET_ENVIRONMENT}/${GIT_SHA}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/site.zip"
          aws s3 cp .artifacts/elastic-beanstalk/site.zip \
            "s3://${ARTIFACTS_BUCKET}/${s3_key}" \
            --region us-east-1
          aws elasticbeanstalk create-application-version \
            --application-name "${APPLICATION}" \
            --version-label "${version_label}" \
            --description "GitHub Actions ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
            --source-bundle "S3Bucket=${ARTIFACTS_BUCKET},S3Key=${s3_key}" \
            --process \
            --region us-east-1
          status="UNPROCESSED"
          for _ in $(seq 1 36); do
            status="$(aws elasticbeanstalk describe-application-versions \
              --application-name "${APPLICATION}" \
              --version-labels "${version_label}" \
              --region us-east-1 \
              --query 'ApplicationVersions[0].Status' \
              --output text)"
            echo "application version status: $status"
            if [ "$status" = "PROCESSED" ]; then
              break
            fi
            if [ "$status" = "FAILED" ]; then
              echo "Elastic Beanstalk failed to process ${version_label}." >&2
              exit 1
            fi
            sleep 5
          done
          if [ "$status" != "PROCESSED" ]; then
            echo "Application version did not become PROCESSED." >&2
            exit 1
          fi
          aws elasticbeanstalk update-environment \
            --environment-name "${ENVIRONMENT_NAME}" \
            --version-label "${version_label}" \
            --region us-east-1
          echo "version_label=${version_label}" >> "${GITHUB_ENV}"
          echo "environment_updated=true" >> "${GITHUB_ENV}"

      - name: Verify exact application version is active
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        env:
          ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }}
        run: |
          set -euo pipefail
          expected="${version_label}"
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$expected" ]; then
                echo "Environment became Ready on version $current, not the expected $expected." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Expected application version is Ready and healthy."
                exit 0
              fi
              echo "Expected version is active; waiting for health to leave $health."
            fi
            sleep 15
          done
          echo "Expected application version did not become Ready and healthy within the deployment window (last seen: status=$status version=$current health=$health)." >&2
          exit 1

      - name: Post-deploy smoke
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        run: bash scripts/smoke-elastic-beanstalk.sh "${{ steps.deploy.outputs.smoke_url }}"

      - name: Verify webhook secret source is operational
        if: needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true'
        env:
          SMOKE_URL: ${{ steps.deploy.outputs.smoke_url }}
        run: |
          set -euo pipefail
          response_file="$(mktemp)"
          trap 'rm -f "$response_file"' EXIT
          status="$(curl --silent --show-error \
            --output "$response_file" \
            --write-out '%{http_code}' \
            --request POST \
            --header 'Content-Type: application/json' \
            --header "X-SH-Timestamp: $(date +%s)" \
            --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
            --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
            --data '{}' \
            "${SMOKE_URL}/api/webhooks/work-orders")"
          if [ "$status" != "401" ]; then
            echo "Expected 401 from enabled webhook; received $status." >&2
            sed -n '1,20p' "$response_file" >&2
            exit 1
          fi

      - name: Restore previous application version on failure (schema is not reverted)
        if: ${{ failure() && !cancelled() && (needs.target.outputs.environment != 'prod' || steps.prod-gate.outputs.skip_aws != 'true') }}
        env:
          ENVIRONMENT_NAME: ${{ steps.deploy.outputs.environment_name }}
        run: |
          set -euo pipefail
          prev="${previous_version_label:-}"
          if [ "${environment_updated:-}" != "true" ]; then
            echo "Environment was not updated; nothing to roll back."
            exit 0
          fi
          if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
            echo "No previous version recorded; nothing to roll back." >&2
            exit 0
          fi
          if [ "$prev" = "${version_label:-}" ]; then
            echo "Previous version is the failed release; nothing to roll back." >&2
            exit 0
          fi

          echo "Waiting for any in-flight environment update to settle..."
          status="Unknown"
          current="Unknown"
          health="Unknown"
          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              break
            fi
            sleep 15
          done
          if [ "$status" != "Ready" ]; then
            echo "Environment did not settle before rollback." >&2
            exit 1
          fi
          if [ "$current" = "$prev" ]; then
            echo "Environment is already on previous version $prev."
            exit 0
          fi

          echo "Restoring previous application version $prev."
          aws elasticbeanstalk update-environment \
            --environment-name "${ENVIRONMENT_NAME}" \
            --version-label "${prev}" \
            --region us-east-1

          for _ in $(seq 1 80); do
            read -r status current health < <(
              aws elasticbeanstalk describe-environments \
                --environment-names "${ENVIRONMENT_NAME}" \
                --region us-east-1 \
                --query 'Environments[0].[Status,VersionLabel,Health]' \
                --output text
            )
            echo "environment status: $status; version: $current; health: $health"
            if [ "$status" = "Ready" ]; then
              if [ "$current" != "$prev" ]; then
                echo "Environment became Ready on version $current, not the previous $prev." >&2
                exit 1
              fi
              if [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; then
                echo "Previous application version is Ready and healthy."
                exit 0
              fi
              echo "Previous version is active; waiting for health to leave $health."
            fi
            sleep 15
          done
          echo "Environment did not return to Ready and healthy within the rollback window (last seen: status=$status version=$current health=$health)." >&2
          exit 1