ci: add protected staging deployment lane

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: Backend CI
 
 on:
   pull_request:
-    branches: [main, dev]
+    branches: [main, dev, staging]
 
 permissions:
   contents: read

.github/workflows/deploy-staging.yml

@@ -0,0 +1,255 @@
+name: Validate and deploy staging
+
+on:
+  pull_request:
+    branches: [staging]
+  push:
+    branches: [staging]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: Validate deployable source bundle
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+        with:
+          dotnet-version: "8.0.x"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22.22.1"
+          cache: npm
+          cache-dependency-path: infra/cdk/package-lock.json
+
+      - name: Repository quality gate
+        run: bash scripts/governance-check.sh
+
+      - name: Validate CDK deployment infrastructure
+        run: |
+          npm ci --prefix infra/cdk
+          npm run synth --prefix infra/cdk
+
+      - name: Build Elastic Beanstalk source bundle
+        run: bash scripts/package-elastic-beanstalk.sh
+
+      - name: Inspect source bundle contract
+        run: |
+          set -euo pipefail
+          unzip -t .artifacts/elastic-beanstalk/site.zip
+          unzip -Z1 .artifacts/elastic-beanstalk/site.zip \
+            > .artifacts/elastic-beanstalk/zip-contents.txt
+          grep -Fxq "efbundle" .artifacts/elastic-beanstalk/zip-contents.txt
+          grep -Fxq ".ebextensions/01_migrations.config" \
+            .artifacts/elastic-beanstalk/zip-contents.txt
+          grep -Fxq ".ebextensions/02_webhook_config.config" \
+            .artifacts/elastic-beanstalk/zip-contents.txt
+          unzip -p .artifacts/elastic-beanstalk/site.zip \
+            .ebextensions/02_webhook_config.config \
+            > .artifacts/elastic-beanstalk/webhook-config.txt
+          grep -Fxq '    WorkOrderWebhook__Enabled: "true"' \
+            .artifacts/elastic-beanstalk/webhook-config.txt
+          grep -Fxq '    WorkOrderWebhook__Region: us-east-1' \
+            .artifacts/elastic-beanstalk/webhook-config.txt
+          grep -Fxq \
+            '    WorkOrderWebhook__SecretId: arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB' \
+            .artifacts/elastic-beanstalk/webhook-config.txt
+
+  deploy:
+    name: Deploy shoc-backend to Elastic Beanstalk staging
+    if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging')
+    needs: validate
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    environment:
+      name: staging
+    concurrency:
+      group: deploy-staging
+      cancel-in-progress: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+        with:
+          dotnet-version: "8.0.x"
+
+      - name: Build Elastic Beanstalk source bundle
+        run: bash scripts/package-elastic-beanstalk.sh
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          role-to-assume: arn:aws:iam::396287094661:role/githubdeploy-shoc-backend-staging
+          aws-region: us-east-1
+          audience: sts.amazonaws.com
+
+      - name: Capture current environment version
+        run: |
+          set -euo pipefail
+          prev="$(aws elasticbeanstalk describe-environments \
+            --environment-names shoc-backend-staging \
+            --region us-east-1 \
+            --query 'Environments[0].VersionLabel' \
+            --output text)"
+          echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt
+          echo "Previous version label: $prev"
+
+      - name: Deploy prebuilt bundle to existing environment
+        uses: aws-actions/aws-elasticbeanstalk-deploy@7883cdd454c162051bf6fc13389536b045149b4c # v1.0.8
+        with:
+          aws-region: us-east-1
+          application-name: shoc-backend
+          environment-name: shoc-backend-staging
+          version-label: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          deployment-package-path: .artifacts/elastic-beanstalk/site.zip
+          s3-bucket-name: elasticbeanstalk-us-east-1-396287094661
+          create-application-if-not-exists: "false"
+          create-environment-if-not-exists: "false"
+          create-s3-bucket-if-not-exists: "false"
+          use-existing-application-version-if-available: "false"
+          wait-for-deployment: "true"
+          wait-for-environment-recovery: "true"
+
+      - name: Verify exact application version is active
+        run: |
+          set -euo pipefail
+          expected="${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}"
+          status="Unknown"
+          current="Unknown"
+          health="Unknown"
+
+          for _ in $(seq 1 80); do
+            read -r status current health < <(
+              aws elasticbeanstalk describe-environments \
+                --environment-names shoc-backend-staging \
+                --region us-east-1 \
+                --query 'Environments[0].[Status,VersionLabel,Health]' \
+                --output text
+            )
+            echo "environment status: $status; version: $current; health: $health"
+
+            if [ "$status" = "Ready" ]; then
+              if [ "$current" = "$expected" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then
+                echo "Expected application version is Ready and healthy."
+                exit 0
+              fi
+              echo "Environment became Ready without activating expected version $expected." >&2
+              exit 1
+            fi
+            sleep 15
+          done
+
+          echo "Expected application version did not become Ready within the deployment window." >&2
+          exit 1
+
+      - name: Post-deploy smoke
+        run: bash scripts/smoke-elastic-beanstalk.sh https://api.staging.seahaven.com
+
+      - name: Verify webhook secret source is operational
+        run: |
+          set -euo pipefail
+          response_file="$(mktemp)"
+          trap 'rm -f "$response_file"' EXIT
+          status="$(curl --silent --show-error \
+            --output "$response_file" \
+            --write-out '%{http_code}' \
+            --request POST \
+            --header 'Content-Type: application/json' \
+            --header "X-SH-Timestamp: $(date +%s)" \
+            --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \
+            --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \
+            --data '{}' \
+            https://api.staging.seahaven.com/api/webhooks/work-orders)"
+          if [ "$status" != "401" ]; then
+            echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2
+            sed -n '1,20p' "$response_file" >&2
+            exit 1
+          fi
+
+      - name: Restore previous application version on failure (schema is not reverted)
+        if: failure()
+        run: |
+          set -euo pipefail
+          prev_file=".artifacts/elastic-beanstalk/previous-version.txt"
+          if [ ! -f "$prev_file" ]; then
+            echo "No previous version captured; nothing to roll back." >&2
+            exit 0
+          fi
+          prev="$(cat "$prev_file")"
+          if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then
+            echo "No previous version recorded; nothing to roll back." >&2
+            exit 0
+          fi
+
+          echo "Waiting for any in-flight environment update to settle..."
+          status="Unknown"
+          current="Unknown"
+          health="Unknown"
+          for _ in $(seq 1 80); do
+            read -r status current health < <(
+              aws elasticbeanstalk describe-environments \
+                --environment-names shoc-backend-staging \
+                --region us-east-1 \
+                --query 'Environments[0].[Status,VersionLabel,Health]' \
+                --output text
+            )
+            echo "environment status: $status; version: $current; health: $health"
+            if [ "$status" = "Ready" ]; then
+              break
+            fi
+            sleep 15
+          done
+
+          if [ "$status" != "Ready" ]; then
+            echo "Environment did not settle before rollback." >&2
+            exit 1
+          fi
+          if [ "$current" = "$prev" ]; then
+            echo "Environment is already on previous version $prev."
+            exit 0
+          fi
+
+          echo "Restoring shoc-backend-staging application code to version label: $prev"
+          echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy."
+          aws elasticbeanstalk update-environment \
+            --environment-name shoc-backend-staging \
+            --version-label "$prev" \
+            --region us-east-1
+
+          echo "Waiting for previous version to become healthy..."
+          for _ in $(seq 1 80); do
+            read -r status current health < <(
+              aws elasticbeanstalk describe-environments \
+                --environment-names shoc-backend-staging \
+                --region us-east-1 \
+                --query 'Environments[0].[Status,VersionLabel,Health]' \
+                --output text
+            )
+            echo "environment status: $status; version: $current; health: $health"
+            if [ "$status" = "Ready" ]; then
+              if [ "$current" = "$prev" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then
+                echo "Application version restore complete; previous code is Ready and healthy."
+                exit 0
+              fi
+              echo "Rollback reached Ready in an unexpected version/health state." >&2
+              exit 1
+            fi
+            sleep 15
+          done
+
+          echo "Environment did not return to Ready within rollback window." >&2
+          exit 1

infra/cdk/README.md

@@ -1,8 +1,10 @@
-# shoc-backend CDK (dev deployment IAM)
+# shoc-backend CDK (dev/staging deployment IAM)
 
-This CDK v2 app owns exactly one thing in the `shoc-backend` AWS account
+This CDK v2 app owns the **GitHub OIDC deploy roles** used by the `dev` and
-(`396287094661`, `us-east-1`): the **GitHub OIDC deploy role** used by the
+`staging` deployment workflows in `.github/workflows/deploy.yml` and
-`dev` deployment workflow in `.github/workflows/deploy.yml`.
+`.github/workflows/deploy-staging.yml`. The staging stack also owns one
+identity policy attached to the existing staging runtime role; its deliberately
+narrow scope is documented below.
 
 ## Ownership boundary (deliberate)
 
@@ -270,3 +272,62 @@ application starts. Migrations must therefore use an expand/contract sequence:
 
 This contract preserves the usefulness of application-version recovery without
 misrepresenting it as a tested database downgrade.
+
+## Staging stack (`shoc-backend-deploy-staging`)
+
+`deploy-staging-stack.ts` mirrors the dev ownership boundary for staging. It is
+instantiated in `app.ts` as stack `shoc-backend-deploy-staging`
+(termination-protected, same account/region) and owns:
+
+- The IAM role `githubdeploy-shoc-backend-staging`, retained on stack deletion
+  (`DeletionPolicy=Retain`, `UpdateReplacePolicy=Retain`).
+- Its OIDC trust to
+  `arn:aws:iam::396287094661:oidc-provider/token.actions.githubusercontent.com`
+  with exact `StringEquals` for `aud=sts.amazonaws.com` and
+  `sub=repo:Sea-Haven-Industries/shoc-backend:environment:staging` (no
+  wildcard subject).
+- Its least-privilege inline policy: the same read-only discovery set as dev,
+  but with `elasticbeanstalk:UpdateEnvironment` scoped to environment
+  `shoc-backend-staging`, CloudFormation mutations scoped to the EB-managed
+  stack `awseb-e-6c9m4vb62z-stack`, Auto Scaling mutations scoped to the
+  `awseb-e-6c9m4vb62z-stack-*` ASG name prefix, and S3 limited to the existing
+  Elastic Beanstalk bucket namespace. It grants no dev resource, no IAM
+  mutation, no `PassRole`, and no RDS/Secrets Manager access to the deploy
+  role.
+
+It references — and never creates, imports as CDK constructs, or modifies — the
+existing Elastic Beanstalk application `shoc-backend`, environment
+`shoc-backend-staging`, and S3 bucket
+`elasticbeanstalk-us-east-1-396287094661`.
+
+### Runtime role identity grants (staging only)
+
+The staging stack additionally attaches one identity policy to the existing
+Elastic Beanstalk runtime role `shoc-backend-staging` (imported by exact name;
+owned by Elastic Beanstalk, not by CDK):
+
+- `secretsmanager:GetSecretValue` and `secretsmanager:DescribeSecret` on only
+  `arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB`.
+- `kms:Decrypt` on only
+  `arn:aws:kms:us-east-1:011934824531:key/d10fd1f0-a61a-4405-8568-85e9fd11ba18`,
+  conditioned on `kms:ViaService = secretsmanager.us-east-1.amazonaws.com` so
+  the key is usable only through Secrets Manager.
+
+The secret and KMS key live in the **external** account `011934824531`.
+Cross-account access additionally requires resource policies on that secret
+and key granted by the external account. This stack does **not** modify,
+create, or solve those external resource policies; until the external account
+grants them, the staging webhook secret access remains unresolved. No secret
+values are stored or referenced anywhere in this repository.
+
+### Staging workflow integration
+
+`.github/workflows/deploy-staging.yml` validates PRs and pushes targeting
+`staging`, and deploys only on push to `refs/heads/staging` or manual dispatch
+on that exact ref. It uses GitHub environment `staging` and assumes the exact,
+non-secret role ARN
+`arn:aws:iam::396287094661:role/githubdeploy-shoc-backend-staging` (the
+`GithubDeployRoleArn` output of this stack). After deploy it verifies the exact
+active version label, smokes `https://api.staging.seahaven.com`, verifies an
+invalid webhook probe returns 401, and restores the previous application
+version on failure — the database schema is never reverted.

infra/cdk/app.ts

@@ -1,5 +1,6 @@
 import * as cdk from 'aws-cdk-lib';
 import { DeployDevStack } from './deploy-dev-stack.js';
+import { DeployStagingStack } from './deploy-staging-stack.js';
 
 const app = new cdk.App();
 
@@ -17,4 +18,18 @@ new DeployDevStack(app, 'shoc-backend-deploy-dev', {
   },
 });
 
+new DeployStagingStack(app, 'shoc-backend-deploy-staging', {
+  env: {
+    account: '396287094661',
+    region: 'us-east-1',
+  },
+  terminationProtection: true,
+  tags: {
+    Project: 'shoc-backend',
+    Environment: 'staging',
+    ManagedBy: 'cdk',
+    Component: 'deploy-role',
+  },
+});
+
 app.synth();

infra/cdk/deploy-staging-stack.ts

@@ -0,0 +1,184 @@
+import * as cdk from 'aws-cdk-lib';
+import * as iam from 'aws-cdk-lib/aws-iam';
+import { Construct } from 'constructs';
+
+const ACCOUNT_ID = '396287094661';
+const REGION = 'us-east-1';
+const APPLICATION_NAME = 'shoc-backend';
+const ENVIRONMENT_NAME = 'shoc-backend-staging';
+const ENVIRONMENT_ID = 'e-6c9m4vb62z';
+const ENVIRONMENT_STACK_NAME = `awseb-${ENVIRONMENT_ID}-stack`;
+const REPO = 'Sea-Haven-Industries/shoc-backend';
+const RUNTIME_ROLE_NAME = 'shoc-backend-staging';
+const EXTERNAL_WEBHOOK_SECRET_ARN =
+  'arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB';
+const EXTERNAL_WEBHOOK_KEY_ARN =
+  'arn:aws:kms:us-east-1:011934824531:key/d10fd1f0-a61a-4405-8568-85e9fd11ba18';
+
+export class DeployStagingStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: cdk.StackProps = {}) {
+    super(scope, id, props);
+
+    const applicationArn = `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:application/${APPLICATION_NAME}`;
+    const environmentArn = `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:environment/${APPLICATION_NAME}/${ENVIRONMENT_NAME}`;
+    const oidcProviderArn = `arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com`;
+
+    const deployRole = new iam.Role(this, 'GithubDeployRole', {
+      roleName: 'githubdeploy-shoc-backend-staging',
+      description:
+        'Least-privilege GitHub OIDC deploy role for shoc-backend staging. CDK-owned; application/environment/S3 are owned by Elastic Beanstalk.',
+      assumedBy: new iam.FederatedPrincipal(
+        oidcProviderArn,
+        {
+          StringEquals: {
+            'token.actions.githubusercontent.com:aud': 'sts.amazonaws.com',
+            'token.actions.githubusercontent.com:sub': `repo:${REPO}:environment:staging`,
+          },
+        },
+        'sts:AssumeRoleWithWebIdentity',
+      ),
+    });
+
+    deployRole.applyRemovalPolicy(cdk.RemovalPolicy.RETAIN);
+    const cfnRole = deployRole.node.defaultChild as iam.CfnRole;
+    cfnRole.cfnOptions.deletionPolicy = cdk.CfnDeletionPolicy.RETAIN;
+    cfnRole.cfnOptions.updateReplacePolicy = cdk.CfnDeletionPolicy.RETAIN;
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: [
+          'autoscaling:Describe*',
+          'ec2:Describe*',
+          'elasticbeanstalk:DescribeEnvironments',
+          'elasticbeanstalk:DescribeApplicationVersions',
+          'elasticbeanstalk:DescribeEvents',
+          'elasticloadbalancing:Describe*',
+        ],
+        resources: ['*'],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ['elasticbeanstalk:CreateApplicationVersion'],
+        resources: [
+          applicationArn,
+          `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:applicationversion/${APPLICATION_NAME}/*`,
+        ],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ['elasticbeanstalk:UpdateEnvironment'],
+        resources: [environmentArn],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: [
+          'cloudformation:DescribeStackEvents',
+          'cloudformation:DescribeStackResource',
+          'cloudformation:GetTemplate',
+          'cloudformation:DescribeStackResources',
+          'cloudformation:DescribeStacks',
+          'cloudformation:ListStackResources',
+          'cloudformation:CancelUpdateStack',
+          'cloudformation:UpdateStack',
+        ],
+        resources: [
+          `arn:aws:cloudformation:${REGION}:${ACCOUNT_ID}:stack/${ENVIRONMENT_STACK_NAME}/*`,
+        ],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: [
+          'autoscaling:PutNotificationConfiguration',
+          'autoscaling:ResumeProcesses',
+          'autoscaling:SuspendProcesses',
+        ],
+        resources: [
+          `arn:aws:autoscaling:${REGION}:${ACCOUNT_ID}:autoScalingGroup:*:autoScalingGroupName/${ENVIRONMENT_STACK_NAME}-*`,
+        ],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: ['s3:Delete*', 's3:Get*', 's3:Put*'],
+        // AWS Support case 178526484500047 confirmed that UpdateEnvironment
+        // reads, writes, versions, ACL-checks, and removes objects in both the
+        // account bucket and AWS-owned Elastic Beanstalk service buckets.
+        resources: ['arn:aws:s3:::elasticbeanstalk-*/*'],
+      }),
+    );
+
+    deployRole.addToPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        actions: [
+          's3:GetBucket*',
+          's3:ListBucket',
+          's3:PutBucketOwnershipControls',
+          's3:PutBucketPolicy',
+          's3:PutBucketPublicAccessBlock',
+        ],
+        // This is AWS Support's bucket-level UpdateEnvironment set, excluding
+        // CreateBucket because the workflow deploys only to an existing
+        // application/environment and disables bucket creation.
+        resources: ['arn:aws:s3:::elasticbeanstalk-*'],
+      }),
+    );
+
+    new cdk.CfnOutput(this, 'GithubDeployRoleArn', {
+      value: deployRole.roleArn,
+      description: 'ARN of the GitHub OIDC deploy role for shoc-backend staging.',
+      exportName: 'shoc-backend-deploy-staging-role-arn',
+    });
+
+    // Identity-side grants on the existing Elastic Beanstalk runtime role so
+    // the staging application can read the cross-account webhook secret and
+    // decrypt it through Secrets Manager only. The secret and KMS key live in
+    // the external account 011934824531; their resource policies are owned by
+    // that account and are NOT modified or solved by this stack.
+    const runtimeRole = iam.Role.fromRoleName(
+      this,
+      'RuntimeRole',
+      RUNTIME_ROLE_NAME,
+    );
+
+    new iam.Policy(this, 'RuntimeRoleWebhookSecretPolicy', {
+      policyName: 'shoc-backend-staging-webhook-secret-access',
+      roles: [runtimeRole],
+      statements: [
+        new iam.PolicyStatement({
+          effect: iam.Effect.ALLOW,
+          actions: [
+            'secretsmanager:GetSecretValue',
+            'secretsmanager:DescribeSecret',
+          ],
+          resources: [EXTERNAL_WEBHOOK_SECRET_ARN],
+        }),
+        new iam.PolicyStatement({
+          effect: iam.Effect.ALLOW,
+          actions: ['kms:Decrypt'],
+          resources: [EXTERNAL_WEBHOOK_KEY_ARN],
+          conditions: {
+            StringEquals: {
+              'kms:ViaService': 'secretsmanager.us-east-1.amazonaws.com',
+            },
+          },
+        }),
+      ],
+    });
+  }
+}

scripts/package-elastic-beanstalk.sh

@@ -58,6 +58,11 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1
 export DOTNET_NOLOGO=1
 export TZ=UTC
 export SOURCE_DATE_EPOCH="315532800"
+# EF tooling executes application startup code while creating the bundle.
+# Make the target environment explicit so build hosts never fall back to
+# Development configuration or development user-secrets behavior.
+export ASPNETCORE_ENVIRONMENT="Production"
+export DOTNET_ENVIRONMENT="Production"
 
 API_PROJECT="Api.SeaHavenIndustries/Api.SeaHavenIndustries.csproj"
 MIGRATIONS_PROJECT="Data.SeaHavenIndustries/Data.SeaHavenIndustries.csproj"