diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 335b87c..0e218f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: Backend CI on: pull_request: - branches: [main, dev] + branches: [main, dev, staging] permissions: contents: read diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..50b48c1 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,255 @@ +name: Validate and deploy staging + +on: + pull_request: + branches: [staging] + push: + branches: [staging] + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: Validate deployable source bundle + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Set up .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22.22.1" + cache: npm + cache-dependency-path: infra/cdk/package-lock.json + + - name: Repository quality gate + run: bash scripts/governance-check.sh + + - name: Validate CDK deployment infrastructure + run: | + npm ci --prefix infra/cdk + npm run synth --prefix infra/cdk + + - name: Build Elastic Beanstalk source bundle + run: bash scripts/package-elastic-beanstalk.sh + + - name: Inspect source bundle contract + run: | + set -euo pipefail + unzip -t .artifacts/elastic-beanstalk/site.zip + unzip -Z1 .artifacts/elastic-beanstalk/site.zip \ + > .artifacts/elastic-beanstalk/zip-contents.txt + grep -Fxq "efbundle" .artifacts/elastic-beanstalk/zip-contents.txt + grep -Fxq ".ebextensions/01_migrations.config" \ + .artifacts/elastic-beanstalk/zip-contents.txt + grep -Fxq ".ebextensions/02_webhook_config.config" \ + .artifacts/elastic-beanstalk/zip-contents.txt + unzip -p .artifacts/elastic-beanstalk/site.zip \ + .ebextensions/02_webhook_config.config \ + > .artifacts/elastic-beanstalk/webhook-config.txt + grep -Fxq ' WorkOrderWebhook__Enabled: "true"' \ + .artifacts/elastic-beanstalk/webhook-config.txt + grep -Fxq ' WorkOrderWebhook__Region: us-east-1' \ + .artifacts/elastic-beanstalk/webhook-config.txt + grep -Fxq \ + ' WorkOrderWebhook__SecretId: arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB' \ + .artifacts/elastic-beanstalk/webhook-config.txt + + deploy: + name: Deploy shoc-backend to Elastic Beanstalk staging + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/staging') + needs: validate + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + environment: + name: staging + concurrency: + group: deploy-staging + cancel-in-progress: false + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: "8.0.x" + + - name: Build Elastic Beanstalk source bundle + run: bash scripts/package-elastic-beanstalk.sh + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: arn:aws:iam::396287094661:role/githubdeploy-shoc-backend-staging + aws-region: us-east-1 + audience: sts.amazonaws.com + + - name: Capture current environment version + run: | + set -euo pipefail + prev="$(aws elasticbeanstalk describe-environments \ + --environment-names shoc-backend-staging \ + --region us-east-1 \ + --query 'Environments[0].VersionLabel' \ + --output text)" + echo "$prev" > .artifacts/elastic-beanstalk/previous-version.txt + echo "Previous version label: $prev" + + - name: Deploy prebuilt bundle to existing environment + uses: aws-actions/aws-elasticbeanstalk-deploy@7883cdd454c162051bf6fc13389536b045149b4c # v1.0.8 + with: + aws-region: us-east-1 + application-name: shoc-backend + environment-name: shoc-backend-staging + version-label: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + deployment-package-path: .artifacts/elastic-beanstalk/site.zip + s3-bucket-name: elasticbeanstalk-us-east-1-396287094661 + create-application-if-not-exists: "false" + create-environment-if-not-exists: "false" + create-s3-bucket-if-not-exists: "false" + use-existing-application-version-if-available: "false" + wait-for-deployment: "true" + wait-for-environment-recovery: "true" + + - name: Verify exact application version is active + run: | + set -euo pipefail + expected="${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" + status="Unknown" + current="Unknown" + health="Unknown" + + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names shoc-backend-staging \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + + if [ "$status" = "Ready" ]; then + if [ "$current" = "$expected" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then + echo "Expected application version is Ready and healthy." + exit 0 + fi + echo "Environment became Ready without activating expected version $expected." >&2 + exit 1 + fi + sleep 15 + done + + echo "Expected application version did not become Ready within the deployment window." >&2 + exit 1 + + - name: Post-deploy smoke + run: bash scripts/smoke-elastic-beanstalk.sh https://api.staging.seahaven.com + + - name: Verify webhook secret source is operational + run: | + set -euo pipefail + response_file="$(mktemp)" + trap 'rm -f "$response_file"' EXIT + status="$(curl --silent --show-error \ + --output "$response_file" \ + --write-out '%{http_code}' \ + --request POST \ + --header 'Content-Type: application/json' \ + --header "X-SH-Timestamp: $(date +%s)" \ + --header 'X-SH-Key-Id: deployment-smoke-invalid-key' \ + --header "X-SH-Signature: v1=$(printf '0%.0s' {1..64})" \ + --data '{}' \ + https://api.staging.seahaven.com/api/webhooks/work-orders)" + if [ "$status" != "401" ]; then + echo "Expected enabled webhook with an operational secret source to reject the invalid probe with 401; received $status." >&2 + sed -n '1,20p' "$response_file" >&2 + exit 1 + fi + + - name: Restore previous application version on failure (schema is not reverted) + if: failure() + run: | + set -euo pipefail + prev_file=".artifacts/elastic-beanstalk/previous-version.txt" + if [ ! -f "$prev_file" ]; then + echo "No previous version captured; nothing to roll back." >&2 + exit 0 + fi + prev="$(cat "$prev_file")" + if [ -z "$prev" ] || [ "$prev" = "null" ] || [ "$prev" = "None" ] || [ "$prev" = "N/A" ]; then + echo "No previous version recorded; nothing to roll back." >&2 + exit 0 + fi + + echo "Waiting for any in-flight environment update to settle..." + status="Unknown" + current="Unknown" + health="Unknown" + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names shoc-backend-staging \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + if [ "$status" = "Ready" ]; then + break + fi + sleep 15 + done + + if [ "$status" != "Ready" ]; then + echo "Environment did not settle before rollback." >&2 + exit 1 + fi + if [ "$current" = "$prev" ]; then + echo "Environment is already on previous version $prev." + exit 0 + fi + + echo "Restoring shoc-backend-staging application code to version label: $prev" + echo "Database migrations are not reverted; deployable migrations must follow the expand/contract policy." + aws elasticbeanstalk update-environment \ + --environment-name shoc-backend-staging \ + --version-label "$prev" \ + --region us-east-1 + + echo "Waiting for previous version to become healthy..." + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names shoc-backend-staging \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + if [ "$status" = "Ready" ]; then + if [ "$current" = "$prev" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then + echo "Application version restore complete; previous code is Ready and healthy." + exit 0 + fi + echo "Rollback reached Ready in an unexpected version/health state." >&2 + exit 1 + fi + sleep 15 + done + + echo "Environment did not return to Ready within rollback window." >&2 + exit 1 diff --git a/infra/cdk/README.md b/infra/cdk/README.md index d4dab94..14cb8f0 100644 --- a/infra/cdk/README.md +++ b/infra/cdk/README.md @@ -1,8 +1,10 @@ -# shoc-backend CDK (dev deployment IAM) +# shoc-backend CDK (dev/staging deployment IAM) -This CDK v2 app owns exactly one thing in the `shoc-backend` AWS account -(`396287094661`, `us-east-1`): the **GitHub OIDC deploy role** used by the -`dev` deployment workflow in `.github/workflows/deploy.yml`. +This CDK v2 app owns the **GitHub OIDC deploy roles** used by the `dev` and +`staging` deployment workflows in `.github/workflows/deploy.yml` and +`.github/workflows/deploy-staging.yml`. The staging stack also owns one +identity policy attached to the existing staging runtime role; its deliberately +narrow scope is documented below. ## Ownership boundary (deliberate) @@ -270,3 +272,62 @@ application starts. Migrations must therefore use an expand/contract sequence: This contract preserves the usefulness of application-version recovery without misrepresenting it as a tested database downgrade. + +## Staging stack (`shoc-backend-deploy-staging`) + +`deploy-staging-stack.ts` mirrors the dev ownership boundary for staging. It is +instantiated in `app.ts` as stack `shoc-backend-deploy-staging` +(termination-protected, same account/region) and owns: + +- The IAM role `githubdeploy-shoc-backend-staging`, retained on stack deletion + (`DeletionPolicy=Retain`, `UpdateReplacePolicy=Retain`). +- Its OIDC trust to + `arn:aws:iam::396287094661:oidc-provider/token.actions.githubusercontent.com` + with exact `StringEquals` for `aud=sts.amazonaws.com` and + `sub=repo:Sea-Haven-Industries/shoc-backend:environment:staging` (no + wildcard subject). +- Its least-privilege inline policy: the same read-only discovery set as dev, + but with `elasticbeanstalk:UpdateEnvironment` scoped to environment + `shoc-backend-staging`, CloudFormation mutations scoped to the EB-managed + stack `awseb-e-6c9m4vb62z-stack`, Auto Scaling mutations scoped to the + `awseb-e-6c9m4vb62z-stack-*` ASG name prefix, and S3 limited to the existing + Elastic Beanstalk bucket namespace. It grants no dev resource, no IAM + mutation, no `PassRole`, and no RDS/Secrets Manager access to the deploy + role. + +It references — and never creates, imports as CDK constructs, or modifies — the +existing Elastic Beanstalk application `shoc-backend`, environment +`shoc-backend-staging`, and S3 bucket +`elasticbeanstalk-us-east-1-396287094661`. + +### Runtime role identity grants (staging only) + +The staging stack additionally attaches one identity policy to the existing +Elastic Beanstalk runtime role `shoc-backend-staging` (imported by exact name; +owned by Elastic Beanstalk, not by CDK): + +- `secretsmanager:GetSecretValue` and `secretsmanager:DescribeSecret` on only + `arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB`. +- `kms:Decrypt` on only + `arn:aws:kms:us-east-1:011934824531:key/d10fd1f0-a61a-4405-8568-85e9fd11ba18`, + conditioned on `kms:ViaService = secretsmanager.us-east-1.amazonaws.com` so + the key is usable only through Secrets Manager. + +The secret and KMS key live in the **external** account `011934824531`. +Cross-account access additionally requires resource policies on that secret +and key granted by the external account. This stack does **not** modify, +create, or solve those external resource policies; until the external account +grants them, the staging webhook secret access remains unresolved. No secret +values are stored or referenced anywhere in this repository. + +### Staging workflow integration + +`.github/workflows/deploy-staging.yml` validates PRs and pushes targeting +`staging`, and deploys only on push to `refs/heads/staging` or manual dispatch +on that exact ref. It uses GitHub environment `staging` and assumes the exact, +non-secret role ARN +`arn:aws:iam::396287094661:role/githubdeploy-shoc-backend-staging` (the +`GithubDeployRoleArn` output of this stack). After deploy it verifies the exact +active version label, smokes `https://api.staging.seahaven.com`, verifies an +invalid webhook probe returns 401, and restores the previous application +version on failure — the database schema is never reverted. diff --git a/infra/cdk/app.ts b/infra/cdk/app.ts index 2031dfc..b833a8b 100644 --- a/infra/cdk/app.ts +++ b/infra/cdk/app.ts @@ -1,5 +1,6 @@ import * as cdk from 'aws-cdk-lib'; import { DeployDevStack } from './deploy-dev-stack.js'; +import { DeployStagingStack } from './deploy-staging-stack.js'; const app = new cdk.App(); @@ -17,4 +18,18 @@ new DeployDevStack(app, 'shoc-backend-deploy-dev', { }, }); +new DeployStagingStack(app, 'shoc-backend-deploy-staging', { + env: { + account: '396287094661', + region: 'us-east-1', + }, + terminationProtection: true, + tags: { + Project: 'shoc-backend', + Environment: 'staging', + ManagedBy: 'cdk', + Component: 'deploy-role', + }, +}); + app.synth(); diff --git a/infra/cdk/deploy-staging-stack.ts b/infra/cdk/deploy-staging-stack.ts new file mode 100644 index 0000000..d9591d6 --- /dev/null +++ b/infra/cdk/deploy-staging-stack.ts @@ -0,0 +1,184 @@ +import * as cdk from 'aws-cdk-lib'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import { Construct } from 'constructs'; + +const ACCOUNT_ID = '396287094661'; +const REGION = 'us-east-1'; +const APPLICATION_NAME = 'shoc-backend'; +const ENVIRONMENT_NAME = 'shoc-backend-staging'; +const ENVIRONMENT_ID = 'e-6c9m4vb62z'; +const ENVIRONMENT_STACK_NAME = `awseb-${ENVIRONMENT_ID}-stack`; +const REPO = 'Sea-Haven-Industries/shoc-backend'; +const RUNTIME_ROLE_NAME = 'shoc-backend-staging'; +const EXTERNAL_WEBHOOK_SECRET_ARN = + 'arn:aws:secretsmanager:us-east-1:011934824531:secret:workorder-ingest/shoc-webhook-hmac-puYTcB'; +const EXTERNAL_WEBHOOK_KEY_ARN = + 'arn:aws:kms:us-east-1:011934824531:key/d10fd1f0-a61a-4405-8568-85e9fd11ba18'; + +export class DeployStagingStack extends cdk.Stack { + constructor(scope: Construct, id: string, props: cdk.StackProps = {}) { + super(scope, id, props); + + const applicationArn = `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:application/${APPLICATION_NAME}`; + const environmentArn = `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:environment/${APPLICATION_NAME}/${ENVIRONMENT_NAME}`; + const oidcProviderArn = `arn:aws:iam::${ACCOUNT_ID}:oidc-provider/token.actions.githubusercontent.com`; + + const deployRole = new iam.Role(this, 'GithubDeployRole', { + roleName: 'githubdeploy-shoc-backend-staging', + description: + 'Least-privilege GitHub OIDC deploy role for shoc-backend staging. CDK-owned; application/environment/S3 are owned by Elastic Beanstalk.', + assumedBy: new iam.FederatedPrincipal( + oidcProviderArn, + { + StringEquals: { + 'token.actions.githubusercontent.com:aud': 'sts.amazonaws.com', + 'token.actions.githubusercontent.com:sub': `repo:${REPO}:environment:staging`, + }, + }, + 'sts:AssumeRoleWithWebIdentity', + ), + }); + + deployRole.applyRemovalPolicy(cdk.RemovalPolicy.RETAIN); + const cfnRole = deployRole.node.defaultChild as iam.CfnRole; + cfnRole.cfnOptions.deletionPolicy = cdk.CfnDeletionPolicy.RETAIN; + cfnRole.cfnOptions.updateReplacePolicy = cdk.CfnDeletionPolicy.RETAIN; + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'autoscaling:Describe*', + 'ec2:Describe*', + 'elasticbeanstalk:DescribeEnvironments', + 'elasticbeanstalk:DescribeApplicationVersions', + 'elasticbeanstalk:DescribeEvents', + 'elasticloadbalancing:Describe*', + ], + resources: ['*'], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['elasticbeanstalk:CreateApplicationVersion'], + resources: [ + applicationArn, + `arn:aws:elasticbeanstalk:${REGION}:${ACCOUNT_ID}:applicationversion/${APPLICATION_NAME}/*`, + ], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['elasticbeanstalk:UpdateEnvironment'], + resources: [environmentArn], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'cloudformation:DescribeStackEvents', + 'cloudformation:DescribeStackResource', + 'cloudformation:GetTemplate', + 'cloudformation:DescribeStackResources', + 'cloudformation:DescribeStacks', + 'cloudformation:ListStackResources', + 'cloudformation:CancelUpdateStack', + 'cloudformation:UpdateStack', + ], + resources: [ + `arn:aws:cloudformation:${REGION}:${ACCOUNT_ID}:stack/${ENVIRONMENT_STACK_NAME}/*`, + ], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'autoscaling:PutNotificationConfiguration', + 'autoscaling:ResumeProcesses', + 'autoscaling:SuspendProcesses', + ], + resources: [ + `arn:aws:autoscaling:${REGION}:${ACCOUNT_ID}:autoScalingGroup:*:autoScalingGroupName/${ENVIRONMENT_STACK_NAME}-*`, + ], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['s3:Delete*', 's3:Get*', 's3:Put*'], + // AWS Support case 178526484500047 confirmed that UpdateEnvironment + // reads, writes, versions, ACL-checks, and removes objects in both the + // account bucket and AWS-owned Elastic Beanstalk service buckets. + resources: ['arn:aws:s3:::elasticbeanstalk-*/*'], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 's3:GetBucket*', + 's3:ListBucket', + 's3:PutBucketOwnershipControls', + 's3:PutBucketPolicy', + 's3:PutBucketPublicAccessBlock', + ], + // This is AWS Support's bucket-level UpdateEnvironment set, excluding + // CreateBucket because the workflow deploys only to an existing + // application/environment and disables bucket creation. + resources: ['arn:aws:s3:::elasticbeanstalk-*'], + }), + ); + + new cdk.CfnOutput(this, 'GithubDeployRoleArn', { + value: deployRole.roleArn, + description: 'ARN of the GitHub OIDC deploy role for shoc-backend staging.', + exportName: 'shoc-backend-deploy-staging-role-arn', + }); + + // Identity-side grants on the existing Elastic Beanstalk runtime role so + // the staging application can read the cross-account webhook secret and + // decrypt it through Secrets Manager only. The secret and KMS key live in + // the external account 011934824531; their resource policies are owned by + // that account and are NOT modified or solved by this stack. + const runtimeRole = iam.Role.fromRoleName( + this, + 'RuntimeRole', + RUNTIME_ROLE_NAME, + ); + + new iam.Policy(this, 'RuntimeRoleWebhookSecretPolicy', { + policyName: 'shoc-backend-staging-webhook-secret-access', + roles: [runtimeRole], + statements: [ + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + resources: [EXTERNAL_WEBHOOK_SECRET_ARN], + }), + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['kms:Decrypt'], + resources: [EXTERNAL_WEBHOOK_KEY_ARN], + conditions: { + StringEquals: { + 'kms:ViaService': 'secretsmanager.us-east-1.amazonaws.com', + }, + }, + }), + ], + }); + } +} diff --git a/scripts/package-elastic-beanstalk.sh b/scripts/package-elastic-beanstalk.sh index c9e81df..06f56e7 100755 --- a/scripts/package-elastic-beanstalk.sh +++ b/scripts/package-elastic-beanstalk.sh @@ -58,6 +58,11 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_NOLOGO=1 export TZ=UTC export SOURCE_DATE_EPOCH="315532800" +# EF tooling executes application startup code while creating the bundle. +# Make the target environment explicit so build hosts never fall back to +# Development configuration or development user-secrets behavior. +export ASPNETCORE_ENVIRONMENT="Production" +export DOTNET_ENVIRONMENT="Production" API_PROJECT="Api.SeaHavenIndustries/Api.SeaHavenIndustries.csproj" MIGRATIONS_PROJECT="Data.SeaHavenIndustries/Data.SeaHavenIndustries.csproj"