diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5e23077..83cf517 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -119,6 +119,38 @@ jobs: wait-for-deployment: "true" wait-for-environment-recovery: "true" + - name: Verify exact application version is active + run: | + set -euo pipefail + expected="${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" + status="Unknown" + current="Unknown" + health="Unknown" + + for _ in $(seq 1 80); do + read -r status current health < <( + aws elasticbeanstalk describe-environments \ + --environment-names shoc-backend-dev \ + --region us-east-1 \ + --query 'Environments[0].[Status,VersionLabel,Health]' \ + --output text + ) + echo "environment status: $status; version: $current; health: $health" + + if [ "$status" = "Ready" ]; then + if [ "$current" = "$expected" ] && { [ "$health" = "Green" ] || [ "$health" = "Yellow" ]; }; then + echo "Expected application version is Ready and healthy." + exit 0 + fi + echo "Environment became Ready without activating expected version $expected." >&2 + exit 1 + fi + sleep 15 + done + + echo "Expected application version did not become Ready within the deployment window." >&2 + exit 1 + - name: Post-deploy smoke run: bash scripts/smoke-elastic-beanstalk.sh https://api.dev.seahaven.com diff --git a/infra/cdk/README.md b/infra/cdk/README.md index 7555c73..93273f3 100644 --- a/infra/cdk/README.md +++ b/infra/cdk/README.md @@ -40,15 +40,18 @@ The role grants only: describe actions are not reliably constrained by resource ARN. - `elasticbeanstalk:CreateApplicationVersion` on application `shoc-backend`. - `elasticbeanstalk:UpdateEnvironment` on environment `shoc-backend-dev` only. -- `s3:ListBucket` on `elasticbeanstalk-us-east-1-396287094661` (the official - action's ownership-safe `HeadBucket` check), plus `s3:CreateBucket` on the - same bucket-level ARN. Under the `shoc-backend/` object prefix only: - `s3:PutObject` plus `s3:GetObject` and `s3:GetObjectVersion`, which the - pinned official deployment action requires to validate the - `CreateApplicationVersion` source bundle after upload. +- `s3:ListBucket` and `s3:GetBucketLocation` on + `elasticbeanstalk-us-east-1-396287094661` (the official action's + ownership-safe bucket checks), plus `s3:CreateBucket` and + `s3:PutBucketOwnershipControls` on the same bucket-level ARN. Under the + `shoc-backend/` object prefix only: + `s3:PutObject`, `s3:GetObject`, `s3:GetObjectAcl`, and + `s3:GetObjectVersion`, which the pinned official deployment action and + Elastic Beanstalk require to validate the `CreateApplicationVersion` source + bundle after upload. `s3:CreateBucket` is part of the pinned - `aws-actions/aws-elasticbeanstalk-deploy@cfad3e5e...` (v1.0.6) IAM + `aws-actions/aws-elastic-beanstalk-deploy@cfad3e5e...` (v1.0.6) IAM contract even though the workflow sets `create-s3-bucket-if-not-exists: "false"`. That input prevents the action's explicit bucket-creation helper; it does not remove the permission @@ -59,9 +62,65 @@ The role grants only: exact bucket-level ARN only (no object prefix, no wildcard resource), so it cannot create any other bucket. -It grants **no** IAM mutation or `PassRole`, **no** RDS / EC2 / Secrets Manager -access, and **no** administrator policy. There are no wildcard mutation -surfaces. + `s3:PutBucketOwnershipControls` was added after a second live deployment + (run 30375409934) failed at `UpdateEnvironment` with `AccessDenied` for + `s3:PutBucketOwnershipControls` on the same service bucket. That call is + emitted by Elastic Beanstalk's `UpdateEnvironment` path after the source + bundle upload succeeds; AWS classifies it as a bucket-level permission, so + it is scoped to the same exact bucket-level ARN (no object prefix, no + wildcard resource). It does not widen object-prefix permissions, does not + grant `PutBucketPolicy`, `PutBucketPublicAccessBlock`, or any object-level write, + and does not change `create-s3-bucket-if-not-exists: "false"`. + + `s3:GetBucketLocation` was added after CloudTrail showed that run + `30375409934` attempt 4 invoked it as + `githubdeploy-shoc-backend-dev/GitHubActions` and was denied. It is scoped to + the exact bucket-level ARN and grants no object access. + +- The six CloudFormation discovery calls observed across the failed OIDC and + successful administrator deployments (`DescribeStackEvents`, + `DescribeStackResource`, `DescribeStackResources`, `DescribeStacks`, + `GetTemplate`, and `ListStackResources`) on the Elastic Beanstalk-managed + stack `awseb-e-hehnrqjjrt-stack`, scoped to + `arn:aws:cloudformation:us-east-1:396287094661:stack/awseb-e-hehnrqjjrt-stack/*`. + These read-only calls are emitted by Elastic Beanstalk's + `UpdateEnvironment` path under the GitHub deploy role. `GetTemplate` was + added after run `30375409934` attempt 2 advanced past the S3 + ownership-controls step and was denied on the EB-managed stack instance + `awseb-e-hehnrqjjrt-stack/112f77c0-7718-11f1-a1a9-0e48750aef13`. + CloudTrail then showed attempt 4 denied `DescribeStackResources` and + `ListStackResources` on that same stack instance. + CloudFormation stack ARNs carry a random GUID instance suffix, so the + permission is scoped to that one stack-name prefix (`/*`) rather than a + single instance ARN. The statement grants no CloudFormation mutation, no + `Resource: "*"`, and no access to any other stack. CDK does not own or + mutate that stack; it is owned by Elastic Beanstalk and referenced by + identifier only. + +- `ec2:DescribeAvailabilityZones`, `ec2:DescribeImages`, and + `ec2:DescribeSubnets` as read-only account-level discovery queries. + CloudTrail identified the GitHub deploy role as the caller during run + `30375409934`; attempt 5 confirmed the first two denials after + `DescribeSubnets` was allowed. EC2 does not support resource-level + constraints for these Describe actions, so IAM requires `Resource: "*"`. + No EC2 mutation action is granted. +- The Auto Scaling discovery calls `DescribeAutoScalingGroups` and + `DescribeScalingActivities` on `Resource: "*"` plus + `PutNotificationConfiguration`, `ResumeProcesses`, and `SuspendProcesses` + on only Auto Scaling groups whose name starts with + `awseb-e-hehnrqjjrt-stack-`. These are the exact calls recorded during the + successful administrator deployment. AWS supports resource-level + constraints for all three mutations, so replacement ASGs remain covered + without granting access to another environment. +- `s3:GetObjectAcl` under the existing + `elasticbeanstalk-us-east-1-396287094661/shoc-backend/*` object prefix. + Elastic Beanstalk emitted this read during the same attempt while validating + the uploaded application bundle. It grants no bucket-wide or cross-prefix + object access. +It grants **no** IAM mutation or `PassRole`, **no** RDS / Secrets Manager +access, no EC2 mutation, and **no** administrator policy. The only non-EB/S3 +mutations are the three deployment-process Auto Scaling calls, restricted to +this environment's ASG name pattern. There are no wildcard mutation surfaces. ## Prerequisites @@ -92,11 +151,29 @@ All commands run from `infra/cdk/`. - Trust policy `StringEquals` matches the exact audience and subject above. - The inline policy contains no `Resource: "*"` mutation action and no service - outside `elasticbeanstalk` / `s3`. + outside `elasticbeanstalk` / `s3` / `cloudformation` / `ec2` / + `autoscaling`. CloudFormation discovery is limited to the single EB-managed + stack prefix. EC2 and Auto Scaling discovery use `Resource: "*"` only where + the IAM resource model requires it; Auto Scaling mutations are limited to + this environment's ASG name pattern. The workflow's `AWS_DEPLOY_ROLE_ARN` repository secret (environment `dev`) must hold the ARN output by this stack (`GithubDeployRoleArn`). +The OIDC deployment is currently fail-closed, not repaired. Elastic Beanstalk +still reports a generic `s3:GetObjectAcl` denial after the account-owned source +prefix and every exact cross-account object referenced by the sanitized live +stack were tested independently. The runtime-prefix, platform-assets, and +launch-control hypotheses were disproved and are intentionally absent from the +policy. Do not broaden `GetObjectAcl` without an exact principal/action/resource +record from AWS Support or the AWS-owned bucket's diagnostic owner. + +The pinned deployment action can return success after Elastic Beanstalk emits a +fatal deployment event. The following workflow step therefore verifies that +the exact immutable version label is active and healthy before smoke testing. +Any mismatch fails and invokes rollback. This guard prevents false success; it +does not make the unresolved OIDC deployment path release-ready. + The GitHub `dev` environment is an external release control and must restrict deployments to the `dev` branch. Required reviewers should be configured when the repository plan supports environment reviewers. The workflow also checks diff --git a/infra/cdk/deploy-dev-stack.ts b/infra/cdk/deploy-dev-stack.ts index b1e188c..3575452 100644 --- a/infra/cdk/deploy-dev-stack.ts +++ b/infra/cdk/deploy-dev-stack.ts @@ -6,6 +6,8 @@ const ACCOUNT_ID = '396287094661'; const REGION = 'us-east-1'; const APPLICATION_NAME = 'shoc-backend'; const ENVIRONMENT_NAME = 'shoc-backend-dev'; +const ENVIRONMENT_ID = 'e-hehnrqjjrt'; +const ENVIRONMENT_STACK_NAME = `awseb-${ENVIRONMENT_ID}-stack`; const REPO = 'Sea-Haven-Industries/shoc-backend'; const BUCKET_NAME = `elasticbeanstalk-${REGION}-${ACCOUNT_ID}`; @@ -73,7 +75,66 @@ export class DeployDevStack extends cdk.Stack { deployRole.addToPolicy( new iam.PolicyStatement({ effect: iam.Effect.ALLOW, - actions: ['s3:ListBucket', 's3:CreateBucket'], + actions: [ + 'cloudformation:DescribeStackEvents', + 'cloudformation:DescribeStackResource', + 'cloudformation:GetTemplate', + 'cloudformation:DescribeStackResources', + 'cloudformation:DescribeStacks', + 'cloudformation:ListStackResources', + ], + resources: [ + `arn:aws:cloudformation:${REGION}:${ACCOUNT_ID}:stack/${ENVIRONMENT_STACK_NAME}/*`, + ], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ec2:DescribeAvailabilityZones', + 'ec2:DescribeImages', + 'ec2:DescribeSubnets', + ], + resources: ['*'], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'autoscaling:DescribeAutoScalingGroups', + 'autoscaling:DescribeScalingActivities', + ], + resources: ['*'], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'autoscaling:PutNotificationConfiguration', + 'autoscaling:ResumeProcesses', + 'autoscaling:SuspendProcesses', + ], + resources: [ + `arn:aws:autoscaling:${REGION}:${ACCOUNT_ID}:autoScalingGroup:*:autoScalingGroupName/${ENVIRONMENT_STACK_NAME}-*`, + ], + }), + ); + + deployRole.addToPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 's3:ListBucket', + 's3:CreateBucket', + 's3:PutBucketOwnershipControls', + 's3:GetBucketLocation', + ], resources: [bucketArn], }), ); @@ -81,7 +142,7 @@ export class DeployDevStack extends cdk.Stack { deployRole.addToPolicy( new iam.PolicyStatement({ effect: iam.Effect.ALLOW, - actions: ['s3:GetObject', 's3:GetObjectVersion', 's3:PutObject'], + actions: ['s3:GetObject', 's3:GetObjectAcl', 's3:GetObjectVersion', 's3:PutObject'], resources: [`${bucketArn}/${APPLICATION_NAME}/*`], }), );