From af5e2183e05363f5dd89148e47673c323cd27a51 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Tue, 15 Sep 2026 19:19:25 -0400 Subject: [PATCH] feat(infra): migrate afterhours to HCP Terraform (PLAT-74) Replace the mgmt SAM stack with a prod-only HCP workspace, in-repo hcptf IAM, stub Lambdas, and zip CD on push to main. --- .github/workflows/ci.yaml | 85 +- .github/workflows/deploy.yaml | 208 ++-- .gitignore | 5 +- README.md | 82 +- SETUP.md | 101 +- samconfig.toml.example | 9 - scripts/cutover/copy_dynamodb.py | 80 ++ scripts/cutover/recreate_holiday_schedules.py | 114 ++ scripts/package_lambdas.py | 140 +++ src/shared/shared/sentry_init.py | 30 +- template.yaml | 1073 ----------------- terraform/.terraform.lock.hcl | 47 + terraform/alarms.tf | 155 +++ terraform/apigateway.tf | 82 ++ terraform/artifacts.tf | 118 ++ terraform/bootstrap/stub/handler.py | 9 + terraform/dynamodb.tf | 21 + terraform/events.tf | 76 ++ terraform/hcp_iam.tf | 843 +++++++++++++ terraform/iam_github_deploy.tf | 101 ++ terraform/lambda.tf | 281 +++++ terraform/lambda_boundary.tf | 141 +++ terraform/locals.tf | 87 ++ terraform/logs.tf | 11 + terraform/outputs.tf | 39 + terraform/providers.tf | 12 + terraform/scheduler.tf | 50 + terraform/secrets.tf | 15 + terraform/ssm.tf | 15 + terraform/variables.tf | 66 + terraform/versions.tf | 22 + tests/infra/test_hcp_contract.py | 92 ++ tests/requirements.txt | 5 +- tests/scripts/test_package_lambdas.py | 54 + tests/shared/test_sentry_init.py | 14 + tests/weekly_post/test_handler.py | 4 +- 36 files changed, 3015 insertions(+), 1272 deletions(-) delete mode 100644 samconfig.toml.example create mode 100644 scripts/cutover/copy_dynamodb.py create mode 100644 scripts/cutover/recreate_holiday_schedules.py create mode 100644 scripts/package_lambdas.py delete mode 100644 template.yaml create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/alarms.tf create mode 100644 terraform/apigateway.tf create mode 100644 terraform/artifacts.tf create mode 100644 terraform/bootstrap/stub/handler.py create mode 100644 terraform/dynamodb.tf create mode 100644 terraform/events.tf create mode 100644 terraform/hcp_iam.tf create mode 100644 terraform/iam_github_deploy.tf create mode 100644 terraform/lambda.tf create mode 100644 terraform/lambda_boundary.tf create mode 100644 terraform/locals.tf create mode 100644 terraform/logs.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/scheduler.tf create mode 100644 terraform/secrets.tf create mode 100644 terraform/ssm.tf create mode 100644 terraform/variables.tf create mode 100644 terraform/versions.tf create mode 100644 tests/infra/test_hcp_contract.py create mode 100644 tests/scripts/test_package_lambdas.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2832ca..c343804 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,5 @@ name: CI + on: pull_request: branches: [main] @@ -8,8 +9,84 @@ permissions: contents: read jobs: + pytest: + name: Pytest + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install test dependencies + run: | + set -euo pipefail + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + pip install -r src/slack-bot/requirements.txt + pip install -r src/weekly-post/requirements.txt + pip install -r src/shared/requirements.txt + + - name: Pytest + run: pytest + + terraform: + name: Terraform + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: terraform + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: "1.16.0" + terraform_wrapper: false + + - name: Terraform fmt + run: terraform fmt -check -recursive + + - name: Terraform init + run: terraform init -backend=false + + - name: Terraform validate + run: terraform validate + ci: - uses: Sea-Haven-Industries/.github/.github/workflows/ci-python-sam.yaml@4a6cbfd362140a68810f0f46d338026863b8e827 # v1.0.10 - with: - source-dirs: "src/slack-bot src/weekly-post src/roster-sync src/roster-api src/ring-scheduler src/shared/shared tests" - run-tests: true + name: ci / ci + needs: [pytest, terraform] + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check jobs + env: + PYTEST_RESULT: ${{ needs.pytest.result }} + TERRAFORM_RESULT: ${{ needs.terraform.result }} + run: | + set -euo pipefail + fail=0 + check() { + local name="$1" + local result="$2" + case "${result}" in + success) + echo "${name}: ${result}" + ;; + *) + echo "${name}: ${result}" >&2 + fail=1 + ;; + esac + } + check pytest "${PYTEST_RESULT}" + check terraform "${TERRAFORM_RESULT}" + exit "${fail}" diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index e5525b3..37db4d5 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,120 +1,150 @@ name: Deploy + +# Terraform owns Lambda skeletons. This workflow ships zips to prod and calls +# update-function-code. It never creates an HCP run. No GitHub Releases and no +# tagging in this workflow. + on: push: branches: [main] + paths-ignore: + - "terraform/**" + - "docs/**" + - "README.md" + - "SETUP.md" + - "AGENTS.md" + workflow_dispatch: + inputs: + ref: + description: "Git ref to build and deploy (tag, branch, or SHA). Empty means the workflow ref." + required: false + type: string + default: "" permissions: - id-token: write contents: read -concurrency: - group: deploy - cancel-in-progress: false - jobs: deploy: - uses: Sea-Haven-Industries/.github/.github/workflows/cd-sam.yaml@4a6cbfd362140a68810f0f46d338026863b8e827 # v1.0.10 - with: - stack-name: afterhours-shift-manager - cfn-role-arn: arn:aws:iam::328440206208:role/github-cfn-execution-role - secrets: - deploy-role-arn: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} - parameter-overrides: ${{ secrets.SAM_PARAMETER_OVERRIDES }} - - # Tag + announce a release once the deploy succeeds. This lives in the deploy - # workflow (gated on `needs: deploy`) rather than a separate workflow_run- - # triggered job on purpose: a push-to-main run is a trusted context, so - # checking out and running repo code with write/OIDC is safe here — unlike - # workflow_run, which CodeQL (rightly) flags for untrusted checkout + cache - # poisoning. Gating on `needs: deploy` still guarantees we never announce a - # version that isn't live, and the `deploy` concurrency group serializes - # releases. When the top CHANGELOG version already has a Release, this no-ops. - release: - needs: deploy + name: Deploy to prod runs-on: ubuntu-latest + timeout-minutes: 30 + environment: prod + concurrency: + group: deploy-afterhours-prod + cancel-in-progress: false permissions: - contents: write # create the tag + GitHub Release - id-token: write # OIDC to assume the notifier-invoke role + contents: read + id-token: write + env: + AWS_REGION: us-east-1 + DEPLOY_ROLE_ARN: ${{ vars.DEPLOY_ROLE_ARN }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 - fetch-tags: true + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + persist-credentials: false + + - name: Resolve commit + id: commit + run: | + set -euo pipefail + sha="$(git rev-parse HEAD)" + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + echo "Building ${sha}" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - - name: Determine release - id: rel + - name: Build function zips env: - GH_TOKEN: ${{ github.token }} + GIT_SHA: ${{ steps.commit.outputs.sha }} run: | - TOP=$(python scripts/changelog_cli.py top-version CHANGELOG.md) - if [ -z "$TOP" ]; then - echo "No version entry in CHANGELOG.md — nothing to release." - echo "release=false" >> "$GITHUB_OUTPUT"; exit 0 - fi - PREV=$(git tag -l 'v*' --sort=-v:refname | head -1) - PREV="${PREV:-v0.0.0}" - KIND=$(python scripts/changelog_cli.py bump-kind CHANGELOG.md "$PREV") + set -euo pipefail + python scripts/package_lambdas.py --git-sha "${GIT_SHA}" --out-dir build/packages + python - <<'PY' + import os, zipfile + from pathlib import Path + sha = os.environ["GIT_SHA"] + names = [ + "slack_bot", + "weekly_post", + "roster_sync", + "roster_api", + "ring_scheduler", + "holiday_router", + "release_notifier", + ] + for name in names: + path = Path("build/packages") / f"{name}.zip" + if not path.is_file(): + raise SystemExit(f"missing {path}") + with zipfile.ZipFile(path) as zf: + info = zf.read("shared/build_info.py").decode() + if sha not in info: + raise SystemExit(f"{path} missing GIT_SHA {sha}") + if "shared/sentry_init.py" not in zf.namelist(): + raise SystemExit(f"{path} missing bundled shared package") + print("zips ok") + PY - RELEASE_EXISTS=false - gh release view "v$TOP" >/dev/null 2>&1 && RELEASE_EXISTS=true - - echo "version=$TOP" >> "$GITHUB_OUTPUT" - echo "kind=$KIND" >> "$GITHUB_OUTPUT" - # Act only on a clean SemVer bump whose Release isn't published yet. - if [ "$KIND" != "none" ] && [ "$RELEASE_EXISTS" = "false" ]; then - echo "release=true" >> "$GITHUB_OUTPUT" - else - echo "release=false" >> "$GITHUB_OUTPUT" - echo "v$TOP: kind=$KIND release_exists=$RELEASE_EXISTS — no action." - fi - - - name: Build release notes - if: ${{ steps.rel.outputs.release == 'true' }} - run: | - python scripts/changelog_cli.py payload CHANGELOG.md "${{ steps.rel.outputs.version }}" > payload.json - python -c "import json; print(json.load(open('payload.json'))['notes'])" > notes.md - - # Announce BEFORE publishing the Release: the Release is the durable "done" - # marker (the step above skips once it exists), so announcing first keeps - # this retryable. Minor/major only, and only once the invoke-role variable - # has been bootstrapped (see README). - - name: Configure AWS credentials - if: ${{ steps.rel.outputs.release == 'true' && steps.rel.outputs.kind != 'patch' && vars.RELEASE_NOTIFY_INVOKE_ROLE_ARN != '' }} + - name: Configure AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6.2.4 with: - role-to-assume: ${{ vars.RELEASE_NOTIFY_INVOKE_ROLE_ARN }} + role-to-assume: ${{ env.DEPLOY_ROLE_ARN }} aws-region: us-east-1 + audience: sts.amazonaws.com - - name: Announce in Slack - if: ${{ steps.rel.outputs.release == 'true' && steps.rel.outputs.kind != 'patch' && vars.RELEASE_NOTIFY_INVOKE_ROLE_ARN != '' }} + - name: Get deploy parameters + id: deploy run: | - aws lambda invoke \ - --function-name afterhours-release-notifier \ - --cli-binary-format raw-in-base64-out \ - --payload file://payload.json \ - --output json response.json > invoke-meta.json - # aws lambda invoke only emits a FunctionError key when the handler errored. - if grep -q '"FunctionError"' invoke-meta.json; then - echo "::error::release-notifier returned an error"; cat response.json; exit 1 - fi - echo "Announced v${{ steps.rel.outputs.version }}." + set -euo pipefail + prefix=/afterhours-shift-manager/deploy + ARTIFACTS_BUCKET=$(aws ssm get-parameter --name "${prefix}/artifacts-bucket" --query Parameter.Value --output text) + { + echo "artifacts_bucket=${ARTIFACTS_BUCKET}" + echo "slack_bot=$(aws ssm get-parameter --name "${prefix}/slack_bot-function-name" --query Parameter.Value --output text)" + echo "weekly_post=$(aws ssm get-parameter --name "${prefix}/weekly_post-function-name" --query Parameter.Value --output text)" + echo "roster_sync=$(aws ssm get-parameter --name "${prefix}/roster_sync-function-name" --query Parameter.Value --output text)" + echo "roster_api=$(aws ssm get-parameter --name "${prefix}/roster_api-function-name" --query Parameter.Value --output text)" + echo "ring_scheduler=$(aws ssm get-parameter --name "${prefix}/ring_scheduler-function-name" --query Parameter.Value --output text)" + echo "holiday_router=$(aws ssm get-parameter --name "${prefix}/holiday_router-function-name" --query Parameter.Value --output text)" + echo "release_notifier=$(aws ssm get-parameter --name "${prefix}/release_notifier-function-name" --query Parameter.Value --output text)" + } >> "${GITHUB_OUTPUT}" - - name: Warn if announcement skipped (not bootstrapped) - if: ${{ steps.rel.outputs.release == 'true' && steps.rel.outputs.kind != 'patch' && vars.RELEASE_NOTIFY_INVOKE_ROLE_ARN == '' }} - run: echo "::warning::RELEASE_NOTIFY_INVOKE_ROLE_ARN is unset — tagging + releasing but not announcing. Set the repo variable from the stack output." - - - name: Publish GitHub Release - if: ${{ steps.rel.outputs.release == 'true' }} + - name: Upload zips and update function code env: - GH_TOKEN: ${{ github.token }} + ARTIFACTS_BUCKET: ${{ steps.deploy.outputs.artifacts_bucket }} + GIT_SHA: ${{ steps.commit.outputs.sha }} + SLACK_BOT: ${{ steps.deploy.outputs.slack_bot }} + WEEKLY_POST: ${{ steps.deploy.outputs.weekly_post }} + ROSTER_SYNC: ${{ steps.deploy.outputs.roster_sync }} + ROSTER_API: ${{ steps.deploy.outputs.roster_api }} + RING_SCHEDULER: ${{ steps.deploy.outputs.ring_scheduler }} + HOLIDAY_ROUTER: ${{ steps.deploy.outputs.holiday_router }} + RELEASE_NOTIFIER: ${{ steps.deploy.outputs.release_notifier }} run: | - # gh creates the tag at the deployed commit and the Release together. - gh release create "v${{ steps.rel.outputs.version }}" \ - --repo "${{ github.repository }}" \ - --title "v${{ steps.rel.outputs.version }}" \ - --notes-file notes.md \ - --target "${{ github.sha }}" + set -euo pipefail + keys=( + slack_bot:"${SLACK_BOT}" + weekly_post:"${WEEKLY_POST}" + roster_sync:"${ROSTER_SYNC}" + roster_api:"${ROSTER_API}" + ring_scheduler:"${RING_SCHEDULER}" + holiday_router:"${HOLIDAY_ROUTER}" + release_notifier:"${RELEASE_NOTIFIER}" + ) + for pair in "${keys[@]}"; do + name="${pair%%:*}" + fn="${pair#*:}" + key="functions/${name}/${GIT_SHA}.zip" + aws s3 cp "build/packages/${name}.zip" "s3://${ARTIFACTS_BUCKET}/${key}" + aws lambda update-function-code \ + --function-name "${fn}" \ + --s3-bucket "${ARTIFACTS_BUCKET}" \ + --s3-key "${key}" \ + --query '{Function:FunctionName,Sha256:CodeSha256,Updated:LastModified}' \ + --output table + aws lambda wait function-updated-v2 --function-name "${fn}" + done diff --git a/.gitignore b/.gitignore index 0d99426..e348d44 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ venv/ .env samconfig.toml output.json -.idea/ \ No newline at end of file +.idea/ +build/ +terraform/.terraform/ +terraform/build/ \ No newline at end of file diff --git a/README.md b/README.md index 8e7888f..902fc86 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # After-Hours Shift Manager ![Python](https://img.shields.io/badge/Python-3776AB?logo=python&logoColor=white) -![AWS SAM](https://img.shields.io/badge/AWS-SAM-FF9900?logo=amazonaws&logoColor=white) +![Terraform](https://img.shields.io/badge/Terraform-844FBA?logo=terraform&logoColor=white) ![Slack](https://img.shields.io/badge/Slack-integration-4A154B?logo=slack&logoColor=white) ![CI](https://github.com/Sea-Haven-Industries/afterhours-shift-manager/actions/workflows/ci.yaml/badge.svg) @@ -56,9 +56,9 @@ Example: `/oncall admin holiday add 2026-07-04 2 x2 Independence Day` schedules ## Architecture -- **Runtime**: Python 3.12 on AWS Lambda (arm64) +- **Runtime**: Python 3.12 on AWS Lambda (arm64), seahaven-prod `011934824531` - **Data**: DynamoDB single-table (`afterhours-shifts`) -- **IaC**: AWS SAM (`template.yaml`) with shared Lambda Layer +- **IaC**: HCP Terraform workspace `afterhours-shift-manager-prod` (containers) plus GitHub Actions `deploy.yaml` (zips). `src/shared` is bundled into each function zip. Terraform does not package `src/`. - **Slack**: Slack Bolt framework with `/oncall` slash command - **3CX Integration**: Queue routing updated directly via 3CX Queue XAPI - **Secrets**: AWS Secrets Manager (`afterhours-shift-manager/*`) @@ -73,7 +73,7 @@ Example: `/oncall admin holiday add 2026-07-04 2 x2 Independence Day` schedules | `afterhours-roster-api` | API Gateway (PUT /roster, DELETE /roster/{extension}) | Bearer-authenticated roster upsert/delete for the identity processor | | `afterhours-ring-scheduler` | EventBridge (daily 8am ET + weekend 5pm ET) | Updates 3CX queue routing based on who's on shift | | `afterhours-holiday-router` | EventBridge Scheduler (per-holiday one-off: 8am activate / 5pm deactivate ET) | Repoints the IVR to the holiday queue and sets queue agents for a holiday day shift; reverts at 5pm (see [Holidays](#holidays)) | -| `afterhours-release-notifier` | Invoked by the Deploy workflow's release job on minor/major releases | Posts a "What's New" announcement to the shift channel | +| `afterhours-release-notifier` | Skeleton only until tagging exists | Posts a "What's New" announcement to the shift channel | ### Project Layout @@ -86,7 +86,8 @@ src/ ring-scheduler/ 3CX queue routing updates holiday-router/ 3CX IVR/queue repoint for holiday day shifts (activate/deactivate) release-notifier/ Posts release announcements to Slack - shared/ Lambda Layer (schedule, blocks, changelog, 3CX client, secrets) + shared/ Bundled into each function zip (schedule, blocks, changelog, 3CX client, secrets) +terraform/ HCP Terraform (function skeletons, API, DDB, IAM, schedules) scripts/ changelog CLI + CI guard + in-package copy sync tests/ pytest suite (mirrors src/, one dir per Lambda + shared) ``` @@ -133,7 +134,7 @@ Map updates on the record (see above) so the slot count can't be oversubscribed. creates two **one-off EventBridge Scheduler** schedules for that date — `holiday-activate-` at 08:00 ET and `holiday-deactivate-` at 17:00 ET — whose names are stored on the record's `schedule_names`. Scheduler -assumes `HolidaySchedulerExecutionRole` to invoke `afterhours-holiday-router`: +assumes `afterhours-shift-manager-holiday-scheduler` to invoke `afterhours-holiday-router`: - **Activate (08:00):** capture both IVR `ivr_number` (800) routes — key-0 **and** no-input/timeout — into `CONFIG.captured_ivr_routes` (skipped if they already @@ -183,7 +184,7 @@ A slot claimed after the shift has started always needs an admin to approve it. ### Roster HTTP API -Identity hire/offboard in `paychex-integrations` calls this API. It is a separate Lambda on the same implicit HTTP API as Slack (`POST /slack/events` is unchanged). +Identity hire/offboard in `paychex-integrations` calls this API. It is a separate Lambda on the same HTTP API as Slack (`POST /slack/events` is unchanged). | Method | Path | Body | Success | |---|---|---|---| @@ -192,17 +193,11 @@ Identity hire/offboard in `paychex-integrations` calls this API. It is a separat Header: `Authorization: Bearer {token}`. Missing or wrong token is 401. Invalid JSON or fields is 400. A secret-read failure is 503. -Set processor `AFTERHOURS_BASE_URL` to the stack output `AfterhoursApiBaseUrl`: - -``` -https://${ServerlessHttpApi}.execute-api.us-east-1.amazonaws.com -``` - -That value is the API origin only. Do not append `/mgmt` or `/roster`. +Set processor `AFTERHOURS_BASE_URL` to the Terraform output `api_origin` (HCP variable `afterhours_base_url` on `paychex-integrations-prod`). That value is the API origin only. Do not append `/roster`. Flip it at cutover after DynamoDB is copied, not before. Daily `afterhours-roster-sync` still owns the 3CX `DEFAULT` group at 6am ET: rows absent from that group are deleted. Hire is safe because 3CX create (into `DEFAULT`) happens before the roster PUT. An HTTP-only row that is not in that group will be removed on the next sync. Sync preserves `slack_user_id` on existing rows and does not overwrite a just-created API row's Slack id. -Token rotation is a maintenance-window action. The Lambda caches the token per execution environment. Update the mgmt secret and the prod copy together, then recycle `afterhours-roster-api`. Updating only one copy, or recycling environments out of order, causes 401s until both sides match. +Token rotation is a maintenance-window action. The Lambda caches the token per execution environment. Update `afterhours-shift-manager/roster-api-token` and `paychex-integrations/afterhours-roster-token` together, then recycle `afterhours-roster-api`. Updating only one copy, or recycling environments out of order, causes 401s until both sides match. ## Documentation @@ -212,20 +207,15 @@ The canonical map of Sea Haven's AWS infrastructure lives in Confluence. This pr ## Deployment -Merges to `main` are automatically deployed via **GitHub Actions** using reusable SAM workflows from the Sea Haven org. +Infrastructure is applied by HCP Terraform workspace `afterhours-shift-manager-prod` (VCS on `main`, working directory `terraform/`, file trigger `terraform/**` only). Function code is shipped by `.github/workflows/deploy.yaml` on push to `main` (`environment: prod`). A terraform-only merge does not run the zip deploy. A mixed app+terraform merge may race the apply; re-run the deploy job if the functions are still stubs. -For manual deploys: - -```bash -sam build -sam deploy -``` +Manual zip redeploy: Actions → Deploy → Run workflow (`workflow_dispatch`, always prod). Do not `terraform apply` locally to prod. ## Monitoring & Alarms -All CloudWatch alarms are defined in `template.yaml` and notify the shared +All CloudWatch alarms are defined in `terraform/alarms.tf` and notify the shared `site-alerts` SNS topic (→ AWS Chatbot → Slack). None set `OKActions` — recovery -is not paged. Alarm names follow the in-template convention `Lambda--` +is not paged. Alarm names follow `Lambda--` (e.g. `Lambda-Errors-afterhours-ring-scheduler`). **Lambda alarms** (all seven functions: `afterhours-shift-manager`, @@ -252,8 +242,7 @@ transition normally. `ThrottledRequests` and `SystemErrors` are intentionally **not** alarmed: AWS emits them only at `TableName`+`Operation` granularity, so a `TableName`-only alarm would sit permanently in `INSUFFICIENT_DATA`. -**API Gateway alarms** (implicit HTTP API `ServerlessHttpApi`, `AWS/ApiGateway` -v2 metrics, `ApiId` dimension): +**API Gateway alarms** (HTTP API, `AWS/ApiGateway` v2 metrics, `ApiId` dimension): | Alarm | Metric | Condition | |---|---|---| @@ -266,40 +255,13 @@ v2 metrics, `ApiId` dimension): ## Releases & Versioning -The bot is versioned with SemVer, driven entirely by **`CHANGELOG.md`** — it is -the single source of truth for both the version number and the human-readable -notes. There is no separate tagging tool. +The bot is versioned with SemVer, driven entirely by **`CHANGELOG.md`**. The +**App Home** tab reads the copy that ships in the slack-bot zip. Run +`python scripts/sync_changelog.py` after editing the root file. Changelog Guard +enforces that the in-package copy matches. -**To cut a release**, in your feature PR add a new `## vX.Y.Z — Month D, YYYY` -section at the top of `CHANGELOG.md` (plain language, written for on-call staff), -bumping per SemVer, then run `python scripts/sync_changelog.py` to update the -in-package copy. The `Changelog Guard` PR check enforces that the bump is a clean -single SemVer step above the latest tag and that the two copies match. - -On the **deploy-then-merge** path, once the merge's Deploy succeeds, the Deploy -workflow's `release` job (`needs: deploy`) tags the new version, publishes a -GitHub Release with the notes, and — for **minor and major** bumps only (patches -stay silent) — invokes `afterhours-release-notifier` to post a "What's New" -message in the shift channel. The **App Home** tab ("About" page on the bot) -always shows the current version's notes, read from the CHANGELOG that ships in -the slack-bot package. - -> The release job lives inside the Deploy workflow (gated on `needs: deploy`) -> rather than a separate `workflow_run`-triggered workflow. A push-to-main run is -> a trusted context, so checking out and running repo code with write/OIDC is safe -> — whereas `workflow_run` is flagged by CodeQL for untrusted checkout. Gating on -> `needs: deploy` still guarantees we never announce a version that isn't live. - -**One-time setup (per environment):** after the first deploy creates the -`ReleaseNotifyInvokeRole`, copy its ARN from the `ReleaseNotifyInvokeRoleArn` stack -output into the repo **variable** `RELEASE_NOTIFY_INVOKE_ROLE_ARN` (Settings → -Secrets and variables → Actions → Variables). Until it's set, releases still tag -and publish but skip the Slack announcement (with a warning). - -> **Convention note (deliberate deviation).** The Sea Haven handbook says internal -> SAM stacks generally need no versioning and that tags are applied manually. This -> bot is versioned by owner choice (it has staff-facing release notes) and tagged -> automatically by the Deploy workflow's release job. This is intentional — not drift. +GitHub Releases and git tags are not cut by `deploy.yaml`. `afterhours-release-notifier` +exists as a function skeleton; CD does not invoke it until tagging exists. ## Testing @@ -318,6 +280,6 @@ pytest Each Lambda has its own `app.py`, so the per-package `conftest.py` loads each one under a unique module name (importlib mode) to avoid collisions. CI runs the same -suite on every PR via the org `ci-python-sam` workflow (`run-tests: true`). +suite on every PR via pytest plus `terraform fmt` / `init -backend=false` / `validate`. See [SETUP.md](SETUP.md) for full deployment and Slack app creation instructions. diff --git a/SETUP.md b/SETUP.md index f56c7d5..a270207 100644 --- a/SETUP.md +++ b/SETUP.md @@ -58,42 +58,66 @@ aws secretsmanager create-secret \ Create `afterhours-shift-manager/roster-api-token` **before** the first deploy that includes `afterhours-roster-api`, or live PUT/DELETE calls return 503. -Rotation is coordinated: write the new value to both the mgmt secret and the -prod copy, then recycle `afterhours-roster-api` so cached execution environments -pick it up. Updating only one copy causes 401s. The identity processor -`AFTERHOURS_BASE_URL` is the stack output `AfterhoursApiBaseUrl` (origin only, -no `/mgmt` or `/roster` suffix). Leave that URL empty until the API is live -and smoke-tested. +Rotation is coordinated: write the new value to both +`afterhours-shift-manager/roster-api-token` and +`paychex-integrations/afterhours-roster-token`, then recycle +`afterhours-roster-api` so cached execution environments pick it up. Updating +only one copy causes 401s. The identity processor `AFTERHOURS_BASE_URL` is the +Terraform output `api_origin` (origin only, no `/roster` suffix). Flip that HCP +variable on `paychex-integrations-prod` at cutover after DynamoDB is copied. Daily roster-sync still removes DynamoDB rows that are not in the 3CX `DEFAULT` group. Hire stays safe because 3CX create lands the extension in that group before the identity processor PUTs `/roster`. -> The Slack **channel ID** is not a secret — it's passed as the `ShiftChannel` -> deploy parameter in step 3, not stored in Secrets Manager or SSM. +> The Slack **channel ID** is not a secret. It is Terraform variable +> `shift_channel` (default `C0APATP612N`), not stored in Secrets Manager. -## 3. Deploy the Stack +## 3. HCP Terraform and GitHub Environment -```bash -# Build and deploy. ShiftChannel is the Slack channel ID for schedule posts -# (right-click the channel in Slack → Copy link → the ID is the last segment). -sam build -sam deploy --guided \ - --stack-name afterhours-shift-manager \ - --region us-east-1 \ - --parameter-overrides ShiftChannel=C0XXXXXXX QueueNumber=801 +Prod only. Workspace `afterhours-shift-manager-prod` in project `seahaven-prod` +(account `011934824531`). No seahaven-dev workspace. -# Note SlackBotApiUrl (Slack Request URL) and AfterhoursApiBaseUrl -# (paychex AFTERHOURS_BASE_URL origin). -``` +First apply uses the hcptf-bootstrap window (exact `StringEquals` trust, never +`StringLike`): + +1. Create the HCP workspace. Auto-apply off. No project-level variable set. + Working directory `terraform`. File trigger prefix `terraform/**` only. + Speculative plans on. VCS on `main`. +2. From `seahaven-org-baseline`: + `scripts/create-hcptf-bootstrap-roles.sh --account prod --allow-workspace afterhours-shift-manager-prod` +3. Point workspace `TFC_AWS_APPLY_ROLE_ARN` / `TFC_AWS_PLAN_ROLE_ARN` at + `hcptf-bootstrap` / `hcptf-bootstrap-plan`. Set `TFC_AWS_PROVIDER_AUTH=true`. +4. One manual apply with `schedules_enabled=false`. This creates the scoped + `hcptf-*` roles, the Lambda boundary, and the rest of the stack. If 3CX + secrets already exist from seahaven-door-unlock-api, import those three + names instead of creating them: + `terraform import 'aws_secretsmanager_secret.this["afterhours-shift-manager/3cx-domain"]' afterhours-shift-manager/3cx-domain` + (and the client-id / client-secret names). Do not overwrite 3CX values. +5. Retarget `TFC_AWS_*` to `hcptf-afterhours-shift-manager` / + `hcptf-afterhours-shift-manager-plan`. Re-run the create script with no + `--allow-workspace`. +6. Second manual apply as the scoped role. Then seal auto-apply on. + +GitHub Environment `prod`: reviewers, branch policy `main` only, Environment +variable `DEPLOY_ROLE_ARN` = Terraform output `github_deploy_role_arn`. + +Function zips: Actions → Deploy on push to `main`, or `workflow_dispatch`. +Keep `schedules_enabled=false` until Slack and Paychex point at this stack. + +HCP outputs to copy: `slack_request_url`, `api_origin`, +`holiday_scheduler_role_arn`, `github_deploy_role_arn`. ## 4. Set the Slack Request URL -After deploy, copy the `SlackBotApiUrl` from the SAM output. Go back to your Slack app settings: +Reuse the existing Slack app. After the zip deploy, copy `slack_request_url` +from HCP outputs: -- **Slash Commands** → edit `/oncall` → set **Request URL** to the output URL +- **Slash Commands** → edit `/oncall` → set **Request URL** to that URL - **Interactivity & Shortcuts** → set **Request URL** to the same URL +Do this in the cutover window, not before DynamoDB is copied. + ## 5. Seed the Schedule ```bash @@ -117,6 +141,39 @@ To have the 3CX scheduler read overrides from DynamoDB (so Slack-driven changes Without this step, the Slack bot still works — it invokes the 3CX scheduler Lambda directly for same-day changes. Future-date overrides would only take effect if the scheduler reads DynamoDB. +## 8. Prod cutover (PLAT-74) + +Avoid Monday 06:00-08:00 ET and any holiday 08:00/17:00 ET window. Dry-run the +scripts first (`--execute` is required for writes). + +1. Merge this repo's PR (SAM CD is gone). First HCP apply is the bootstrap + window above with `schedules_enabled=false`. +2. Copy DynamoDB `afterhours-shifts` mgmt → prod. Verify item counts: + `python scripts/cutover/copy_dynamodb.py --src-profile mgmt --dst-profile prod` + then `--execute`. +3. Confirm secrets in prod. `copy_secrets.py` writes Slack bot token, Slack + signing secret, and roster-api-token into empty Terraform shells and skips + dest names that already have a value. It never writes 3CX secrets: + `python scripts/cutover/copy_secrets.py --src-profile mgmt --dst-profile prod` + then `--execute`. Strip trailing newlines is built in. +4. GHA `workflow_dispatch` (or the merge deploy; re-run if it raced apply) to + overwrite stubs. +5. Recreate outstanding future `holiday-activate-*` / `holiday-deactivate-*` + in prod against the new router ARN and scheduler role: + `python scripts/cutover/recreate_holiday_schedules.py --src-profile mgmt --dst-profile prod` +6. Instant cut: Slack Request URL → prod `/slack/events`; Paychex HCP variable + `afterhours_base_url` on `paychex-integrations-prod` → prod `api_origin`; + `schedules_enabled=true` via a terraform-only merge; disable mgmt EventBridge. + Smoke: Slack `/oncall`, roster PUT/DELETE, weekly-post SendMessage (or + simulate-principal-policy plus one smoke message), ring-scheduler invoke, + holiday GetSchedule. +7. Seal auto-apply on. Delete the mgmt SAM stack. Remove the mgmt SQS principal + from `paychex-checkcomponents`. Update Confluence AWS Architecture Map and + check PLAT-71 item 4. + +Do not dual-run 3CX writers. Do not flip `afterhours_base_url` before DynamoDB +is copied. + ## Commands Reference | Command | Description | diff --git a/samconfig.toml.example b/samconfig.toml.example deleted file mode 100644 index daba8cd..0000000 --- a/samconfig.toml.example +++ /dev/null @@ -1,9 +0,0 @@ -version = 0.1 - -[default.deploy.parameters] -stack_name = "afterhours-shift-manager" -resolve_s3 = true -s3_prefix = "afterhours-shift-manager" -region = "us-east-1" -capabilities = "CAPABILITY_IAM" -confirm_changeset = true diff --git a/scripts/cutover/copy_dynamodb.py b/scripts/cutover/copy_dynamodb.py new file mode 100644 index 0000000..d381f78 --- /dev/null +++ b/scripts/cutover/copy_dynamodb.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Copy afterhours-shifts from mgmt to prod. Dry-run unless --execute.""" + +from __future__ import annotations + +import argparse +import sys + +import boto3 + + +TABLE = "afterhours-shifts" +SRC_ACCOUNT = "328440206208" +DST_ACCOUNT = "011934824531" + + +def _client(profile: str, region: str): + session = boto3.Session(profile_name=profile, region_name=region) + return session.client("dynamodb") + + +def _scan_all(client): + items = [] + kwargs = {"TableName": TABLE} + while True: + resp = client.scan(**kwargs) + items.extend(resp.get("Items", [])) + start = resp.get("LastEvaluatedKey") + if not start: + return items + kwargs["ExclusiveStartKey"] = start + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--src-profile", required=True) + parser.add_argument("--dst-profile", required=True) + parser.add_argument("--region", default="us-east-1") + parser.add_argument("--execute", action="store_true") + args = parser.parse_args() + + src = _client(args.src_profile, args.region) + dst = _client(args.dst_profile, args.region) + src_id = boto3.Session(profile_name=args.src_profile).client("sts").get_caller_identity()["Account"] + dst_id = boto3.Session(profile_name=args.dst_profile).client("sts").get_caller_identity()["Account"] + if src_id != SRC_ACCOUNT: + print(f"src account {src_id} is not mgmt {SRC_ACCOUNT}", file=sys.stderr) + return 2 + if dst_id != DST_ACCOUNT: + print(f"dst account {dst_id} is not prod {DST_ACCOUNT}", file=sys.stderr) + return 2 + + items = _scan_all(src) + dst_count = dst.describe_table(TableName=TABLE)["Table"]["ItemCount"] + print(f"src items={len(items)} dst describe ItemCount={dst_count}") + if not args.execute: + print("dry-run; pass --execute to BatchWriteItem") + return 0 + + written = 0 + batch = [] + for item in items: + batch.append({"PutRequest": {"Item": item}}) + if len(batch) == 25: + dst.batch_write_item(RequestItems={TABLE: batch}) + written += len(batch) + batch = [] + if batch: + dst.batch_write_item(RequestItems={TABLE: batch}) + written += len(batch) + after = _scan_all(dst) + print(f"wrote={written} dst_scan={len(after)}") + if len(after) != len(items): + print("item counts differ after copy", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cutover/recreate_holiday_schedules.py b/scripts/cutover/recreate_holiday_schedules.py new file mode 100644 index 0000000..dcf5cbc --- /dev/null +++ b/scripts/cutover/recreate_holiday_schedules.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Recreate future holiday-* EventBridge Scheduler schedules in prod. + +Reads outstanding holiday-activate-* / holiday-deactivate-* from mgmt and +creates the same names in prod targeting the prod router ARN and scheduler +role. Dry-run unless --execute. +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone + +import boto3 +from botocore.exceptions import ClientError + +SRC_ACCOUNT = "328440206208" +DST_ACCOUNT = "011934824531" +PROD_ROUTER_ARN = "arn:aws:lambda:us-east-1:011934824531:function:afterhours-holiday-router" +PROD_ROLE_ARN = "arn:aws:iam::011934824531:role/tf-managed/afterhours-shift-manager-holiday-scheduler" +PREFIXES = ("holiday-activate-", "holiday-deactivate-") + + +def _client(profile: str, region: str): + return boto3.Session(profile_name=profile, region_name=region).client("scheduler") + + +def _account(profile: str) -> str: + return boto3.Session(profile_name=profile).client("sts").get_caller_identity()["Account"] + + +def _list_holiday(client): + names = [] + token = None + while True: + kwargs = {"GroupName": "default"} + if token: + kwargs["NextToken"] = token + resp = client.list_schedules(**kwargs) + for item in resp.get("Schedules", []): + name = item.get("Name", "") + if name.startswith(PREFIXES): + names.append(name) + token = resp.get("NextToken") + if not token: + return names + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--src-profile", required=True) + parser.add_argument("--dst-profile", required=True) + parser.add_argument("--region", default="us-east-1") + parser.add_argument("--execute", action="store_true") + args = parser.parse_args() + + if _account(args.src_profile) != SRC_ACCOUNT: + print("src profile is not mgmt", file=sys.stderr) + return 2 + if _account(args.dst_profile) != DST_ACCOUNT: + print("dst profile is not prod", file=sys.stderr) + return 2 + + src = _client(args.src_profile, args.region) + dst = _client(args.dst_profile, args.region) + now = datetime.now(timezone.utc) + created = 0 + skipped = 0 + + for name in _list_holiday(src): + detail = src.get_schedule(Name=name, GroupName="default") + expr = detail.get("ScheduleExpression", "") + tzname = detail.get("ScheduleExpressionTimezone", "America/New_York") + at = detail.get("EndDate") or detail.get("StartDate") + if at is not None and at < now: + print(f"skip past {name}") + skipped += 1 + continue + payload = { + "Name": name, + "GroupName": "default", + "ScheduleExpression": expr, + "ScheduleExpressionTimezone": tzname, + "FlexibleTimeWindow": {"Mode": "OFF"}, + "Target": { + "Arn": PROD_ROUTER_ARN, + "RoleArn": PROD_ROLE_ARN, + "Input": detail.get("Target", {}).get("Input", ""), + }, + "ActionAfterCompletion": detail.get("ActionAfterCompletion", "DELETE"), + } + if detail.get("EndDate"): + payload["EndDate"] = detail["EndDate"] + print(f"would create {name} expr={expr} tz={tzname}") + if not args.execute: + continue + try: + dst.create_schedule(**payload) + created += 1 + except ClientError as exc: + if exc.response["Error"]["Code"] == "ConflictException": + print(f"exists {name}") + else: + raise + + print(f"created={created} skipped_past={skipped} execute={args.execute}") + if not args.execute: + print("dry-run; pass --execute to CreateSchedule") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/package_lambdas.py b/scripts/package_lambdas.py new file mode 100644 index 0000000..23ae68b --- /dev/null +++ b/scripts/package_lambdas.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Build Lambda zips with src/shared bundled in. Used by deploy.yaml. + +Each zip is functions//.zip on S3. GIT_SHA is written to +shared/build_info.py inside the zip so Sentry release is the commit, not a +runtime env var Terraform would own. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# Keys match terraform/locals.tf local.functions. +FUNCTIONS = { + "slack_bot": ROOT / "src" / "slack-bot", + "weekly_post": ROOT / "src" / "weekly-post", + "roster_sync": ROOT / "src" / "roster-sync", + "roster_api": ROOT / "src" / "roster-api", + "ring_scheduler": ROOT / "src" / "ring-scheduler", + "holiday_router": ROOT / "src" / "holiday-router", + "release_notifier": ROOT / "src" / "release-notifier", +} + +SKIP_INSTALL_PREFIXES = ("boto3", "botocore") +SKIP_COPY_NAMES = {"requirements.txt", "__pycache__"} + + +def _req_lines(path: Path) -> list[str]: + lines: list[str] = [] + if not path.is_file(): + return lines + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + lower = line.lower() + if any(lower.startswith(prefix) for prefix in SKIP_INSTALL_PREFIXES): + continue + lines.append(line) + return lines + + +def _copy_tree(src: Path, dest: Path) -> None: + dest.mkdir(parents=True, exist_ok=True) + for item in src.iterdir(): + if item.name in SKIP_COPY_NAMES or item.name.endswith(".pyc"): + continue + target = dest / item.name + if item.is_dir(): + if item.name == "__pycache__": + continue + shutil.copytree(item, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + else: + shutil.copy2(item, target) + + +def build_function(name: str, src: Path, git_sha: str, out_dir: Path) -> Path: + with tempfile.TemporaryDirectory(prefix=f"afterhours-{name}-") as tmp: + dest = Path(tmp) + _copy_tree(src, dest) + shared_src = ROOT / "src" / "shared" / "shared" + _copy_tree(shared_src, dest / "shared") + (dest / "shared" / "build_info.py").write_text( + f'"""Pinned at zip time by scripts/package_lambdas.py."""\n\nGIT_SHA = "{git_sha}"\n', + encoding="utf-8", + ) + + reqs = _req_lines(src / "requirements.txt") + _req_lines( + ROOT / "src" / "shared" / "requirements.txt" + ) + # Preserve order, drop duplicates. + seen: set[str] = set() + unique: list[str] = [] + for line in reqs: + if line not in seen: + seen.add(line) + unique.append(line) + if unique: + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-compile", + "--python-version", + "3.12", + "--platform", + "manylinux2014_aarch64", + "--only-binary=:all:", + "--target", + str(dest), + *unique, + ] + subprocess.run(cmd, check=True) + + out_dir.mkdir(parents=True, exist_ok=True) + zip_path = out_dir / f"{name}.zip" + if zip_path.exists(): + zip_path.unlink() + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for dirpath, dirnames, filenames in os.walk(dest): + dirnames[:] = [d for d in dirnames if d != "__pycache__"] + for filename in filenames: + if filename.endswith(".pyc"): + continue + full = Path(dirpath) / filename + rel = full.relative_to(dest) + zf.write(full, rel.as_posix()) + return zip_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--out-dir", type=Path, default=ROOT / "build" / "packages") + parser.add_argument("--only", nargs="*", default=()) + args = parser.parse_args() + selected = args.only or list(FUNCTIONS) + missing = [name for name in selected if name not in FUNCTIONS] + if missing: + print(f"unknown function keys: {missing}", file=sys.stderr) + return 2 + for name in selected: + path = build_function(name, FUNCTIONS[name], args.git_sha, args.out_dir) + print(path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/shared/shared/sentry_init.py b/src/shared/shared/sentry_init.py index e7b6bd5..24ef21e 100644 --- a/src/shared/shared/sentry_init.py +++ b/src/shared/shared/sentry_init.py @@ -120,19 +120,31 @@ def _before_send(event, _hint): return event +def _git_sha(): + try: + from shared.build_info import GIT_SHA + except ImportError: + return os.environ.get("GIT_SHA", "").strip() + return str(GIT_SHA or "").strip() + + def init_sentry(): dsn = os.environ.get("SENTRY_DSN") if not dsn: return - sentry_sdk.init( - dsn=dsn, - integrations=[AwsLambdaIntegration(timeout_warning=True)], - send_default_pii=False, - include_local_variables=False, - enable_logs=False, - traces_sample_rate=0.0, - before_send=_before_send, - ) + kwargs = { + "dsn": dsn, + "integrations": [AwsLambdaIntegration(timeout_warning=True)], + "send_default_pii": False, + "include_local_variables": False, + "enable_logs": False, + "traces_sample_rate": 0.0, + "before_send": _before_send, + } + sha = _git_sha() + if sha: + kwargs["release"] = sha + sentry_sdk.init(**kwargs) init_sentry() diff --git a/template.yaml b/template.yaml deleted file mode 100644 index 7616ba3..0000000 --- a/template.yaml +++ /dev/null @@ -1,1073 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Transform: AWS::Serverless-2016-10-31 -Description: After-Hours Shift Manager — Slack bot for managing on-call shifts with 3CX integration - -Parameters: - Timezone: - Type: String - Default: "America/New_York" - ShiftChannel: - Type: String - Description: Slack channel ID for schedule posts and shift notifications - QueueNumber: - Type: String - Default: "801" - Description: 3CX queue extension number to update - SentryDsn: - Type: String - Default: "" - NoEcho: true - Description: Sentry DSN; empty disables error reporting - CheckcomponentsQueueUrl: - Type: String - Default: "https://sqs.us-east-1.amazonaws.com/011934824531/paychex-checkcomponents" - Description: paychex-checkcomponents SQS URL. Empty skips the weekly SendMessage. - CheckcomponentsQueueArn: - Type: String - Default: "arn:aws:sqs:us-east-1:011934824531:paychex-checkcomponents" - Description: paychex-checkcomponents SQS ARN for WeeklyPost SendMessage. - -Globals: - Function: - Runtime: python3.12 - Timeout: 30 - MemorySize: 1024 - Architectures: - - arm64 - PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary - Environment: - Variables: - SENTRY_DSN: !Ref SentryDsn - - # Access logging + default throttling on the implicit HTTP API (audit M-18). - HttpApi: - AccessLogSettings: - DestinationArn: !GetAtt ApiAccessLogGroup.Arn - Format: '{"requestId":"$context.requestId","ip":"$context.identity.sourceIp","requestTime":"$context.requestTime","method":"$context.httpMethod","routeKey":"$context.routeKey","status":"$context.status","protocol":"$context.protocol","responseLength":"$context.responseLength","integrationError":"$context.integrationErrorMessage"}' - DefaultRouteSettings: - ThrottlingBurstLimit: 50 - ThrottlingRateLimit: 100 - -Resources: - ApiAccessLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: /aws/apigateway/afterhours-shift-manager - RetentionInDays: 90 - - # --- Shared Lambda Layer --- - SharedLayer: - Type: AWS::Serverless::LayerVersion - Properties: - LayerName: afterhours-shared - ContentUri: src/shared/ - CompatibleRuntimes: - - python3.12 - CompatibleArchitectures: - - arm64 - Metadata: - BuildMethod: python3.12 - BuildArchitecture: arm64 - - # --- DynamoDB --- - ShiftTable: - Type: AWS::DynamoDB::Table - Properties: - TableName: afterhours-shifts - BillingMode: PAY_PER_REQUEST - AttributeDefinitions: - - AttributeName: PK - AttributeType: S - - AttributeName: SK - AttributeType: S - KeySchema: - - AttributeName: PK - KeyType: HASH - - AttributeName: SK - KeyType: RANGE - TimeToLiveSpecification: - AttributeName: expires_at - Enabled: true - - # --- Slack Bot Lambda --- - SlackBotFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-shift-manager - Handler: handler.handler - CodeUri: src/slack-bot/ - Layers: - - !Ref SharedLayer - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token - SLACK_SIGNING_SECRET: afterhours-shift-manager/slack-signing-secret - SHIFT_CHANNEL: !Ref ShiftChannel - TCX_SECRET_PREFIX: afterhours-shift-manager/3cx- - QUEUE_NUMBER: !Ref QueueNumber - TZ: !Ref Timezone - # Holiday scheduling: per-holiday one-off schedules target the router, - # passing the scheduler exec role; inline activation invokes it directly. - HOLIDAY_ROUTER_ARN: !GetAtt HolidayRouterFunction.Arn - HOLIDAY_SCHEDULER_ROLE_ARN: !GetAtt HolidaySchedulerExecutionRole.Arn - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref ShiftTable - - Statement: - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*" - # Manage the per-holiday EventBridge Scheduler one-off schedules - # (08:00 activate / 17:00 deactivate of the holiday router). - - Effect: Allow - Action: - - scheduler:CreateSchedule - - scheduler:DeleteSchedule - - scheduler:GetSchedule - # Predictable names (holiday-activate-/holiday-deactivate-) - # in the default group — scope to those rather than all schedules. - Resource: - - !Sub "arn:aws:scheduler:${AWS::Region}:${AWS::AccountId}:schedule/default/holiday-*" - # PassRole only for the holiday scheduler exec role, and only when - # handed to EventBridge Scheduler. - - Effect: Allow - Action: iam:PassRole - Resource: !GetAtt HolidaySchedulerExecutionRole.Arn - Condition: - StringEquals: - iam:PassedToService: scheduler.amazonaws.com - # Inline activation (holiday added mid-window) invokes the router now. - # Unqualified ARN only — we invoke the base function, no alias/version. - - Effect: Allow - Action: lambda:InvokeFunction - Resource: !GetAtt HolidayRouterFunction.Arn - Events: - SlackEvents: - Type: HttpApi - Properties: - Path: /slack/events - Method: POST - - # --- Weekly Schedule Post (Monday 7am ET) --- - WeeklyPostFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-weekly-post - Handler: app.handler - CodeUri: src/weekly-post/ - Layers: - - !Ref SharedLayer - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token - SHIFT_CHANNEL: !Ref ShiftChannel - PAY_REPORT_USER: U0A3SC48T47 - TZ: !Ref Timezone - CHECKCOMPONENTS_QUEUE_URL: !Ref CheckcomponentsQueueUrl - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref ShiftTable - - Statement: - # Least privilege: only the Slack bot token, not the whole namespace. - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/slack-bot-token-*" - - Effect: Allow - Action: - - sqs:SendMessage - Resource: - - !Ref CheckcomponentsQueueArn - Events: - # EST: 7am ET = 12:00 UTC (Nov-Mar) - WeeklyPostEST: - Type: Schedule - Properties: - Schedule: cron(0 12 ? * MON *) - Description: "Post weekly schedule Monday 7am EST" - Enabled: true - # EDT: 7am ET = 11:00 UTC (Mar-Nov) - WeeklyPostEDT: - Type: Schedule - Properties: - Schedule: cron(0 11 ? * MON *) - Description: "Post weekly schedule Monday 7am EDT" - Enabled: true - - # --- Roster Sync Lambda (daily sync from 3CX) --- - RosterSyncFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-roster-sync - Handler: app.handler - CodeUri: src/roster-sync/ - Layers: - - !Ref SharedLayer - Timeout: 60 - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - TCX_SECRET_PREFIX: afterhours-shift-manager/3cx- - SYNC_GROUP: DEFAULT - TZ: !Ref Timezone - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref ShiftTable - - Statement: - # Least privilege: only the 3cx-* secrets this function reads. - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/3cx-*" - Events: - # Daily at 6am ET (before the 7am schedule post and 8am 3CX scheduler) - # EST: 6am ET = 11:00 UTC (Nov-Mar) - RosterSyncEST: - Type: Schedule - Properties: - Schedule: cron(0 11 ? * * *) - Description: "Sync roster from 3CX at 6am EST" - Enabled: true - # EDT: 6am ET = 10:00 UTC (Mar-Nov) - RosterSyncEDT: - Type: Schedule - Properties: - Schedule: cron(0 10 ? * * *) - Description: "Sync roster from 3CX at 6am EDT" - Enabled: true - - # --- Roster API (HTTP PUT /roster and DELETE /roster/{extension}) --- - RosterApiFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-roster-api - Handler: app.handler - CodeUri: src/roster-api/ - Layers: - - !Ref SharedLayer - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - ROSTER_API_TOKEN_SECRET: afterhours-shift-manager/roster-api-token - TZ: !Ref Timezone - Policies: - - Statement: - - Effect: Allow - Action: - - dynamodb:UpdateItem - - dynamodb:DeleteItem - Resource: !GetAtt ShiftTable.Arn - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/roster-api-token-*" - Events: - PutRoster: - Type: HttpApi - Properties: - Path: /roster - Method: PUT - DeleteRoster: - Type: HttpApi - Properties: - Path: /roster/{extension} - Method: DELETE - - # --- Ring Scheduler (daily 3CX queue routing updates) --- - RingSchedulerFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-ring-scheduler - Handler: app.handler - CodeUri: src/ring-scheduler/ - Layers: - - !Ref SharedLayer - Timeout: 60 - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - TCX_SECRET_PREFIX: afterhours-shift-manager/3cx- - QUEUE_NUMBER: !Ref QueueNumber - TZ: !Ref Timezone - Policies: - - DynamoDBReadPolicy: - TableName: !Ref ShiftTable - - Statement: - # Least privilege: only the 3cx-* secrets this function reads. - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/3cx-*" - Events: - # Daily at 8am ET — update after-hours routing - DailyScheduleEST: - Type: Schedule - Properties: - Schedule: cron(0 13 ? * * *) - Description: "Update 3CX queue at 8am EST" - Enabled: true - DailyScheduleEDT: - Type: Schedule - Properties: - Schedule: cron(0 12 ? * * *) - Description: "Update 3CX queue at 8am EDT" - Enabled: true - # Weekends at 5pm ET — switch to night shift person - WeekendEveningEST: - Type: Schedule - Properties: - Schedule: cron(0 22 ? * SAT,SUN *) - Description: "Update 3CX queue at 5pm EST weekends" - Enabled: true - WeekendEveningEDT: - Type: Schedule - Properties: - Schedule: cron(0 21 ? * SAT,SUN *) - Description: "Update 3CX queue at 5pm EDT weekends" - Enabled: true - - # Lambda error alarm for the ring scheduler. Mirrors the account-wide - # operational convention (Lambda-Errors-, threshold 1 over one 5-min - # period, Sum, missing=notBreaching) and pages the same site-alerts SNS topic - # → AWS Chatbot → Slack as the other afterhours-* functions. - RingSchedulerErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${RingSchedulerFunction}" - AlarmDescription: "Ring scheduler Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref RingSchedulerFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- Holiday Router (repoints 3CX IVR/queue for a holiday day-shift) --- - # Invoked with {"action": "activate"|"deactivate", "date": ""} at - # 08:00 ET (activate) and 17:00 ET (deactivate) for each holiday date. Both - # operations are idempotent. No standing schedule here — invocation is driven - # per-holiday-date (the holiday record gates the work; off-days are no-ops). - HolidayRouterFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-holiday-router - Handler: app.handler - CodeUri: src/holiday-router/ - Layers: - - !Ref SharedLayer - Timeout: 60 - Environment: - Variables: - SHIFT_TABLE: !Ref ShiftTable - TCX_SECRET_PREFIX: afterhours-shift-manager/3cx- - TZ: !Ref Timezone - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref ShiftTable - - Statement: - # Least privilege: only the 3cx-* secrets this function reads. - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/3cx-*" - - # Lambda error alarm for the holiday router. Mirrors the account-wide - # operational convention (Lambda-Errors-, threshold 1 over one 5-min - # period, Sum, missing=notBreaching) and pages the same site-alerts SNS topic - # → AWS Chatbot → Slack as the other afterhours-* functions. - HolidayRouterErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${HolidayRouterFunction}" - AlarmDescription: "Holiday router Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref HolidayRouterFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # EventBridge Scheduler execution role. The slack-bot creates one-off - # schedules per holiday date (08:00 activate / 17:00 deactivate); Scheduler - # assumes this role to invoke the holiday router. Auto-named (no RoleName) so - # the deploy's CAPABILITY_IAM suffices — cd-sam does not pass - # CAPABILITY_NAMED_IAM. The permissions boundary is REQUIRED: the scoped - # github-cfn-execution-role gates iam:CreateRole/PutRolePolicy on roles - # carrying exactly this boundary, so the CI deploy is denied without it. - HolidaySchedulerExecutionRole: - Type: AWS::IAM::Role - Properties: - PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: scheduler.amazonaws.com - Action: sts:AssumeRole - Condition: - StringEquals: - aws:SourceAccount: !Ref AWS::AccountId - # Only schedules this stack creates (holiday-* in the default group) - # may assume the role — not any schedule in the account. - ArnLike: - aws:SourceArn: !Sub "arn:aws:scheduler:${AWS::Region}:${AWS::AccountId}:schedule/default/holiday-*" - Policies: - - PolicyName: invoke-holiday-router - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: lambda:InvokeFunction - Resource: !GetAtt HolidayRouterFunction.Arn - - # --- Release Notifier (invoked by release.yaml on minor/major releases) --- - ReleaseNotifierFunction: - Type: AWS::Serverless::Function - Properties: - FunctionName: afterhours-release-notifier - Handler: app.handler - CodeUri: src/release-notifier/ - Layers: - - !Ref SharedLayer - Environment: - Variables: - SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token - SHIFT_CHANNEL: !Ref ShiftChannel - TZ: !Ref Timezone - Policies: - # Least privilege: only the Slack bot token, not the whole namespace. - - Statement: - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/slack-bot-token-*" - - # GitHub-OIDC role assumed by release.yaml to invoke the notifier. Auto-named - # (no RoleName) so the deploy's CAPABILITY_IAM is sufficient — cd-sam does not - # pass CAPABILITY_NAMED_IAM. Trust + permission are scoped to the minimum: this - # repo's main ref + release workflow, and InvokeFunction on the notifier alone. - # Its ARN is surfaced as a stack output and set once as the - # RELEASE_NOTIFY_INVOKE_ROLE_ARN repo variable (see README). - # - # The permissions boundary is REQUIRED, not optional: the scoped - # github-cfn-execution-role's IAM policy gates iam:CreateRole/PutRolePolicy on - # the role carrying exactly this boundary, so the CI deploy is denied without - # it. The boundary itself permits lambda:InvokeFunction (Sid LambdaInvoke), so - # it does not restrict this role's one job. - ReleaseNotifyInvokeRole: - Type: AWS::IAM::Role - Properties: - PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com" - Action: sts:AssumeRoleWithWebIdentity - Condition: - StringEquals: - token.actions.githubusercontent.com:aud: sts.amazonaws.com - StringLike: - token.actions.githubusercontent.com:sub: "repo:Sea-Haven-Industries/afterhours-shift-manager:ref:refs/heads/main" - # Defense-in-depth: only the Deploy workflow's release job may assume - # this role, not any workflow running on main. (The release job lives - # in deploy.yaml; this must match that workflow's path.) - token.actions.githubusercontent.com:job_workflow_ref: "Sea-Haven-Industries/afterhours-shift-manager/.github/workflows/deploy.yaml@refs/heads/main" - Policies: - - PolicyName: invoke-release-notifier - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: lambda:InvokeFunction - Resource: !GetAtt ReleaseNotifierFunction.Arn - - # =========================================================================== - # CloudWatch alarm coverage (Wave 1 PR A). All alarms page the same - # site-alerts SNS topic → AWS Chatbot → Slack as the existing Errors alarms. - # No OKActions by design (the existing Errors alarms have none either). - # Naming follows the in-template convention: Lambda--${Fn}. - # =========================================================================== - - # --- Lambda Errors alarms (clone of HolidayRouterErrorAlarm) --- - # Errors for ring-scheduler + holiday-router already exist above. - - RosterSyncErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${RosterSyncFunction}" - AlarmDescription: "Roster sync Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref RosterSyncFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - ReleaseNotifierErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${ReleaseNotifierFunction}" - AlarmDescription: "Release notifier Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref ReleaseNotifierFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # ORPHAN ADOPTION: live alarms named Lambda-Errors-afterhours-shift-manager - # and Lambda-Errors-afterhours-weekly-post already exist OUTSIDE the stack. - # They MUST be deleted immediately before this stack deploys, or CloudFormation - # will fail to create these resources (AlarmName collision). See PR body. - SlackBotErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${SlackBotFunction}" - AlarmDescription: "Slack bot Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref SlackBotFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - WeeklyPostErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${WeeklyPostFunction}" - AlarmDescription: "Weekly post Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref WeeklyPostFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- Lambda Duration alarms (Maximum, ms; 2-of-3 evaluation) --- - # Thresholds set to ~80% of each function's timeout, with 2-of-3 evaluation. - # Timeouts: slack-bot/weekly-post/release-notifier/roster-api = 30s (global - # default); roster-sync/ring-scheduler/holiday-router = 60s. - SlackBotDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${SlackBotFunction}" - AlarmDescription: "Slack bot Lambda duration approaching its 30s timeout (>=24s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref SlackBotFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 24000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - WeeklyPostDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${WeeklyPostFunction}" - AlarmDescription: "Weekly post Lambda duration approaching its 30s timeout (>=24s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref WeeklyPostFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 24000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RosterSyncDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${RosterSyncFunction}" - AlarmDescription: "Roster sync Lambda duration approaching its 60s timeout (>=48s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref RosterSyncFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 48000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RingSchedulerDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${RingSchedulerFunction}" - AlarmDescription: "Ring scheduler Lambda duration approaching its 60s timeout (>=48s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref RingSchedulerFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 48000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - HolidayRouterDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${HolidayRouterFunction}" - AlarmDescription: "Holiday router Lambda duration approaching its 60s timeout (>=48s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref HolidayRouterFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 48000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - ReleaseNotifierDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${ReleaseNotifierFunction}" - AlarmDescription: "Release notifier Lambda duration approaching its 30s timeout (>=24s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref ReleaseNotifierFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 24000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- Lambda Throttles alarms (Sum; threshold 1 over one 5-min period) --- - SlackBotThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${SlackBotFunction}" - AlarmDescription: "Slack bot Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref SlackBotFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - WeeklyPostThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${WeeklyPostFunction}" - AlarmDescription: "Weekly post Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref WeeklyPostFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RosterSyncThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${RosterSyncFunction}" - AlarmDescription: "Roster sync Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref RosterSyncFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RingSchedulerThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${RingSchedulerFunction}" - AlarmDescription: "Ring scheduler Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref RingSchedulerFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - HolidayRouterThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${HolidayRouterFunction}" - AlarmDescription: "Holiday router Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref HolidayRouterFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RosterApiErrorAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Errors-${RosterApiFunction}" - AlarmDescription: "Roster API Lambda reported one or more errors" - Namespace: AWS/Lambda - MetricName: Errors - Dimensions: - - Name: FunctionName - Value: !Ref RosterApiFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RosterApiDurationAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Duration-${RosterApiFunction}" - AlarmDescription: "Roster API Lambda duration approaching its 30s timeout (>=24s)" - Namespace: AWS/Lambda - MetricName: Duration - Dimensions: - - Name: FunctionName - Value: !Ref RosterApiFunction - Statistic: Maximum - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 24000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - RosterApiThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${RosterApiFunction}" - AlarmDescription: "Roster API Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref RosterApiFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - ReleaseNotifierThrottlesAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "Lambda-Throttles-${ReleaseNotifierFunction}" - AlarmDescription: "Release notifier Lambda was throttled (concurrency limit hit)" - Namespace: AWS/Lambda - MetricName: Throttles - Dimensions: - - Name: FunctionName - Value: !Ref ReleaseNotifierFunction - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- DynamoDB alarms (afterhours-shifts table) --- - # ReadThrottleEvents / WriteThrottleEvents are the table-level throttle - # signals: AWS/DynamoDB emits them at the TableName dimension, so these - # alarms transition normally. (ThrottledRequests and SystemErrors are NOT - # emitted at TableName-only granularity — only at TableName+Operation — so - # alarms on them sit permanently in INSUFFICIENT_DATA and never fire.) - ShiftTableReadThrottleAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "DDB-ReadThrottle-${ShiftTable}" - AlarmDescription: "afterhours-shifts table had one or more read throttle events" - Namespace: AWS/DynamoDB - MetricName: ReadThrottleEvents - Dimensions: - - Name: TableName - Value: !Ref ShiftTable - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 0 - ComparisonOperator: GreaterThanThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - ShiftTableWriteThrottleAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "DDB-WriteThrottle-${ShiftTable}" - AlarmDescription: "afterhours-shifts table had one or more write throttle events" - Namespace: AWS/DynamoDB - MetricName: WriteThrottleEvents - Dimensions: - - Name: TableName - Value: !Ref ShiftTable - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 0 - ComparisonOperator: GreaterThanThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- API Gateway v2 (HTTP API) alarms on the implicit ServerlessHttpApi --- - # AWS::ApiGatewayV2 metric names: 4xx, 5xx, Latency; dimension ApiId. - ApiGateway4xxAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "ApiGateway-4xx-${ServerlessHttpApi}" - AlarmDescription: "Elevated 4xx responses on the afterhours HTTP API" - Namespace: AWS/ApiGateway - MetricName: 4xx - Dimensions: - - Name: ApiId - Value: !Ref ServerlessHttpApi - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 5 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - ApiGateway5xxAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "ApiGateway-5xx-${ServerlessHttpApi}" - AlarmDescription: "5xx responses on the afterhours HTTP API" - Namespace: AWS/ApiGateway - MetricName: 5xx - Dimensions: - - Name: ApiId - Value: !Ref ServerlessHttpApi - Statistic: Sum - Period: 300 - EvaluationPeriods: 1 - Threshold: 1 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # Latency p99 via ExtendedStatistic. ~3000ms target chosen alongside the - # Lambda Duration thresholds (Slack requires a fast 3s ack). - ApiGatewayLatencyAlarm: - Type: AWS::CloudWatch::Alarm - Properties: - AlarmName: !Sub "ApiGateway-Latency-${ServerlessHttpApi}" - AlarmDescription: "p99 latency on the afterhours HTTP API exceeded 3s" - Namespace: AWS/ApiGateway - MetricName: Latency - Dimensions: - - Name: ApiId - Value: !Ref ServerlessHttpApi - ExtendedStatistic: p99 - Period: 300 - EvaluationPeriods: 3 - DatapointsToAlarm: 2 - Threshold: 3000 - ComparisonOperator: GreaterThanOrEqualToThreshold - TreatMissingData: notBreaching - AlarmActions: - - !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts" - - # --- CloudWatch Log Groups (explicit 60-day retention) --- - SlackBotLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${SlackBotFunction}" - RetentionInDays: 60 - - WeeklyPostLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${WeeklyPostFunction}" - RetentionInDays: 60 - - RosterSyncLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${RosterSyncFunction}" - RetentionInDays: 60 - - RosterApiLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${RosterApiFunction}" - RetentionInDays: 60 - - RingSchedulerLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${RingSchedulerFunction}" - RetentionInDays: 60 - - HolidayRouterLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${HolidayRouterFunction}" - RetentionInDays: 60 - - ReleaseNotifierLogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${ReleaseNotifierFunction}" - RetentionInDays: 60 - -Outputs: - SlackBotApiUrl: - Description: URL for Slack app Request URL configuration - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/slack/events" - AfterhoursApiBaseUrl: - Description: Origin for AFTERHOURS_BASE_URL (no /mgmt or /roster suffix) - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" - ShiftTableName: - Value: !Ref ShiftTable - SlackBotFunctionArn: - Value: !GetAtt SlackBotFunction.Arn - WeeklyPostFunctionArn: - Value: !GetAtt WeeklyPostFunction.Arn - RosterSyncFunctionArn: - Value: !GetAtt RosterSyncFunction.Arn - RingSchedulerFunctionArn: - Value: !GetAtt RingSchedulerFunction.Arn - HolidayRouterFunctionArn: - Value: !GetAtt HolidayRouterFunction.Arn - HolidaySchedulerExecutionRoleArn: - Description: Role EventBridge Scheduler assumes to invoke the holiday router; the slack-bot passes this when creating per-holiday schedules - Value: !GetAtt HolidaySchedulerExecutionRole.Arn - ReleaseNotifierFunctionArn: - Value: !GetAtt ReleaseNotifierFunction.Arn - ReleaseNotifyInvokeRoleArn: - Description: Set this as the RELEASE_NOTIFY_INVOKE_ROLE_ARN repo variable for release.yaml - Value: !GetAtt ReleaseNotifyInvokeRole.Arn diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..5422af0 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,47 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.8.1" + constraints = "~> 2.8" + hashes = [ + "h1:aLNmq6dc3cDcqZc8s/8eKtn0I+UQXyJMGrmo4rRFtNw=", + "zh:03de290604114a89fcd45c2e5bc7787d5a1ebfc5f964fb5989306bea7a4c79ec", + "zh:0a7d69dc9fbbc48960bc2f04588c8fb1bd92c78a8f306566b7fb17fc4a4f2058", + "zh:4df1f3981379c35f1757da957470f7f7724496b57219da3485177cf1647bdf59", + "zh:50f0e72ba53bfe6e11b03b7fc899e1f72536354381d302a743a274ae2a3f45f4", + "zh:5c4e15a04c98e2a8cafb1cd9632b48ad318051e8462d105b1153006277985c35", + "zh:66069e604bcf5c4af0278e15997d9e6bd755c54fb3801d78885838b889729c5f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:979765db3f42601870ab377104ba70befb029547b278337ec4ada980e3582de6", + "zh:b165254da774f49945a73fbccc3ef1b63d70ea00a98fa9e14716665ba80ecaad", + "zh:c0bb2697b525da9fec4511f569ed2bd2b42f25d5c1f9fa00f8f3645b50cfc2e9", + "zh:c48f6695d12df0d0fa5231f7c1b8d518a4feae91a733fdd35a5804af3a303831", + "zh:d589954c93f075180c9f4e2ce91780d5edcb56dfd0d3cda8ba08e13c10b52249", + "zh:f5792ed06da65d0daf7ca3711f5399ff78c7cb4400afe53fbce9a926fbde4477", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.64.0" + constraints = "~> 6.64" + hashes = [ + "h1:wXARLY+IeQ7ufYxCLTPCwToWGMRvOpiOTfJS97iwUzI=", + "zh:07172315d67bc9781240272759cdfc7bd32b7e72384a56862c2c1da3cca99a81", + "zh:154ce7d2659de9a59ddfe96d7cab41a9ddc2cb267a7d4bcdf4e737ff2ffdec06", + "zh:17324d4335a7a7ac01cc23eded530775606680ff53b47cb74a3cb95d1121f836", + "zh:307ab92324ec5a61b124881ab8cac1d9e316f4527dfd0e1b59794c229407eb4e", + "zh:31e25f1903661332e36a95283042dd3ec50b47c186db00663fbd976a11e6a6b2", + "zh:3311d9f3bd12a24886027dbe73859dcd1e67bd0e3046227a338cf2c7ca04d18e", + "zh:37916156a3aac3b29be3acebd15d53145ea4ab5d4aaa825eaebe75481fa00500", + "zh:4158cb8c38b3ac6aa98eb15935ec6bd7c30838d85d2b00acc9812df8382ae908", + "zh:5bfb9499c66d9db5b34dc5c60f426a1ab1baa5457ce2aefebca826a9c3f92fb0", + "zh:6eb29ead5a4aca3b1f35812e7e8c75419180e1928e479b458f206861277736db", + "zh:7a82b6dd0c0cdef8045a4adfbddd36acb86b6b23fcbed8e189c2d71f7dc4a502", + "zh:9556bd792032c3f7e73ea4dd08cec88dc1327f5a4a57d79c30ba844ae2b9a3c0", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c5234180464cb800c83a41f57462742b802c150ad7d4417626fcd9cb511c01d2", + "zh:cd776b83b1f7b36635957350afe7ce28ba4e4ea3a5e2deb00d13dbd3b35d9d40", + "zh:fb583a7b791c6f915b86573d04f05ddbf7f1a5e4120c5d8a7450a3086c1225c4", + ] +} diff --git a/terraform/alarms.tf b/terraform/alarms.tf new file mode 100644 index 0000000..24c0a98 --- /dev/null +++ b/terraform/alarms.tf @@ -0,0 +1,155 @@ +locals { + lambda_alarm_matrix = { + errors = { + metric_name = "Errors" + statistic = "Sum" + evaluation_periods = 1 + datapoints_to_alarm = 1 + threshold = 1 + comparison = "GreaterThanOrEqualToThreshold" + period = 300 + } + throttles = { + metric_name = "Throttles" + statistic = "Sum" + evaluation_periods = 1 + datapoints_to_alarm = 1 + threshold = 1 + comparison = "GreaterThanOrEqualToThreshold" + period = 300 + } + } + + lambda_alarms = { + for pair in flatten([ + for fn_key, fn in local.functions : [ + for metric_key, metric in local.lambda_alarm_matrix : { + key = "${fn_key}-${metric_key}" + fn_key = fn_key + function = fn.function_name + metric_key = metric_key + metric_name = metric.metric_name + statistic = metric.statistic + evaluation = metric.evaluation_periods + datapoints = metric.datapoints_to_alarm + threshold = metric.threshold + comparison = metric.comparison + period = metric.period + description = metric_key == "errors" ? "${fn.function_name} reported one or more errors" : "${fn.function_name} was throttled (concurrency limit hit)" + } + ] + ]) : pair.key => pair + } +} + +resource "aws_cloudwatch_metric_alarm" "lambda_errors_throttles" { + for_each = local.lambda_alarms + + alarm_name = "Lambda-${title(each.value.metric_key)}-${each.value.function}" + alarm_description = each.value.description + namespace = "AWS/Lambda" + metric_name = each.value.metric_name + dimensions = { FunctionName = each.value.function } + statistic = each.value.statistic + period = each.value.period + evaluation_periods = each.value.evaluation + datapoints_to_alarm = each.value.datapoints + threshold = each.value.threshold + comparison_operator = each.value.comparison + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "lambda_duration" { + for_each = local.functions + + alarm_name = "Lambda-Duration-${each.value.function_name}" + alarm_description = "${each.value.function_name} duration approaching its ${each.value.timeout}s timeout (>=${each.value.duration_ms}ms)" + namespace = "AWS/Lambda" + metric_name = "Duration" + dimensions = { FunctionName = each.value.function_name } + statistic = "Maximum" + period = 300 + evaluation_periods = 3 + datapoints_to_alarm = 2 + threshold = each.value.duration_ms + comparison_operator = "GreaterThanOrEqualToThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "ddb_read_throttle" { + alarm_name = "DDB-ReadThrottle-${local.table_name}" + alarm_description = "afterhours-shifts table had one or more read throttle events" + namespace = "AWS/DynamoDB" + metric_name = "ReadThrottleEvents" + dimensions = { TableName = aws_dynamodb_table.shifts.name } + statistic = "Sum" + period = 300 + evaluation_periods = 1 + threshold = 0 + comparison_operator = "GreaterThanThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "ddb_write_throttle" { + alarm_name = "DDB-WriteThrottle-${local.table_name}" + alarm_description = "afterhours-shifts table had one or more write throttle events" + namespace = "AWS/DynamoDB" + metric_name = "WriteThrottleEvents" + dimensions = { TableName = aws_dynamodb_table.shifts.name } + statistic = "Sum" + period = 300 + evaluation_periods = 1 + threshold = 0 + comparison_operator = "GreaterThanThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "api_4xx" { + alarm_name = "ApiGateway-4xx-${aws_apigatewayv2_api.http.id}" + alarm_description = "Elevated 4xx responses on the afterhours HTTP API" + namespace = "AWS/ApiGateway" + metric_name = "4xx" + dimensions = { ApiId = aws_apigatewayv2_api.http.id } + statistic = "Sum" + period = 300 + evaluation_periods = 1 + threshold = 5 + comparison_operator = "GreaterThanOrEqualToThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "api_5xx" { + alarm_name = "ApiGateway-5xx-${aws_apigatewayv2_api.http.id}" + alarm_description = "5xx responses on the afterhours HTTP API" + namespace = "AWS/ApiGateway" + metric_name = "5xx" + dimensions = { ApiId = aws_apigatewayv2_api.http.id } + statistic = "Sum" + period = 300 + evaluation_periods = 1 + threshold = 1 + comparison_operator = "GreaterThanOrEqualToThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} + +resource "aws_cloudwatch_metric_alarm" "api_latency" { + alarm_name = "ApiGateway-Latency-${aws_apigatewayv2_api.http.id}" + alarm_description = "p99 latency on the afterhours HTTP API exceeded 3s" + namespace = "AWS/ApiGateway" + metric_name = "Latency" + dimensions = { ApiId = aws_apigatewayv2_api.http.id } + extended_statistic = "p99" + period = 300 + evaluation_periods = 3 + datapoints_to_alarm = 2 + threshold = 3000 + comparison_operator = "GreaterThanOrEqualToThreshold" + treat_missing_data = "notBreaching" + alarm_actions = [local.site_alerts_arn] +} diff --git a/terraform/apigateway.tf b/terraform/apigateway.tf new file mode 100644 index 0000000..92df917 --- /dev/null +++ b/terraform/apigateway.tf @@ -0,0 +1,82 @@ +# HTTP API: Slack events plus the Paychex roster contract. + +resource "aws_apigatewayv2_api" "http" { + name = local.project + protocol_type = "HTTP" + description = "afterhours-shift-manager Slack and roster API" +} + +resource "aws_apigatewayv2_integration" "slack_bot" { + api_id = aws_apigatewayv2_api.http.id + integration_type = "AWS_PROXY" + integration_method = "POST" + integration_uri = aws_lambda_function.this["slack_bot"].invoke_arn + payload_format_version = "2.0" + timeout_milliseconds = 30000 +} + +resource "aws_apigatewayv2_integration" "roster_api" { + api_id = aws_apigatewayv2_api.http.id + integration_type = "AWS_PROXY" + integration_method = "POST" + integration_uri = aws_lambda_function.this["roster_api"].invoke_arn + payload_format_version = "2.0" + timeout_milliseconds = 30000 +} + +resource "aws_apigatewayv2_route" "slack_events" { + api_id = aws_apigatewayv2_api.http.id + route_key = "POST /slack/events" + target = "integrations/${aws_apigatewayv2_integration.slack_bot.id}" +} + +resource "aws_apigatewayv2_route" "put_roster" { + api_id = aws_apigatewayv2_api.http.id + route_key = "PUT /roster" + target = "integrations/${aws_apigatewayv2_integration.roster_api.id}" +} + +resource "aws_apigatewayv2_route" "delete_roster" { + api_id = aws_apigatewayv2_api.http.id + route_key = "DELETE /roster/{extension}" + target = "integrations/${aws_apigatewayv2_integration.roster_api.id}" +} + +resource "aws_apigatewayv2_stage" "default" { + api_id = aws_apigatewayv2_api.http.id + name = "$default" + auto_deploy = true + + access_log_settings { + destination_arn = aws_cloudwatch_log_group.api_access.arn + format = "{\"requestId\":\"$context.requestId\",\"ip\":\"$context.identity.sourceIp\",\"requestTime\":\"$context.requestTime\",\"method\":\"$context.httpMethod\",\"routeKey\":\"$context.routeKey\",\"status\":\"$context.status\",\"protocol\":\"$context.protocol\",\"responseLength\":\"$context.responseLength\",\"integrationError\":\"$context.integrationErrorMessage\"}" + } + + default_route_settings { + throttling_burst_limit = 50 + throttling_rate_limit = 100 + } + + depends_on = [ + aws_apigatewayv2_route.slack_events, + aws_apigatewayv2_route.put_roster, + aws_apigatewayv2_route.delete_roster, + aws_iam_role_policy.hcptf_apply_services, + ] +} + +resource "aws_lambda_permission" "api_slack_bot" { + statement_id = "AllowApiGatewayInvokeSlackBot" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.this["slack_bot"].function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.http.execution_arn}/*/*" +} + +resource "aws_lambda_permission" "api_roster_api" { + statement_id = "AllowApiGatewayInvokeRosterApi" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.this["roster_api"].function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.http.execution_arn}/*/*" +} diff --git a/terraform/artifacts.tf b/terraform/artifacts.tf new file mode 100644 index 0000000..bd85a0e --- /dev/null +++ b/terraform/artifacts.tf @@ -0,0 +1,118 @@ +# Lambda artifacts bucket. Terraform ships only the bootstrap stub. +# .github/workflows/deploy.yaml uploads functions//.zip and calls +# update-function-code. Functions ignore code attributes afterwards. + +resource "aws_s3_bucket" "artifacts" { + bucket = local.artifacts_bucket_name + + tags = { + Purpose = "Lambda deployment packages for afterhours-shift-manager" + } +} + +resource "aws_s3_bucket_public_access_block" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_ownership_controls" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_versioning" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + + rule { + id = "expire-noncurrent-packages" + status = "Enabled" + + filter {} + + noncurrent_version_expiration { + noncurrent_days = 180 + } + } + + rule { + id = "abort-incomplete-multipart" + status = "Enabled" + + filter {} + + abort_incomplete_multipart_upload { + days_after_initiation = 7 + } + } + + depends_on = [aws_s3_bucket_versioning.artifacts] +} + +data "aws_iam_policy_document" "artifacts" { + statement { + sid = "DenyInsecureTransport" + effect = "Deny" + + principals { + type = "*" + identifiers = ["*"] + } + + actions = ["s3:*"] + resources = [ + aws_s3_bucket.artifacts.arn, + "${aws_s3_bucket.artifacts.arn}/*", + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + +resource "aws_s3_bucket_policy" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + policy = data.aws_iam_policy_document.artifacts.json + + depends_on = [aws_s3_bucket_public_access_block.artifacts] +} + +data "archive_file" "bootstrap_stub" { + type = "zip" + source_dir = "${path.module}/bootstrap/stub" + output_path = "${path.module}/build/packages/bootstrap-stub.zip" +} + +resource "aws_s3_object" "bootstrap_stub" { + bucket = aws_s3_bucket.artifacts.id + key = "functions/bootstrap-stub.zip" + content_base64 = filebase64(data.archive_file.bootstrap_stub.output_path) + source_hash = data.archive_file.bootstrap_stub.output_base64sha256 +} diff --git a/terraform/bootstrap/stub/handler.py b/terraform/bootstrap/stub/handler.py new file mode 100644 index 0000000..bd91b91 --- /dev/null +++ b/terraform/bootstrap/stub/handler.py @@ -0,0 +1,9 @@ +"""Bootstrap stub. GitHub Actions replaces this zip via update-function-code.""" + + +def handler(event, context): + return { + "statusCode": 503, + "headers": {"content-type": "application/json"}, + "body": '{"error":{"code":"NOT_DEPLOYED","message":"Function code has not been deployed yet."}}', + } diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf new file mode 100644 index 0000000..4dfaacb --- /dev/null +++ b/terraform/dynamodb.tf @@ -0,0 +1,21 @@ +resource "aws_dynamodb_table" "shifts" { + name = local.table_name + billing_mode = "PAY_PER_REQUEST" + hash_key = "PK" + range_key = "SK" + + attribute { + name = "PK" + type = "S" + } + + attribute { + name = "SK" + type = "S" + } + + ttl { + attribute_name = "expires_at" + enabled = true + } +} diff --git a/terraform/events.tf b/terraform/events.tf new file mode 100644 index 0000000..12203ab --- /dev/null +++ b/terraform/events.tf @@ -0,0 +1,76 @@ +# EventBridge schedules. Every schedule is an EST/EDT pair firing the same +# function one hour apart in UTC: EventBridge cron has no timezone. Both fire +# year-round and the handlers are idempotent. Keep schedules_enabled=false +# until Slack and Paychex point at this stack. + +locals { + schedules = { + weekly-post-est = { + description = "Post weekly schedule Monday 7am EST" + schedule = "cron(0 12 ? * MON *)" + function_key = "weekly_post" + } + weekly-post-edt = { + description = "Post weekly schedule Monday 7am EDT" + schedule = "cron(0 11 ? * MON *)" + function_key = "weekly_post" + } + roster-sync-est = { + description = "Sync roster from 3CX at 6am EST" + schedule = "cron(0 11 ? * * *)" + function_key = "roster_sync" + } + roster-sync-edt = { + description = "Sync roster from 3CX at 6am EDT" + schedule = "cron(0 10 ? * * *)" + function_key = "roster_sync" + } + ring-scheduler-daily-est = { + description = "Update 3CX queue at 8am EST" + schedule = "cron(0 13 ? * * *)" + function_key = "ring_scheduler" + } + ring-scheduler-daily-edt = { + description = "Update 3CX queue at 8am EDT" + schedule = "cron(0 12 ? * * *)" + function_key = "ring_scheduler" + } + ring-scheduler-weekend-est = { + description = "Update 3CX queue at 5pm EST weekends" + schedule = "cron(0 22 ? * SAT,SUN *)" + function_key = "ring_scheduler" + } + ring-scheduler-weekend-edt = { + description = "Update 3CX queue at 5pm EDT weekends" + schedule = "cron(0 21 ? * SAT,SUN *)" + function_key = "ring_scheduler" + } + } +} + +resource "aws_cloudwatch_event_rule" "schedule" { + for_each = local.schedules + + name = "${local.project}-${each.key}" + description = each.value.description + schedule_expression = each.value.schedule + state = var.schedules_enabled ? "ENABLED" : "DISABLED" +} + +resource "aws_cloudwatch_event_target" "schedule" { + for_each = local.schedules + + rule = aws_cloudwatch_event_rule.schedule[each.key].name + target_id = "${local.project}-${each.key}" + arn = aws_lambda_function.this[each.value.function_key].arn +} + +resource "aws_lambda_permission" "schedule" { + for_each = local.schedules + + statement_id = "AllowEventBridgeInvoke-${each.key}" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.this[each.value.function_key].function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.schedule[each.key].arn +} diff --git a/terraform/hcp_iam.tf b/terraform/hcp_iam.tf new file mode 100644 index 0000000..1ac2b8c --- /dev/null +++ b/terraform/hcp_iam.tf @@ -0,0 +1,843 @@ +# HCP plan/apply roles for afterhours-shift-manager-prod (PLAT-74 / PLAT-144). +# Copy of seahaven-org-baseline/examples/hcptf-workspace-iam/hcp_iam.tf.example +# with the afterhours service set. Create, do not import. +# +# Live seahaven-hcptf-iam-management DenySelfMutation blocks DetachRolePolicy +# and PutRolePolicy on hcptf-* (including this role). First-apply sequence: +# 1. seahaven-org-baseline scripts/create-hcptf-bootstrap-roles.sh +# --account prod --allow-workspace afterhours-shift-manager-prod +# 2. Point this workspace's TFC_AWS_* at hcptf-bootstrap / +# hcptf-bootstrap-plan (workspace vars, never a project set). +# 3. One Manual apply (create roles + scoped inline + boundary + stack, +# schedules_enabled=false). +# 4. Point TFC_AWS_* back at hcptf-afterhours-shift-manager / +# hcptf-afterhours-shift-manager-plan. +# 5. Re-run the script without --allow-workspace to pin trust back to +# iam-bootstrap-prod only. +# Later apply-role IAM edits use the same window. Do not add StringLike +# on bootstrap trust. CreatePolicy stays on hcptf-bootstrap only; boundary +# document changes after seal also need that window. + +data "aws_iam_policy_document" "hcptf_apply_trust" { + statement { + sid = "HcpApply" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = ["arn:aws:iam::${local.account_id}:oidc-provider/app.terraform.io"] + } + + condition { + test = "StringEquals" + variable = "app.terraform.io:aud" + values = ["aws.workload.identity"] + } + + condition { + test = "StringEquals" + variable = "app.terraform.io:sub" + values = [ + "organization:seahaven:project:${local.hcp_project}:workspace:${local.hcp_workspace}:run_phase:apply", + ] + } + } +} + +data "aws_iam_policy_document" "hcptf_plan_trust" { + statement { + sid = "HcpPlan" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = ["arn:aws:iam::${local.account_id}:oidc-provider/app.terraform.io"] + } + + condition { + test = "StringEquals" + variable = "app.terraform.io:aud" + values = ["aws.workload.identity"] + } + + condition { + test = "StringEquals" + variable = "app.terraform.io:sub" + values = [ + "organization:seahaven:project:${local.hcp_project}:workspace:${local.hcp_workspace}:run_phase:plan", + ] + } + } +} + +data "aws_iam_policy_document" "hcptf_scoped_iam" { + statement { + sid = "DenyCreatePolicy" + effect = "Deny" + actions = [ + "iam:CreatePolicy", + "iam:CreatePolicyVersion", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:SetDefaultPolicyVersion", + ] + resources = ["*"] + } + + statement { + sid = "CreateExecRoleWithBoundary" + effect = "Allow" + actions = ["iam:CreateRole"] + resources = [ + "arn:aws:iam::${local.account_id}:role/tf-managed/${local.stack_prefix}*", + ] + + condition { + test = "StringLike" + variable = "iam:PermissionsBoundary" + values = [ + "arn:aws:iam::${local.account_id}:policy/tf-managed/${local.stack_prefix}*", + "arn:aws:iam::${local.account_id}:policy/seahaven-lambda-execution-boundary", + "arn:aws:iam::${local.account_id}:policy/seahaven-lambda-execution-boundary-${local.stack_name}", + ] + } + } + + statement { + sid = "MutateExecRoleWithBoundary" + effect = "Allow" + actions = [ + "iam:AttachRolePolicy", + "iam:PutRolePolicy", + "iam:PutRolePermissionsBoundary", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/tf-managed/${local.stack_prefix}*", + ] + + condition { + test = "StringLike" + variable = "iam:PermissionsBoundary" + values = [ + "arn:aws:iam::${local.account_id}:policy/tf-managed/${local.stack_prefix}*", + "arn:aws:iam::${local.account_id}:policy/seahaven-lambda-execution-boundary", + "arn:aws:iam::${local.account_id}:policy/seahaven-lambda-execution-boundary-${local.stack_name}", + ] + } + } + + statement { + sid = "WriteExecRoles" + effect = "Allow" + actions = [ + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DetachRolePolicy", + "iam:TagRole", + "iam:UntagRole", + "iam:UpdateAssumeRolePolicy", + "iam:UpdateRole", + "iam:UpdateRoleDescription", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/tf-managed/${local.stack_prefix}*", + ] + } + + statement { + sid = "PassExecRolesToLambda" + effect = "Allow" + actions = ["iam:PassRole"] + resources = ["arn:aws:iam::${local.account_id}:role/tf-managed/${local.stack_prefix}*"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["lambda.amazonaws.com"] + } + } + + statement { + sid = "PassHolidaySchedulerRole" + effect = "Allow" + actions = ["iam:PassRole"] + resources = ["arn:aws:iam::${local.account_id}:role/tf-managed/afterhours-shift-manager-holiday-scheduler"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["scheduler.amazonaws.com"] + } + } + + # githubdeploy-afterhours-shift-manager lives at /tf-managed/ so + # DenySelfMutation (role/githubdeploy-*) does not match. Create without a + # permissions boundary; this is not a Lambda execution role. + statement { + sid = "CreateDeployRole" + effect = "Allow" + actions = ["iam:CreateRole"] + resources = ["arn:aws:iam::${local.account_id}:role/tf-managed/${local.deploy_role}"] + + condition { + test = "Null" + variable = "iam:PermissionsBoundary" + values = ["true"] + } + } + + statement { + sid = "WriteDeployRoles" + effect = "Allow" + actions = [ + "iam:AttachRolePolicy", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DetachRolePolicy", + "iam:PutRolePolicy", + "iam:TagRole", + "iam:UntagRole", + "iam:UpdateAssumeRolePolicy", + "iam:UpdateRole", + "iam:UpdateRoleDescription", + ] + resources = ["arn:aws:iam::${local.account_id}:role/tf-managed/${local.deploy_role}"] + } + + statement { + sid = "IamReadOnly" + effect = "Allow" + actions = [ + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicies", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:ListRoleTags", + "iam:ListRoles", + ] + resources = ["*"] + } + + statement { + sid = "DenySelfMutation" + effect = "Deny" + actions = [ + "iam:AttachRolePolicy", + "iam:DeleteRole", + "iam:DeleteRolePolicy", + "iam:DeleteRolePermissionsBoundary", + "iam:DetachRolePolicy", + "iam:PutRolePolicy", + "iam:PutRolePermissionsBoundary", + "iam:UpdateAssumeRolePolicy", + "iam:UpdateRole", + "iam:UpdateRoleDescription", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/hcptf-*", + "arn:aws:iam::${local.account_id}:role/github-cfn-execution-role", + "arn:aws:iam::${local.account_id}:role/githubdeploy-*", + "arn:aws:iam::${local.account_id}:role/cdk-hnb659fds-*", + "arn:aws:iam::${local.account_id}:role/OrganizationAccountAccessRole", + "arn:aws:iam::${local.account_id}:role/seahaven-*", + ] + } + + statement { + sid = "DenyBoundaryTampering" + effect = "Deny" + actions = [ + "iam:DeleteRolePermissionsBoundary", + "iam:DeleteUserPermissionsBoundary", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/*", + "arn:aws:iam::${local.account_id}:user/*", + ] + } + + statement { + sid = "DenyBoundaryPolicyEdit" + effect = "Deny" + actions = [ + "iam:CreatePolicyVersion", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:SetDefaultPolicyVersion", + ] + resources = ["arn:aws:iam::${local.account_id}:policy/seahaven-*"] + } +} + +data "aws_iam_policy_document" "hcptf_apply_services" { + statement { + sid = "LambdaAll" + effect = "Allow" + actions = [ + "lambda:*", + ] + resources = [ + "arn:aws:lambda:${var.aws_region}:${local.account_id}:function:afterhours-*", + ] + } + + statement { + sid = "LambdaList" + effect = "Allow" + actions = [ + "lambda:ListFunctions", + "lambda:ListLayers", + "lambda:GetAccountSettings", + ] + resources = ["*"] + } + + statement { + sid = "EventBridgeRules" + effect = "Allow" + actions = [ + "events:*", + ] + resources = [ + "arn:aws:events:${var.aws_region}:${local.account_id}:rule/afterhours-shift-manager-*", + ] + } + + statement { + sid = "EventBridgeList" + effect = "Allow" + actions = ["events:ListRules", "events:ListRuleNamesByTarget"] + resources = ["*"] + } + + statement { + sid = "CloudWatchLogs" + effect = "Allow" + actions = [ + "logs:CreateLogGroup", + "logs:DeleteLogGroup", + "logs:PutRetentionPolicy", + "logs:DeleteRetentionPolicy", + "logs:TagResource", + "logs:UntagResource", + "logs:ListTagsForResource", + "logs:PutMetricFilter", + "logs:DeleteMetricFilter", + "logs:DescribeMetricFilters", + ] + resources = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/lambda/afterhours-*", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/apigateway/afterhours-shift-manager", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/apigateway/afterhours-shift-manager:*", + ] + } + + statement { + sid = "CloudWatchLogsDescribe" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + + # CreateStage access_log_settings uses log-delivery APIs. Resource "*" is + # required; these actions do not accept a log-group ARN. + statement { + sid = "ApiGwAccessLogDelivery" + effect = "Allow" + actions = [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + ] + resources = ["*"] + } + + statement { + sid = "StackBuckets" + effect = "Allow" + actions = [ + "s3:*", + ] + resources = [ + "arn:aws:s3:::${local.artifacts_bucket_name}", + "arn:aws:s3:::${local.artifacts_bucket_name}/*", + ] + } + + statement { + sid = "DynamoDBTable" + effect = "Allow" + actions = [ + "dynamodb:*", + ] + resources = [ + "arn:aws:dynamodb:${var.aws_region}:${local.account_id}:table/${local.table_name}", + "arn:aws:dynamodb:${var.aws_region}:${local.account_id}:table/${local.table_name}/*", + ] + } + + statement { + sid = "DynamoDBList" + effect = "Allow" + actions = ["dynamodb:ListTables"] + resources = ["*"] + } + + statement { + sid = "HttpApiManage" + effect = "Allow" + actions = [ + "apigateway:*", + ] + resources = [ + "arn:aws:apigateway:${var.aws_region}::/apis", + "arn:aws:apigateway:${var.aws_region}::/apis/*", + "arn:aws:apigateway:${var.aws_region}::/tags/*", + "arn:aws:apigateway:${var.aws_region}::/vpclinks", + "arn:aws:apigateway:${var.aws_region}::/vpclinks/*", + ] + } + + statement { + sid = "AfterhoursSsm" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:PutParameter", + "ssm:DeleteParameter", + "ssm:AddTagsToResource", + "ssm:RemoveTagsFromResource", + "ssm:ListTagsForResource", + ] + resources = [ + "arn:aws:ssm:${var.aws_region}:${local.account_id}:parameter${local.ssm_prefix}/*", + ] + } + + statement { + sid = "SsmDescribeParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + + statement { + sid = "SecretsManagerReadAndManage" + effect = "Allow" + actions = [ + "secretsmanager:CreateSecret", + "secretsmanager:DeleteSecret", + "secretsmanager:DescribeSecret", + "secretsmanager:GetResourcePolicy", + "secretsmanager:PutResourcePolicy", + "secretsmanager:DeleteResourcePolicy", + "secretsmanager:TagResource", + "secretsmanager:UntagResource", + "secretsmanager:UpdateSecret", + "secretsmanager:ListSecretVersionIds", + ] + resources = [ + "arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/*", + ] + } + + statement { + sid = "SecretsManagerCreateByName" + effect = "Allow" + actions = [ + "secretsmanager:CreateSecret", + ] + resources = ["*"] + + condition { + test = "StringLike" + variable = "secretsmanager:Name" + values = ["afterhours-shift-manager/*"] + } + } + + statement { + sid = "SecretsManagerList" + effect = "Allow" + actions = ["secretsmanager:ListSecrets"] + resources = ["*"] + } + + statement { + sid = "CloudWatchAlarms" + effect = "Allow" + actions = [ + "cloudwatch:PutMetricAlarm", + "cloudwatch:DeleteAlarms", + "cloudwatch:DescribeAlarms", + "cloudwatch:TagResource", + "cloudwatch:UntagResource", + "cloudwatch:ListTagsForResource", + ] + resources = [ + "arn:aws:cloudwatch:${var.aws_region}:${local.account_id}:alarm:Lambda-*-afterhours-*", + "arn:aws:cloudwatch:${var.aws_region}:${local.account_id}:alarm:DDB-*-afterhours-shifts", + "arn:aws:cloudwatch:${var.aws_region}:${local.account_id}:alarm:ApiGateway-*", + ] + } + + statement { + sid = "CloudWatchDescribeAlarms" + effect = "Allow" + actions = ["cloudwatch:DescribeAlarms"] + resources = ["*"] + } + + statement { + sid = "SnsPublishSiteAlerts" + effect = "Allow" + actions = [ + "sns:Publish", + "sns:GetTopicAttributes", + "sns:ListTagsForResource", + ] + resources = [local.site_alerts_arn] + } + + statement { + sid = "ManageTfManagedBoundary" + effect = "Allow" + actions = [ + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:ListPolicyVersions", + "iam:ListPolicyTags", + "iam:TagPolicy", + "iam:UntagPolicy", + ] + resources = [ + "arn:aws:iam::${local.account_id}:policy/tf-managed/${local.stack_prefix}*", + ] + } + + statement { + sid = "EventBridgeScheduler" + effect = "Allow" + actions = [ + "scheduler:CreateSchedule", + "scheduler:DeleteSchedule", + "scheduler:GetSchedule", + "scheduler:UpdateSchedule", + "scheduler:ListTagsForResource", + "scheduler:TagResource", + "scheduler:UntagResource", + ] + resources = [ + "arn:aws:scheduler:${var.aws_region}:${local.account_id}:schedule/default/holiday-*", + ] + } + + statement { + sid = "EventBridgeSchedulerList" + effect = "Allow" + actions = [ + "scheduler:ListSchedules", + "scheduler:ListScheduleGroups", + "scheduler:GetScheduleGroup", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "hcptf_plan_refresh" { + statement { + sid = "RefreshIamRoles" + effect = "Allow" + actions = [ + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListRolePolicies", + "iam:ListAttachedRolePolicies", + "iam:ListRoleTags", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/tf-managed/${local.stack_prefix}*", + "arn:aws:iam::${local.account_id}:role/tf-managed/${local.deploy_role}", + "arn:aws:iam::${local.account_id}:role/${local.apply_role}", + "arn:aws:iam::${local.account_id}:role/${local.plan_role}", + ] + } + + statement { + sid = "RefreshManagedPolicies" + effect = "Allow" + actions = [ + "iam:GetPolicy", + "iam:GetPolicyVersion", + ] + resources = ["*"] + } + + statement { + sid = "RefreshLambda" + effect = "Allow" + actions = [ + "lambda:GetFunction", + "lambda:GetFunctionConfiguration", + "lambda:GetPolicy", + "lambda:GetFunctionCodeSigningConfig", + "lambda:GetFunctionConcurrency", + "lambda:GetFunctionEventInvokeConfig", + "lambda:GetFunctionUrlConfig", + "lambda:GetRuntimeManagementConfig", + "lambda:GetFunctionRecursionConfig", + "lambda:ListTags", + "lambda:ListVersionsByFunction", + "lambda:ListAliases", + ] + resources = [ + "arn:aws:lambda:${var.aws_region}:${local.account_id}:function:afterhours-*", + ] + } + + statement { + sid = "RefreshLambdaList" + effect = "Allow" + actions = [ + "lambda:ListFunctions", + "lambda:ListLayers", + "lambda:GetAccountSettings", + ] + resources = ["*"] + } + + statement { + sid = "RefreshBuckets" + effect = "Allow" + actions = [ + "s3:GetBucketLocation", + "s3:GetBucketAcl", + "s3:GetBucketPolicy", + "s3:GetBucketPolicyStatus", + "s3:GetBucketPublicAccessBlock", + "s3:GetBucketVersioning", + "s3:GetBucketLifecycleConfiguration", + "s3:GetBucketTagging", + "s3:GetBucketOwnershipControls", + "s3:GetEncryptionConfiguration", + "s3:GetBucketCORS", + "s3:GetBucketLogging", + "s3:GetBucketNotification", + "s3:GetBucketObjectLockConfiguration", + "s3:GetBucketRequestPayment", + "s3:GetBucketWebsite", + "s3:GetAccelerateConfiguration", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetObjectTagging", + "s3:ListBucket", + ] + resources = [ + "arn:aws:s3:::${local.artifacts_bucket_name}", + "arn:aws:s3:::${local.artifacts_bucket_name}/*", + ] + } + + statement { + sid = "RefreshDynamoDB" + effect = "Allow" + actions = [ + "dynamodb:DescribeTable", + "dynamodb:DescribeTimeToLive", + "dynamodb:DescribeContinuousBackups", + "dynamodb:DescribeKinesisStreamingDestination", + "dynamodb:ListTagsOfResource", + ] + resources = [ + "arn:aws:dynamodb:${var.aws_region}:${local.account_id}:table/${local.table_name}", + ] + } + + statement { + sid = "RefreshEventBridge" + effect = "Allow" + actions = [ + "events:DescribeRule", + "events:ListTargetsByRule", + "events:ListTagsForResource", + ] + resources = [ + "arn:aws:events:${var.aws_region}:${local.account_id}:rule/afterhours-shift-manager-*", + ] + } + + statement { + sid = "RefreshLogs" + effect = "Allow" + actions = [ + "logs:DescribeLogGroups", + "logs:ListTagsForResource", + ] + resources = ["*"] + } + + statement { + sid = "RefreshHttpApi" + effect = "Allow" + actions = [ + "apigateway:GET", + ] + resources = [ + "arn:aws:apigateway:${var.aws_region}::/apis", + "arn:aws:apigateway:${var.aws_region}::/apis/*", + "arn:aws:apigateway:${var.aws_region}::/tags/*", + ] + } + + statement { + sid = "RefreshScheduler" + effect = "Allow" + actions = [ + "scheduler:GetSchedule", + "scheduler:ListTagsForResource", + ] + resources = [ + "arn:aws:scheduler:${var.aws_region}:${local.account_id}:schedule/default/holiday-*", + ] + } + + statement { + sid = "RefreshSchedulerList" + effect = "Allow" + actions = [ + "scheduler:ListSchedules", + "scheduler:ListScheduleGroups", + "scheduler:GetScheduleGroup", + ] + resources = ["*"] + } + + statement { + sid = "RefreshSsm" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:ListTagsForResource", + ] + resources = [ + "arn:aws:ssm:${var.aws_region}:${local.account_id}:parameter${local.ssm_prefix}/*", + ] + } + + statement { + sid = "RefreshSsmDescribeParameters" + effect = "Allow" + actions = ["ssm:DescribeParameters"] + resources = ["*"] + } + + statement { + sid = "RefreshSecrets" + effect = "Allow" + actions = [ + "secretsmanager:DescribeSecret", + "secretsmanager:GetResourcePolicy", + "secretsmanager:ListSecretVersionIds", + ] + resources = [ + "arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/*", + ] + } + + statement { + sid = "RefreshSecretsList" + effect = "Allow" + actions = ["secretsmanager:ListSecrets"] + resources = ["*"] + } + + statement { + sid = "RefreshAlarms" + effect = "Allow" + actions = [ + "cloudwatch:DescribeAlarms", + "cloudwatch:ListTagsForResource", + ] + resources = ["*"] + } + + statement { + sid = "RefreshSns" + effect = "Allow" + actions = [ + "sns:GetTopicAttributes", + "sns:ListTagsForResource", + ] + resources = [local.site_alerts_arn] + } +} + +resource "aws_iam_role" "hcptf_apply" { + name = local.apply_role + assume_role_policy = data.aws_iam_policy_document.hcptf_apply_trust.json + max_session_duration = 3600 + + tags = { + Owner = "adam@seahavenind.com" + ManagedBy = "terraform" + } +} + +resource "aws_iam_role" "hcptf_plan" { + name = local.plan_role + assume_role_policy = data.aws_iam_policy_document.hcptf_plan_trust.json + max_session_duration = 3600 + + tags = { + Owner = "adam@seahavenind.com" + ManagedBy = "terraform" + } +} + +resource "aws_iam_role_policy" "hcptf_scoped_iam" { + name = "scoped-iam-management" + role = aws_iam_role.hcptf_apply.id + policy = data.aws_iam_policy_document.hcptf_scoped_iam.json +} + +resource "aws_iam_role_policy" "hcptf_apply_services" { + name = "afterhours-shift-manager-services" + role = aws_iam_role.hcptf_apply.id + policy = data.aws_iam_policy_document.hcptf_apply_services.json +} + +resource "aws_iam_role_policy" "hcptf_plan_refresh" { + name = "afterhours-shift-manager-plan-refresh" + role = aws_iam_role.hcptf_plan.id + policy = data.aws_iam_policy_document.hcptf_plan_refresh.json +} + +resource "aws_iam_role_policy_attachment" "hcptf_plan_view_only" { + role = aws_iam_role.hcptf_plan.name + policy_arn = "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess" +} + +resource "aws_iam_role_policy_attachments_exclusive" "hcptf_apply" { + role_name = aws_iam_role.hcptf_apply.name + policy_arns = [] +} + +resource "aws_iam_role_policy_attachments_exclusive" "hcptf_plan" { + role_name = aws_iam_role.hcptf_plan.name + policy_arns = [ + "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess", + ] +} diff --git a/terraform/iam_github_deploy.tf b/terraform/iam_github_deploy.tf new file mode 100644 index 0000000..fb72d1b --- /dev/null +++ b/terraform/iam_github_deploy.tf @@ -0,0 +1,101 @@ +# GitHub Actions OIDC role for .github/workflows/deploy.yaml. +# +# Trust is pinned three ways: aud, sub to Environment prod (immutable subject +# form), and job_workflow_ref to deploy.yaml at refs/heads/main only. No v* +# tags until a later release ticket. A job with environment: does not present +# ref:refs/heads/main. +# +# Not a Lambda execution role: no permissions_boundary. Path /tf-managed/ so +# seahaven-hcptf-iam-management DenySelfMutation (role/githubdeploy-*) does not +# match. + +data "aws_iam_policy_document" "github_deploy_assume" { + statement { + sid = "GithubDeployOidc" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [local.github_oidc_provider_arn] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = [local.github_oidc_sub] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:job_workflow_ref" + values = [ + "${var.github_repo}/.github/workflows/deploy.yaml@refs/heads/${var.github_deploy_branch}", + ] + } + } +} + +resource "aws_iam_role" "github_deploy" { + name = local.deploy_role + path = "/tf-managed/" + description = "GitHub Actions Lambda deploy role for ${var.github_repo} Environment prod" + assume_role_policy = data.aws_iam_policy_document.github_deploy_assume.json + max_session_duration = 3600 +} + +data "aws_iam_policy_document" "github_deploy" { + statement { + sid = "ListArtifactsBucket" + effect = "Allow" + actions = [ + "s3:GetBucketLocation", + "s3:ListBucket", + ] + resources = [aws_s3_bucket.artifacts.arn] + } + + statement { + sid = "UploadFunctionArtifacts" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = ["${aws_s3_bucket.artifacts.arn}/functions/*"] + } + + statement { + sid = "UpdateFunctionCode" + effect = "Allow" + actions = [ + "lambda:GetFunction", + "lambda:GetFunctionConfiguration", + "lambda:UpdateFunctionCode", + ] + resources = [for fn in local.functions : "arn:aws:lambda:${var.aws_region}:${local.account_id}:function:${fn.function_name}"] + } + + statement { + sid = "DeployParams" + effect = "Allow" + actions = [ + "ssm:GetParameter", + ] + resources = [ + "arn:aws:ssm:${var.aws_region}:${local.account_id}:parameter${local.ssm_prefix}/deploy/*", + ] + } +} + +resource "aws_iam_role_policy" "github_deploy" { + name = "afterhours-shift-manager-deploy" + role = aws_iam_role.github_deploy.id + policy = data.aws_iam_policy_document.github_deploy.json +} diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..6af6479 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,281 @@ +# Terraform owns the function skeletons (role, runtime, memory, environment). +# Code is owned by .github/workflows/deploy.yaml, which uploads +# functions//.zip and calls update-function-code. The lifecycle +# block is the seam: an app deploy is not drift, and a Terraform apply never +# rolls the code back to the bootstrap stub. GIT_SHA is written into +# shared/build_info.py at zip time, not set here. + +data "aws_iam_policy_document" "lambda_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +locals { + table_arn = aws_dynamodb_table.shifts.arn + + lambda_identity = { + slack_bot = [ + { + sid = "DdbCrud" + actions = ["dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DescribeTable", "dynamodb:ConditionCheckItem"] + resources = [local.table_arn, "${local.table_arn}/*"] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/*"] + condition = null + }, + { + sid = "HolidaySchedules" + actions = ["scheduler:CreateSchedule", "scheduler:DeleteSchedule", "scheduler:GetSchedule"] + resources = ["arn:aws:scheduler:${var.aws_region}:${local.account_id}:schedule/default/holiday-*"] + condition = null + }, + { + sid = "PassHolidayScheduler" + actions = ["iam:PassRole"] + resources = [local.holiday_scheduler_role_arn] + condition = { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["scheduler.amazonaws.com"] + } + }, + { + sid = "InvokeHolidayRouter" + actions = ["lambda:InvokeFunction"] + resources = [local.holiday_router_arn] + condition = null + }, + ] + weekly_post = [ + { + sid = "DdbCrud" + actions = ["dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DescribeTable", "dynamodb:ConditionCheckItem"] + resources = [local.table_arn, "${local.table_arn}/*"] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/slack-bot-token-*"] + condition = null + }, + { + sid = "CheckcomponentsSend" + actions = ["sqs:SendMessage"] + resources = [var.checkcomponents_queue_arn] + condition = null + }, + ] + roster_sync = [ + { + sid = "DdbCrud" + actions = ["dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DescribeTable", "dynamodb:ConditionCheckItem"] + resources = [local.table_arn, "${local.table_arn}/*"] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/3cx-*"] + condition = null + }, + ] + roster_api = [ + { + sid = "DdbWrite" + actions = ["dynamodb:UpdateItem", "dynamodb:DeleteItem"] + resources = [local.table_arn] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/roster-api-token-*"] + condition = null + }, + ] + ring_scheduler = [ + { + sid = "DdbRead" + actions = ["dynamodb:GetItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchGetItem", "dynamodb:DescribeTable"] + resources = [local.table_arn, "${local.table_arn}/*"] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/3cx-*"] + condition = null + }, + ] + holiday_router = [ + { + sid = "DdbCrud" + actions = ["dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem", "dynamodb:Query", "dynamodb:Scan", "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DescribeTable", "dynamodb:ConditionCheckItem"] + resources = [local.table_arn, "${local.table_arn}/*"] + condition = null + }, + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/3cx-*"] + condition = null + }, + ] + release_notifier = [ + { + sid = "Secrets" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/slack-bot-token-*"] + condition = null + }, + ] + } + + lambda_env = { + slack_bot = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + SLACK_BOT_TOKEN_SECRET = "afterhours-shift-manager/slack-bot-token" + SLACK_SIGNING_SECRET = "afterhours-shift-manager/slack-signing-secret" + SHIFT_CHANNEL = var.shift_channel + TCX_SECRET_PREFIX = "afterhours-shift-manager/3cx-" + QUEUE_NUMBER = var.queue_number + TZ = var.timezone + HOLIDAY_ROUTER_ARN = local.holiday_router_arn + HOLIDAY_SCHEDULER_ROLE_ARN = local.holiday_scheduler_role_arn + SENTRY_DSN = var.sentry_dsn + } + weekly_post = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + SLACK_BOT_TOKEN_SECRET = "afterhours-shift-manager/slack-bot-token" + SHIFT_CHANNEL = var.shift_channel + PAY_REPORT_USER = var.pay_report_user + TZ = var.timezone + CHECKCOMPONENTS_QUEUE_URL = var.checkcomponents_queue_url + SENTRY_DSN = var.sentry_dsn + } + roster_sync = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + TCX_SECRET_PREFIX = "afterhours-shift-manager/3cx-" + SYNC_GROUP = "DEFAULT" + TZ = var.timezone + SENTRY_DSN = var.sentry_dsn + } + roster_api = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + ROSTER_API_TOKEN_SECRET = "afterhours-shift-manager/roster-api-token" + TZ = var.timezone + SENTRY_DSN = var.sentry_dsn + } + ring_scheduler = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + TCX_SECRET_PREFIX = "afterhours-shift-manager/3cx-" + QUEUE_NUMBER = var.queue_number + TZ = var.timezone + SENTRY_DSN = var.sentry_dsn + } + holiday_router = { + SHIFT_TABLE = aws_dynamodb_table.shifts.name + TCX_SECRET_PREFIX = "afterhours-shift-manager/3cx-" + TZ = var.timezone + SENTRY_DSN = var.sentry_dsn + } + release_notifier = { + SLACK_BOT_TOKEN_SECRET = "afterhours-shift-manager/slack-bot-token" + SHIFT_CHANNEL = var.shift_channel + TZ = var.timezone + SENTRY_DSN = var.sentry_dsn + } + } +} + +resource "aws_iam_role" "lambda" { + for_each = local.functions + + name = each.value.role_name + path = "/tf-managed/" + description = "Lambda execution role for ${each.value.function_name}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume.json + permissions_boundary = aws_iam_policy.lambda_boundary.arn +} + +data "aws_iam_policy_document" "lambda" { + for_each = local.functions + + dynamic "statement" { + for_each = local.lambda_identity[each.key] + + content { + sid = statement.value.sid + effect = "Allow" + actions = statement.value.actions + resources = statement.value.resources + + dynamic "condition" { + for_each = try(statement.value.condition, null) == null ? [] : [statement.value.condition] + + content { + test = condition.value.test + variable = condition.value.variable + values = condition.value.values + } + } + } + } +} + +resource "aws_iam_role_policy" "lambda" { + for_each = local.functions + + name = each.key + role = aws_iam_role.lambda[each.key].id + policy = data.aws_iam_policy_document.lambda[each.key].json +} + +resource "aws_iam_role_policy_attachment" "lambda_basic" { + for_each = local.functions + + role = aws_iam_role.lambda[each.key].name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_lambda_function" "this" { + for_each = local.functions + + function_name = each.value.function_name + role = aws_iam_role.lambda[each.key].arn + handler = each.value.handler + runtime = "python3.12" + architectures = ["arm64"] + memory_size = 1024 + timeout = each.value.timeout + + s3_bucket = aws_s3_bucket.artifacts.id + s3_key = aws_s3_object.bootstrap_stub.key + source_code_hash = data.archive_file.bootstrap_stub.output_base64sha256 + + environment { + variables = local.lambda_env[each.key] + } + + lifecycle { + ignore_changes = [filename, s3_bucket, s3_key, s3_object_version, source_code_hash] + } + + depends_on = [ + aws_cloudwatch_log_group.lambda, + aws_iam_role_policy.lambda, + aws_iam_role_policy_attachment.lambda_basic, + ] +} diff --git a/terraform/lambda_boundary.tf b/terraform/lambda_boundary.tf new file mode 100644 index 0000000..ff6aac4 --- /dev/null +++ b/terraform/lambda_boundary.tf @@ -0,0 +1,141 @@ +# Per-workload Lambda permissions boundary. Created on the first (bootstrap) +# apply. The scoped apply role denies iam:CreatePolicy / CreatePolicyVersion, +# so later edits to this document need the hcptf-bootstrap window. + +data "aws_iam_policy_document" "lambda_boundary" { + statement { + sid = "CloudWatchLogsWrite" + effect = "Allow" + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + ] + resources = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/lambda*", + ] + } + + statement { + sid = "CloudWatchLogsDescribe" + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + + statement { + sid = "XRay" + effect = "Allow" + actions = [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ] + resources = ["*"] + } + + statement { + sid = "Ec2Eni" + effect = "Allow" + actions = [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeVpcs", + ] + resources = ["*"] + } + + statement { + sid = "AfterhoursSecrets" + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue", + ] + resources = [ + "arn:aws:secretsmanager:${var.aws_region}:${local.account_id}:secret:afterhours-shift-manager/*", + ] + } + + statement { + sid = "AfterhoursDynamoDB" + effect = "Allow" + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:Query", + "dynamodb:Scan", + "dynamodb:BatchGetItem", + "dynamodb:BatchWriteItem", + "dynamodb:DescribeTable", + "dynamodb:ConditionCheckItem", + ] + resources = [ + "arn:aws:dynamodb:${var.aws_region}:${local.account_id}:table/${local.table_name}", + "arn:aws:dynamodb:${var.aws_region}:${local.account_id}:table/${local.table_name}/*", + ] + } + + statement { + sid = "AfterhoursScheduler" + effect = "Allow" + actions = [ + "scheduler:CreateSchedule", + "scheduler:DeleteSchedule", + "scheduler:GetSchedule", + ] + resources = [ + "arn:aws:scheduler:${var.aws_region}:${local.account_id}:schedule/default/holiday-*", + ] + } + + statement { + sid = "AfterhoursPassRoleScheduler" + effect = "Allow" + actions = [ + "iam:PassRole", + ] + resources = [ + "arn:aws:iam::${local.account_id}:role/tf-managed/afterhours-shift-manager-holiday-scheduler", + ] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["scheduler.amazonaws.com"] + } + } + + statement { + sid = "AfterhoursInvokeHolidayRouter" + effect = "Allow" + actions = [ + "lambda:InvokeFunction", + ] + resources = [ + "arn:aws:lambda:${var.aws_region}:${local.account_id}:function:afterhours-holiday-router", + ] + } + + statement { + sid = "AfterhoursCheckcomponentsSend" + effect = "Allow" + actions = [ + "sqs:SendMessage", + ] + resources = [ + var.checkcomponents_queue_arn, + ] + } +} + +resource "aws_iam_policy" "lambda_boundary" { + name = "afterhours-shift-manager-lambda-boundary" + path = "/tf-managed/" + description = "Per-workload Lambda permissions boundary for afterhours-shift-manager (PLAT-74)." + policy = data.aws_iam_policy_document.lambda_boundary.json +} diff --git a/terraform/locals.tf b/terraform/locals.tf new file mode 100644 index 0000000..7862357 --- /dev/null +++ b/terraform/locals.tf @@ -0,0 +1,87 @@ +locals { + project = "afterhours-shift-manager" + account_id = "011934824531" + environment = "prod" + + hcp_project = "seahaven-prod" + hcp_workspace = "afterhours-shift-manager-prod" + apply_role = "hcptf-afterhours-shift-manager" + plan_role = "hcptf-afterhours-shift-manager-plan" + deploy_role = "githubdeploy-afterhours-shift-manager" + stack_name = local.project + stack_prefix = "afterhours-shift-manager-" + + artifacts_bucket_name = "afterhours-shift-manager-artifacts-${local.account_id}" + ssm_prefix = "/afterhours-shift-manager" + table_name = "afterhours-shifts" + site_alerts_arn = "arn:aws:sns:${var.aws_region}:${local.account_id}:site-alerts" + + github_oidc_provider_arn = "arn:aws:iam::${local.account_id}:oidc-provider/token.actions.githubusercontent.com" + # Org has Actions OIDC use_immutable_subject=true. + github_oidc_sub = "repo:Sea-Haven-Industries@183236204/afterhours-shift-manager@1200846448:environment:prod" + + secret_names = [ + "afterhours-shift-manager/slack-bot-token", + "afterhours-shift-manager/slack-signing-secret", + "afterhours-shift-manager/3cx-domain", + "afterhours-shift-manager/3cx-client-id", + "afterhours-shift-manager/3cx-client-secret", + "afterhours-shift-manager/roster-api-token", + ] + + holiday_router_arn = "arn:aws:lambda:${var.aws_region}:${local.account_id}:function:afterhours-holiday-router" + holiday_scheduler_role_name = "afterhours-shift-manager-holiday-scheduler" + holiday_scheduler_role_arn = "arn:aws:iam::${local.account_id}:role/tf-managed/${local.holiday_scheduler_role_name}" + + functions = { + slack_bot = { + function_name = "afterhours-shift-manager" + role_name = "afterhours-shift-manager-slack-bot" + handler = "handler.handler" + timeout = 30 + duration_ms = 24000 + } + weekly_post = { + function_name = "afterhours-weekly-post" + role_name = "afterhours-shift-manager-weekly-post" + handler = "app.handler" + timeout = 30 + duration_ms = 24000 + } + roster_sync = { + function_name = "afterhours-roster-sync" + role_name = "afterhours-shift-manager-roster-sync" + handler = "app.handler" + timeout = 60 + duration_ms = 48000 + } + roster_api = { + function_name = "afterhours-roster-api" + role_name = "afterhours-shift-manager-roster-api" + handler = "app.handler" + timeout = 30 + duration_ms = 24000 + } + ring_scheduler = { + function_name = "afterhours-ring-scheduler" + role_name = "afterhours-shift-manager-ring-scheduler" + handler = "app.handler" + timeout = 60 + duration_ms = 48000 + } + holiday_router = { + function_name = "afterhours-holiday-router" + role_name = "afterhours-shift-manager-holiday-router" + handler = "app.handler" + timeout = 60 + duration_ms = 48000 + } + release_notifier = { + function_name = "afterhours-release-notifier" + role_name = "afterhours-shift-manager-release-notifier" + handler = "app.handler" + timeout = 30 + duration_ms = 24000 + } + } +} diff --git a/terraform/logs.tf b/terraform/logs.tf new file mode 100644 index 0000000..e2e52d0 --- /dev/null +++ b/terraform/logs.tf @@ -0,0 +1,11 @@ +resource "aws_cloudwatch_log_group" "lambda" { + for_each = local.functions + + name = "/aws/lambda/${each.value.function_name}" + retention_in_days = 60 +} + +resource "aws_cloudwatch_log_group" "api_access" { + name = "/aws/apigateway/${local.project}" + retention_in_days = 90 +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..fd44d8a --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,39 @@ +output "slack_request_url" { + description = "Slack app Request URL (slash command and interactivity)." + value = "${aws_apigatewayv2_api.http.api_endpoint}/slack/events" +} + +output "api_origin" { + description = "HTTP API origin for Paychex AFTERHOURS_BASE_URL. No /roster suffix." + value = aws_apigatewayv2_api.http.api_endpoint +} + +output "shift_table_name" { + description = "DynamoDB table name." + value = aws_dynamodb_table.shifts.name +} + +output "holiday_scheduler_role_arn" { + description = "Role EventBridge Scheduler assumes to invoke the holiday router." + value = aws_iam_role.holiday_scheduler.arn +} + +output "github_deploy_role_arn" { + description = "OIDC role ARN for .github/workflows/deploy.yaml (GitHub Environment prod variable DEPLOY_ROLE_ARN)." + value = aws_iam_role.github_deploy.arn +} + +output "artifacts_bucket_name" { + description = "Lambda artifacts bucket. deploy.yaml uploads functions//.zip." + value = aws_s3_bucket.artifacts.id +} + +output "hcptf_apply_role_arn" { + description = "HCP apply role ARN. Set TFC_AWS_APPLY_ROLE_ARN after the bootstrap window." + value = aws_iam_role.hcptf_apply.arn +} + +output "hcptf_plan_role_arn" { + description = "HCP plan role ARN. Set TFC_AWS_PLAN_ROLE_ARN after the bootstrap window." + value = aws_iam_role.hcptf_plan.arn +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..c3854cb --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,12 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Project = local.project + Environment = "prod" + ManagedBy = "terraform" + Workspace = local.hcp_workspace + } + } +} diff --git a/terraform/scheduler.tf b/terraform/scheduler.tf new file mode 100644 index 0000000..ffa839d --- /dev/null +++ b/terraform/scheduler.tf @@ -0,0 +1,50 @@ +# EventBridge Scheduler execution role. slack-bot creates one-off holiday-* +# schedules at runtime; Terraform does not create those schedules. + +data "aws_iam_policy_document" "holiday_scheduler_assume" { + statement { + sid = "SchedulerAssume" + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["scheduler.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [local.account_id] + } + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:aws:scheduler:${var.aws_region}:${local.account_id}:schedule/default/holiday-*"] + } + } +} + +resource "aws_iam_role" "holiday_scheduler" { + name = local.holiday_scheduler_role_name + path = "/tf-managed/" + description = "EventBridge Scheduler assumes this role to invoke afterhours-holiday-router" + assume_role_policy = data.aws_iam_policy_document.holiday_scheduler_assume.json + permissions_boundary = aws_iam_policy.lambda_boundary.arn +} + +data "aws_iam_policy_document" "holiday_scheduler" { + statement { + sid = "InvokeHolidayRouter" + effect = "Allow" + actions = ["lambda:InvokeFunction"] + resources = [local.holiday_router_arn] + } +} + +resource "aws_iam_role_policy" "holiday_scheduler" { + name = "invoke-holiday-router" + role = aws_iam_role.holiday_scheduler.id + policy = data.aws_iam_policy_document.holiday_scheduler.json +} diff --git a/terraform/secrets.tf b/terraform/secrets.tf new file mode 100644 index 0000000..b9c2290 --- /dev/null +++ b/terraform/secrets.tf @@ -0,0 +1,15 @@ +# Secret shells only. Values are set outside Terraform. 3CX secrets may already +# exist in prod from seahaven-door-unlock-api (PLAT-76); import those names +# rather than recreating: +# terraform import 'aws_secretsmanager_secret.this["afterhours-shift-manager/3cx-domain"]' afterhours-shift-manager/3cx-domain + +resource "aws_secretsmanager_secret" "this" { + for_each = toset(local.secret_names) + + name = each.value + recovery_window_in_days = 30 + + tags = { + Purpose = "afterhours-shift-manager secret shell" + } +} diff --git a/terraform/ssm.tf b/terraform/ssm.tf new file mode 100644 index 0000000..bc5886f --- /dev/null +++ b/terraform/ssm.tf @@ -0,0 +1,15 @@ +resource "aws_ssm_parameter" "deploy_artifacts_bucket" { + name = "${local.ssm_prefix}/deploy/artifacts-bucket" + type = "String" + value = aws_s3_bucket.artifacts.id + description = "Lambda artifacts bucket; deploy.yaml uploads functions//.zip" +} + +resource "aws_ssm_parameter" "deploy_function_name" { + for_each = local.functions + + name = "${local.ssm_prefix}/deploy/${each.key}-function-name" + type = "String" + value = each.value.function_name + description = "Lambda function name for ${each.key}; deploy.yaml calls update-function-code" +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..f6d1488 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,66 @@ +variable "aws_region" { + description = "Region every resource in this configuration is created in." + type = string + default = "us-east-1" +} + +variable "shift_channel" { + description = "Slack channel ID for schedule posts and shift notifications. Not a secret." + type = string + default = "C0APATP612N" +} + +variable "queue_number" { + description = "3CX queue extension number the ring scheduler updates." + type = string + default = "801" +} + +variable "timezone" { + description = "IANA timezone for schedule math and EventBridge cron comments." + type = string + default = "America/New_York" +} + +variable "pay_report_user" { + description = "Slack user ID that receives the weekly pay DM." + type = string + default = "U0A3SC48T47" +} + +variable "sentry_dsn" { + description = "Sentry DSN. Empty disables the SDK. Set in HCP, never in git." + type = string + sensitive = true + default = "" +} + +variable "schedules_enabled" { + description = "When false, EventBridge rules exist but do not fire. Keep false until Slack and Paychex point at this stack." + type = bool + default = false +} + +variable "github_repo" { + description = "GitHub owner/name for the deploy OIDC trust." + type = string + default = "Sea-Haven-Industries/afterhours-shift-manager" +} + +variable "github_deploy_branch" { + description = "Git branch pinned in job_workflow_ref for the deploy role." + type = string + default = "main" +} + +variable "checkcomponents_queue_url" { + description = "paychex-checkcomponents SQS URL. Empty skips the weekly SendMessage." + type = string + default = "https://sqs.us-east-1.amazonaws.com/011934824531/paychex-checkcomponents" +} + +variable "checkcomponents_queue_arn" { + description = "paychex-checkcomponents SQS ARN for WeeklyPost SendMessage." + type = string + default = "arn:aws:sqs:us-east-1:011934824531:paychex-checkcomponents" +} diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 0000000..b6937b2 --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 1.14.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.64" + } + archive = { + source = "hashicorp/archive" + version = "~> 2.8" + } + } + + cloud { + organization = "seahaven" + + workspaces { + name = "afterhours-shift-manager-prod" + } + } +} diff --git a/tests/infra/test_hcp_contract.py b/tests/infra/test_hcp_contract.py new file mode 100644 index 0000000..e8269ce --- /dev/null +++ b/tests/infra/test_hcp_contract.py @@ -0,0 +1,92 @@ +"""Contracts for the HCP Terraform seam (PLAT-74).""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TERRAFORM = ROOT / "terraform" +LAMBDA_TF = (TERRAFORM / "lambda.tf").read_text() +HCP_IAM = (TERRAFORM / "hcp_iam.tf").read_text() +DEPLOY = (ROOT / ".github" / "workflows" / "deploy.yaml").read_text() +CI = (ROOT / ".github" / "workflows" / "ci.yaml").read_text() +LOCALS = (TERRAFORM / "locals.tf").read_text() + + +def test_sam_template_removed(): + assert not (ROOT / "template.yaml").exists() + assert not (ROOT / "samconfig.toml.example").exists() + + +def test_lambda_ignore_changes_includes_code_attributes(): + for attr in ( + "filename", + "s3_bucket", + "s3_key", + "s3_object_version", + "source_code_hash", + ): + assert attr in LAMBDA_TF + assert "lifecycle" in LAMBDA_TF + assert "ignore_changes" in LAMBDA_TF + + +def test_schedules_disabled_by_default(): + chunk = (TERRAFORM / "variables.tf").read_text().split('variable "schedules_enabled"')[1] + chunk = chunk.split("variable ")[0] + assert "default = false" in chunk or "default = false" in chunk + + +def test_prod_only_workspace(): + versions = (TERRAFORM / "versions.tf").read_text() + assert "afterhours-shift-manager-prod" in versions + assert "afterhours-shift-manager-dev" not in versions + assert 'environment = "prod"' in LOCALS + assert "seahaven-dev" not in LOCALS + + +def test_in_repo_hcptf_roles(): + assert 'apply_role = "hcptf-afterhours-shift-manager"' in LOCALS + assert 'plan_role = "hcptf-afterhours-shift-manager-plan"' in LOCALS + assert "hcptf_apply" in HCP_IAM + assert "DenyCreatePolicy" in HCP_IAM + + +def test_deploy_workflow_is_prod_zip_cd(): + assert "release: published" not in DEPLOY + assert "cd-sam" not in DEPLOY + assert "environment: prod" in DEPLOY + assert "deploy-afterhours-prod" in DEPLOY + assert "gh release create" not in DEPLOY + assert "package_lambdas.py" in DEPLOY + assert "update-function-code" in DEPLOY + + +def test_ci_runs_pytest_and_terraform_validate(): + assert "ci-python-sam" not in CI + assert "pytest" in CI + assert "terraform fmt -check" in CI + assert "terraform init -backend=false" in CI + assert "terraform validate" in CI + + +def test_seven_functions_named(): + for name in ( + "afterhours-shift-manager", + "afterhours-weekly-post", + "afterhours-roster-sync", + "afterhours-roster-api", + "afterhours-ring-scheduler", + "afterhours-holiday-router", + "afterhours-release-notifier", + ): + assert name in LOCALS + + +def test_weekly_post_role_is_tf_managed_name(): + assert 'role_name = "afterhours-shift-manager-weekly-post"' in LOCALS + + +def test_github_deploy_trust_is_main_only(): + iam = (TERRAFORM / "iam_github_deploy.tf").read_text() + assert "refs/heads/${var.github_deploy_branch}" in iam + assert "refs/tags/v*" not in iam + assert "environment:prod" in iam or "environment:prod" in LOCALS diff --git a/tests/requirements.txt b/tests/requirements.txt index 91deca3..0053fe4 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,5 @@ -# Test-only dependencies. The CI reusable workflow (ci-python-sam.yaml) installs -# every requirements.txt it finds when run-tests is true, so this file is picked -# up automatically alongside each Lambda's runtime requirements. +# Test-only dependencies. CI's pytest job installs this file plus the runtime +# requirements.txt files the imports need. pytest>=9.1.1 moto[dynamodb,ses,secretsmanager]>=5.2.2 responses>=0.26.2 diff --git a/tests/scripts/test_package_lambdas.py b/tests/scripts/test_package_lambdas.py new file mode 100644 index 0000000..a2c6928 --- /dev/null +++ b/tests/scripts/test_package_lambdas.py @@ -0,0 +1,54 @@ +"""package_lambdas.py zip layout without a full pip install.""" + +import importlib.util +import sys +import zipfile +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] + + +def _load(): + spec = importlib.util.spec_from_file_location( + "package_lambdas", ROOT / "scripts" / "package_lambdas.py" + ) + mod = importlib.util.module_from_spec(spec) + sys.modules["package_lambdas"] = mod + spec.loader.exec_module(mod) + return mod + + +pkg = _load() + + +def test_function_keys_match_terraform_locals(): + locals_tf = (ROOT / "terraform" / "locals.tf").read_text() + for key in pkg.FUNCTIONS: + assert f" {key} =" in locals_tf + for src in pkg.FUNCTIONS.values(): + assert src.is_dir() + assert (src / "requirements.txt").is_file() + + +def test_build_function_bundles_shared_and_git_sha(tmp_path, monkeypatch): + monkeypatch.setattr(pkg, "_req_lines", lambda path: []) + + def fake_run(cmd, check): + raise AssertionError(f"pip should not run when reqs are empty: {cmd}") + + with patch.object(pkg.subprocess, "run", fake_run): + zip_path = pkg.build_function( + "weekly_post", + pkg.FUNCTIONS["weekly_post"], + "deadbeef", + tmp_path, + ) + assert zip_path.is_file() + with zipfile.ZipFile(zip_path) as zf: + names = zf.namelist() + assert "app.py" in names + assert "shared/sentry_init.py" in names + assert "shared/build_info.py" in names + assert 'GIT_SHA = "deadbeef"' in zf.read("shared/build_info.py").decode() + assert "requirements.txt" not in names diff --git a/tests/shared/test_sentry_init.py b/tests/shared/test_sentry_init.py index 53051d8..a0c8948 100644 --- a/tests/shared/test_sentry_init.py +++ b/tests/shared/test_sentry_init.py @@ -1,6 +1,8 @@ """sentry_init: DSN no-op, init options, and before_send scrub.""" import importlib +import sys +from types import ModuleType from unittest.mock import patch from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration @@ -42,6 +44,18 @@ def test_set_dsn_inits_lambda_integration(monkeypatch): assert len(integrations) == 1 assert isinstance(integrations[0], AwsLambdaIntegration) assert integrations[0].timeout_warning is True + assert "release" not in kwargs + + +def test_build_info_sha_sets_sentry_release(monkeypatch): + monkeypatch.setenv("SENTRY_DSN", "https://key@o1.ingest.sentry.io/1") + fake = ModuleType("shared.build_info") + fake.GIT_SHA = "abc123def" + monkeypatch.setitem(sys.modules, "shared.build_info", fake) + with patch("sentry_sdk.init") as mocked: + importlib.reload(sentry_mod) + kwargs = mocked.call_args.kwargs + assert kwargs["release"] == "abc123def" def test_before_send_strips_auth_and_sigv4_headers(): diff --git a/tests/weekly_post/test_handler.py b/tests/weekly_post/test_handler.py index 526a5aa..bdbc5b1 100644 --- a/tests/weekly_post/test_handler.py +++ b/tests/weekly_post/test_handler.py @@ -236,9 +236,9 @@ def test_payload_skips_fallback_and_zero(weeklypost_app): def test_weekly_post_has_no_payroll_email_path(): root = Path(__file__).resolve().parents[2] - template = (root / "template.yaml").read_text() + tf_text = "".join(p.read_text() for p in (root / "terraform").glob("*.tf")) for token in ("PAYROLL_RECIPIENTS", "SES_SENDER", "ses:", "PayrollEmailFailure"): - assert token not in template, token + assert token not in tf_text, token source = (root / "src" / "weekly-post" / "app.py").read_text() for token in ("_send_pay_email", "_build_pay_email_html", 'boto3.client("ses")'): assert token not in source, token