mirror of
https://github.com/Sea-Haven-Industries/afterhours-shift-manager.git
synced 2026-09-30 11:23:12 +00:00
Adam Moussa
38ab8a1487
Extend the in-template Lambda-<Metric>-<fn> alarm convention to full coverage: - Errors (Sum, >=1/5min) for roster-sync and release-notifier, plus orphan adoption of slack-bot and weekly-post (live alarms of those exact names already exist outside the stack and must be deleted before deploy). - Duration (Maximum, ~80% of timeout, 2-of-3) for all six functions. - Throttles (Sum, >=1/5min) for all six functions. - DynamoDB ThrottledRequests (Sum, >=1/5min) on afterhours-shifts. SystemErrors omitted: AWS emits it only per-Operation, so a TableName-only alarm would sit permanently in INSUFFICIENT_DATA. - API Gateway v2 4xx (>=5), 5xx (>=1), and p99 Latency (~3000ms, 2-of-3) on the implicit ServerlessHttpApi. All alarms page the shared site-alerts SNS topic, no OKActions, TreatMissingData notBreaching. README updated with a Monitoring & Alarms section. Duration and API latency thresholds pending sign-off.
936 lines
34 KiB
YAML
936 lines
34 KiB
YAML
AWSTemplateFormatVersion: "2010-09-09"
|
|
Transform: AWS::Serverless-2016-10-31
|
|
Description: After-Hours Shift Manager — Slack bot for managing on-call shifts with 3CX integration
|
|
|
|
Parameters:
|
|
Timezone:
|
|
Type: String
|
|
Default: "America/New_York"
|
|
ShiftChannel:
|
|
Type: String
|
|
Description: Slack channel ID for schedule posts and shift notifications
|
|
QueueNumber:
|
|
Type: String
|
|
Default: "801"
|
|
Description: 3CX queue extension number to update
|
|
|
|
Globals:
|
|
Function:
|
|
Runtime: python3.12
|
|
Timeout: 30
|
|
MemorySize: 1024
|
|
Architectures:
|
|
- arm64
|
|
PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary
|
|
# Access logging + default throttling on the implicit HTTP API (audit M-18).
|
|
HttpApi:
|
|
AccessLogSettings:
|
|
DestinationArn: !GetAtt ApiAccessLogGroup.Arn
|
|
Format: '{"requestId":"$context.requestId","ip":"$context.identity.sourceIp","requestTime":"$context.requestTime","method":"$context.httpMethod","routeKey":"$context.routeKey","status":"$context.status","protocol":"$context.protocol","responseLength":"$context.responseLength","integrationError":"$context.integrationErrorMessage"}'
|
|
DefaultRouteSettings:
|
|
ThrottlingBurstLimit: 50
|
|
ThrottlingRateLimit: 100
|
|
|
|
Resources:
|
|
ApiAccessLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: /aws/apigateway/afterhours-shift-manager
|
|
RetentionInDays: 90
|
|
|
|
# --- Shared Lambda Layer ---
|
|
SharedLayer:
|
|
Type: AWS::Serverless::LayerVersion
|
|
Properties:
|
|
LayerName: afterhours-shared
|
|
ContentUri: src/shared/
|
|
CompatibleRuntimes:
|
|
- python3.12
|
|
CompatibleArchitectures:
|
|
- arm64
|
|
Metadata:
|
|
BuildMethod: python3.12
|
|
BuildArchitecture: arm64
|
|
|
|
# --- DynamoDB ---
|
|
ShiftTable:
|
|
Type: AWS::DynamoDB::Table
|
|
Properties:
|
|
TableName: afterhours-shifts
|
|
BillingMode: PAY_PER_REQUEST
|
|
AttributeDefinitions:
|
|
- AttributeName: PK
|
|
AttributeType: S
|
|
- AttributeName: SK
|
|
AttributeType: S
|
|
KeySchema:
|
|
- AttributeName: PK
|
|
KeyType: HASH
|
|
- AttributeName: SK
|
|
KeyType: RANGE
|
|
TimeToLiveSpecification:
|
|
AttributeName: expires_at
|
|
Enabled: true
|
|
|
|
# --- Slack Bot Lambda ---
|
|
SlackBotFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-shift-manager
|
|
Handler: handler.handler
|
|
CodeUri: src/slack-bot/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Environment:
|
|
Variables:
|
|
SHIFT_TABLE: !Ref ShiftTable
|
|
SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token
|
|
SLACK_SIGNING_SECRET: afterhours-shift-manager/slack-signing-secret
|
|
SHIFT_CHANNEL: !Ref ShiftChannel
|
|
TCX_SECRET_PREFIX: afterhours-shift-manager/3cx-
|
|
QUEUE_NUMBER: !Ref QueueNumber
|
|
TZ: !Ref Timezone
|
|
# Holiday scheduling: per-holiday one-off schedules target the router,
|
|
# passing the scheduler exec role; inline activation invokes it directly.
|
|
HOLIDAY_ROUTER_ARN: !GetAtt HolidayRouterFunction.Arn
|
|
HOLIDAY_SCHEDULER_ROLE_ARN: !GetAtt HolidaySchedulerExecutionRole.Arn
|
|
Policies:
|
|
- DynamoDBCrudPolicy:
|
|
TableName: !Ref ShiftTable
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*"
|
|
# Manage the per-holiday EventBridge Scheduler one-off schedules
|
|
# (08:00 activate / 17:00 deactivate of the holiday router).
|
|
- Effect: Allow
|
|
Action:
|
|
- scheduler:CreateSchedule
|
|
- scheduler:DeleteSchedule
|
|
- scheduler:GetSchedule
|
|
# Predictable names (holiday-activate-/holiday-deactivate-<date>)
|
|
# in the default group — scope to those rather than all schedules.
|
|
Resource:
|
|
- !Sub "arn:aws:scheduler:${AWS::Region}:${AWS::AccountId}:schedule/default/holiday-*"
|
|
# PassRole only for the holiday scheduler exec role, and only when
|
|
# handed to EventBridge Scheduler.
|
|
- Effect: Allow
|
|
Action: iam:PassRole
|
|
Resource: !GetAtt HolidaySchedulerExecutionRole.Arn
|
|
Condition:
|
|
StringEquals:
|
|
iam:PassedToService: scheduler.amazonaws.com
|
|
# Inline activation (holiday added mid-window) invokes the router now.
|
|
# Unqualified ARN only — we invoke the base function, no alias/version.
|
|
- Effect: Allow
|
|
Action: lambda:InvokeFunction
|
|
Resource: !GetAtt HolidayRouterFunction.Arn
|
|
Events:
|
|
SlackEvents:
|
|
Type: HttpApi
|
|
Properties:
|
|
Path: /slack/events
|
|
Method: POST
|
|
|
|
# --- Weekly Schedule Post (Monday 7am ET) ---
|
|
WeeklyPostFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-weekly-post
|
|
Handler: app.handler
|
|
CodeUri: src/weekly-post/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Environment:
|
|
Variables:
|
|
SHIFT_TABLE: !Ref ShiftTable
|
|
SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token
|
|
SHIFT_CHANNEL: !Ref ShiftChannel
|
|
SES_SENDER: noreply@seahaven.com
|
|
PAYROLL_RECIPIENTS: payroll@seahaven.com
|
|
PAY_REPORT_USER: U0A3SC48T47
|
|
TZ: !Ref Timezone
|
|
Policies:
|
|
- DynamoDBCrudPolicy:
|
|
TableName: !Ref ShiftTable
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*"
|
|
- Effect: Allow
|
|
Action:
|
|
- ses:SendEmail
|
|
Resource:
|
|
- !Sub "arn:aws:ses:${AWS::Region}:${AWS::AccountId}:identity/*"
|
|
# The sending identity has a default configuration set
|
|
# (seahaven-email-events); SES authorizes SendEmail against the
|
|
# config-set resource too, so it must be granted alongside the
|
|
# identity or the send is denied. Scoped to the known set name.
|
|
- !Sub "arn:aws:ses:${AWS::Region}:${AWS::AccountId}:configuration-set/seahaven-email-events"
|
|
Events:
|
|
# EST: 7am ET = 12:00 UTC (Nov-Mar)
|
|
WeeklyPostEST:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 12 ? * MON *)
|
|
Description: "Post weekly schedule Monday 7am EST"
|
|
Enabled: true
|
|
# EDT: 7am ET = 11:00 UTC (Mar-Nov)
|
|
WeeklyPostEDT:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 11 ? * MON *)
|
|
Description: "Post weekly schedule Monday 7am EDT"
|
|
Enabled: true
|
|
|
|
# --- Roster Sync Lambda (daily sync from 3CX) ---
|
|
RosterSyncFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-roster-sync
|
|
Handler: app.handler
|
|
CodeUri: src/roster-sync/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Timeout: 60
|
|
Environment:
|
|
Variables:
|
|
SHIFT_TABLE: !Ref ShiftTable
|
|
TCX_SECRET_PREFIX: afterhours-shift-manager/3cx-
|
|
SYNC_GROUP: DEFAULT
|
|
TZ: !Ref Timezone
|
|
Policies:
|
|
- DynamoDBCrudPolicy:
|
|
TableName: !Ref ShiftTable
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*"
|
|
Events:
|
|
# Daily at 6am ET (before the 7am schedule post and 8am 3CX scheduler)
|
|
# EST: 6am ET = 11:00 UTC (Nov-Mar)
|
|
RosterSyncEST:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 11 ? * * *)
|
|
Description: "Sync roster from 3CX at 6am EST"
|
|
Enabled: true
|
|
# EDT: 6am ET = 10:00 UTC (Mar-Nov)
|
|
RosterSyncEDT:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 10 ? * * *)
|
|
Description: "Sync roster from 3CX at 6am EDT"
|
|
Enabled: true
|
|
|
|
# --- Ring Scheduler (daily 3CX queue routing updates) ---
|
|
RingSchedulerFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-ring-scheduler
|
|
Handler: app.handler
|
|
CodeUri: src/ring-scheduler/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Timeout: 60
|
|
Environment:
|
|
Variables:
|
|
SHIFT_TABLE: !Ref ShiftTable
|
|
TCX_SECRET_PREFIX: afterhours-shift-manager/3cx-
|
|
QUEUE_NUMBER: !Ref QueueNumber
|
|
TZ: !Ref Timezone
|
|
Policies:
|
|
- DynamoDBCrudPolicy:
|
|
TableName: !Ref ShiftTable
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*"
|
|
Events:
|
|
# Daily at 8am ET — update after-hours routing
|
|
DailyScheduleEST:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 13 ? * * *)
|
|
Description: "Update 3CX queue at 8am EST"
|
|
Enabled: true
|
|
DailyScheduleEDT:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 12 ? * * *)
|
|
Description: "Update 3CX queue at 8am EDT"
|
|
Enabled: true
|
|
# Weekends at 5pm ET — switch to night shift person
|
|
WeekendEveningEST:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 22 ? * SAT,SUN *)
|
|
Description: "Update 3CX queue at 5pm EST weekends"
|
|
Enabled: true
|
|
WeekendEveningEDT:
|
|
Type: Schedule
|
|
Properties:
|
|
Schedule: cron(0 21 ? * SAT,SUN *)
|
|
Description: "Update 3CX queue at 5pm EDT weekends"
|
|
Enabled: true
|
|
|
|
# Lambda error alarm for the ring scheduler. Mirrors the account-wide
|
|
# operational convention (Lambda-Errors-<fn>, threshold 1 over one 5-min
|
|
# period, Sum, missing=notBreaching) and pages the same site-alerts SNS topic
|
|
# → AWS Chatbot → Slack as the other afterhours-* functions.
|
|
RingSchedulerErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${RingSchedulerFunction}"
|
|
AlarmDescription: "Ring scheduler Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RingSchedulerFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- Holiday Router (repoints 3CX IVR/queue for a holiday day-shift) ---
|
|
# Invoked with {"action": "activate"|"deactivate", "date": "<YYYY-MM-DD>"} at
|
|
# 08:00 ET (activate) and 17:00 ET (deactivate) for each holiday date. Both
|
|
# operations are idempotent. No standing schedule here — invocation is driven
|
|
# per-holiday-date (the holiday record gates the work; off-days are no-ops).
|
|
HolidayRouterFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-holiday-router
|
|
Handler: app.handler
|
|
CodeUri: src/holiday-router/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Timeout: 60
|
|
Environment:
|
|
Variables:
|
|
SHIFT_TABLE: !Ref ShiftTable
|
|
TCX_SECRET_PREFIX: afterhours-shift-manager/3cx-
|
|
TZ: !Ref Timezone
|
|
Policies:
|
|
- DynamoDBCrudPolicy:
|
|
TableName: !Ref ShiftTable
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/*"
|
|
|
|
# Lambda error alarm for the holiday router. Mirrors the account-wide
|
|
# operational convention (Lambda-Errors-<fn>, threshold 1 over one 5-min
|
|
# period, Sum, missing=notBreaching) and pages the same site-alerts SNS topic
|
|
# → AWS Chatbot → Slack as the other afterhours-* functions.
|
|
HolidayRouterErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${HolidayRouterFunction}"
|
|
AlarmDescription: "Holiday router Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref HolidayRouterFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# EventBridge Scheduler execution role. The slack-bot creates one-off
|
|
# schedules per holiday date (08:00 activate / 17:00 deactivate); Scheduler
|
|
# assumes this role to invoke the holiday router. Auto-named (no RoleName) so
|
|
# the deploy's CAPABILITY_IAM suffices — cd-sam does not pass
|
|
# CAPABILITY_NAMED_IAM. The permissions boundary is REQUIRED: the scoped
|
|
# github-cfn-execution-role gates iam:CreateRole/PutRolePolicy on roles
|
|
# carrying exactly this boundary, so the CI deploy is denied without it.
|
|
HolidaySchedulerExecutionRole:
|
|
Type: AWS::IAM::Role
|
|
Properties:
|
|
PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary
|
|
AssumeRolePolicyDocument:
|
|
Version: "2012-10-17"
|
|
Statement:
|
|
- Effect: Allow
|
|
Principal:
|
|
Service: scheduler.amazonaws.com
|
|
Action: sts:AssumeRole
|
|
Condition:
|
|
StringEquals:
|
|
aws:SourceAccount: !Ref AWS::AccountId
|
|
# Only schedules this stack creates (holiday-* in the default group)
|
|
# may assume the role — not any schedule in the account.
|
|
ArnLike:
|
|
aws:SourceArn: !Sub "arn:aws:scheduler:${AWS::Region}:${AWS::AccountId}:schedule/default/holiday-*"
|
|
Policies:
|
|
- PolicyName: invoke-holiday-router
|
|
PolicyDocument:
|
|
Version: "2012-10-17"
|
|
Statement:
|
|
- Effect: Allow
|
|
Action: lambda:InvokeFunction
|
|
Resource: !GetAtt HolidayRouterFunction.Arn
|
|
|
|
# --- Release Notifier (invoked by release.yaml on minor/major releases) ---
|
|
ReleaseNotifierFunction:
|
|
Type: AWS::Serverless::Function
|
|
Properties:
|
|
FunctionName: afterhours-release-notifier
|
|
Handler: app.handler
|
|
CodeUri: src/release-notifier/
|
|
Layers:
|
|
- !Ref SharedLayer
|
|
Environment:
|
|
Variables:
|
|
SLACK_BOT_TOKEN_SECRET: afterhours-shift-manager/slack-bot-token
|
|
SHIFT_CHANNEL: !Ref ShiftChannel
|
|
TZ: !Ref Timezone
|
|
Policies:
|
|
# Least privilege: only the Slack bot token, not the whole namespace.
|
|
- Statement:
|
|
- Effect: Allow
|
|
Action:
|
|
- secretsmanager:GetSecretValue
|
|
Resource:
|
|
- !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:afterhours-shift-manager/slack-bot-token-*"
|
|
|
|
# GitHub-OIDC role assumed by release.yaml to invoke the notifier. Auto-named
|
|
# (no RoleName) so the deploy's CAPABILITY_IAM is sufficient — cd-sam does not
|
|
# pass CAPABILITY_NAMED_IAM. Trust + permission are scoped to the minimum: this
|
|
# repo's main ref + release workflow, and InvokeFunction on the notifier alone.
|
|
# Its ARN is surfaced as a stack output and set once as the
|
|
# RELEASE_NOTIFY_INVOKE_ROLE_ARN repo variable (see README).
|
|
#
|
|
# The permissions boundary is REQUIRED, not optional: the scoped
|
|
# github-cfn-execution-role's IAM policy gates iam:CreateRole/PutRolePolicy on
|
|
# the role carrying exactly this boundary, so the CI deploy is denied without
|
|
# it. The boundary itself permits lambda:InvokeFunction (Sid LambdaInvoke), so
|
|
# it does not restrict this role's one job.
|
|
ReleaseNotifyInvokeRole:
|
|
Type: AWS::IAM::Role
|
|
Properties:
|
|
PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary
|
|
AssumeRolePolicyDocument:
|
|
Version: "2012-10-17"
|
|
Statement:
|
|
- Effect: Allow
|
|
Principal:
|
|
Federated: !Sub "arn:aws:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com"
|
|
Action: sts:AssumeRoleWithWebIdentity
|
|
Condition:
|
|
StringEquals:
|
|
token.actions.githubusercontent.com:aud: sts.amazonaws.com
|
|
StringLike:
|
|
token.actions.githubusercontent.com:sub: "repo:Sea-Haven-Industries/afterhours-shift-manager:ref:refs/heads/main"
|
|
# Defense-in-depth: only the Deploy workflow's release job may assume
|
|
# this role, not any workflow running on main. (The release job lives
|
|
# in deploy.yaml; this must match that workflow's path.)
|
|
token.actions.githubusercontent.com:job_workflow_ref: "Sea-Haven-Industries/afterhours-shift-manager/.github/workflows/deploy.yaml@refs/heads/main"
|
|
Policies:
|
|
- PolicyName: invoke-release-notifier
|
|
PolicyDocument:
|
|
Version: "2012-10-17"
|
|
Statement:
|
|
- Effect: Allow
|
|
Action: lambda:InvokeFunction
|
|
Resource: !GetAtt ReleaseNotifierFunction.Arn
|
|
|
|
# ===========================================================================
|
|
# CloudWatch alarm coverage (Wave 1 PR A). All alarms page the same
|
|
# site-alerts SNS topic → AWS Chatbot → Slack as the existing Errors alarms.
|
|
# No OKActions by design (the existing Errors alarms have none either).
|
|
# Naming follows the in-template convention: Lambda-<Metric>-${Fn}.
|
|
# ===========================================================================
|
|
|
|
# --- Lambda Errors alarms (clone of HolidayRouterErrorAlarm) ---
|
|
# Errors for ring-scheduler + holiday-router already exist above.
|
|
|
|
RosterSyncErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${RosterSyncFunction}"
|
|
AlarmDescription: "Roster sync Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RosterSyncFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
ReleaseNotifierErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${ReleaseNotifierFunction}"
|
|
AlarmDescription: "Release notifier Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref ReleaseNotifierFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# ORPHAN ADOPTION: live alarms named Lambda-Errors-afterhours-shift-manager
|
|
# and Lambda-Errors-afterhours-weekly-post already exist OUTSIDE the stack.
|
|
# They MUST be deleted immediately before this stack deploys, or CloudFormation
|
|
# will fail to create these resources (AlarmName collision). See PR body.
|
|
SlackBotErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${SlackBotFunction}"
|
|
AlarmDescription: "Slack bot Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref SlackBotFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
WeeklyPostErrorAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Errors-${WeeklyPostFunction}"
|
|
AlarmDescription: "Weekly post Lambda reported one or more errors"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Errors
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref WeeklyPostFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- Lambda Duration alarms (Maximum, ms; 2-of-3 evaluation) ---
|
|
# Thresholds set to ~80% of each function's timeout. ALL Duration thresholds
|
|
# and the 3/2 evaluation are PENDING ADAM SIGN-OFF (see PR body).
|
|
# Timeouts: slack-bot/weekly-post/release-notifier = 30s (global default);
|
|
# roster-sync/ring-scheduler/holiday-router = 60s.
|
|
SlackBotDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${SlackBotFunction}"
|
|
AlarmDescription: "Slack bot Lambda duration approaching its 30s timeout (>=24s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref SlackBotFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 24000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
WeeklyPostDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${WeeklyPostFunction}"
|
|
AlarmDescription: "Weekly post Lambda duration approaching its 30s timeout (>=24s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref WeeklyPostFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 24000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
RosterSyncDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${RosterSyncFunction}"
|
|
AlarmDescription: "Roster sync Lambda duration approaching its 60s timeout (>=48s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RosterSyncFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 48000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
RingSchedulerDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${RingSchedulerFunction}"
|
|
AlarmDescription: "Ring scheduler Lambda duration approaching its 60s timeout (>=48s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RingSchedulerFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 48000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
HolidayRouterDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${HolidayRouterFunction}"
|
|
AlarmDescription: "Holiday router Lambda duration approaching its 60s timeout (>=48s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref HolidayRouterFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 48000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
ReleaseNotifierDurationAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Duration-${ReleaseNotifierFunction}"
|
|
AlarmDescription: "Release notifier Lambda duration approaching its 30s timeout (>=24s)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Duration
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref ReleaseNotifierFunction
|
|
Statistic: Maximum
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 24000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- Lambda Throttles alarms (Sum; threshold 1 over one 5-min period) ---
|
|
SlackBotThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${SlackBotFunction}"
|
|
AlarmDescription: "Slack bot Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref SlackBotFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
WeeklyPostThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${WeeklyPostFunction}"
|
|
AlarmDescription: "Weekly post Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref WeeklyPostFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
RosterSyncThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${RosterSyncFunction}"
|
|
AlarmDescription: "Roster sync Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RosterSyncFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
RingSchedulerThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${RingSchedulerFunction}"
|
|
AlarmDescription: "Ring scheduler Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref RingSchedulerFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
HolidayRouterThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${HolidayRouterFunction}"
|
|
AlarmDescription: "Holiday router Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref HolidayRouterFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
ReleaseNotifierThrottlesAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "Lambda-Throttles-${ReleaseNotifierFunction}"
|
|
AlarmDescription: "Release notifier Lambda was throttled (concurrency limit hit)"
|
|
Namespace: AWS/Lambda
|
|
MetricName: Throttles
|
|
Dimensions:
|
|
- Name: FunctionName
|
|
Value: !Ref ReleaseNotifierFunction
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- DynamoDB alarms (afterhours-shifts table) ---
|
|
# ThrottledRequests emits at the TableName dimension → safe to alarm.
|
|
# NOTE: SystemErrors is INTENTIONALLY OMITTED. AWS emits AWS/DynamoDB
|
|
# SystemErrors only at TableName+Operation granularity, never TableName-only,
|
|
# so a TableName-only SystemErrors alarm would sit permanently in
|
|
# INSUFFICIENT_DATA. (Verified: no DynamoDB table in this account has ever
|
|
# emitted SystemErrors, consistent with documented per-Operation emission.)
|
|
ShiftTableThrottleAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "DynamoDB-ThrottledRequests-${ShiftTable}"
|
|
AlarmDescription: "afterhours-shifts table had one or more throttled requests"
|
|
Namespace: AWS/DynamoDB
|
|
MetricName: ThrottledRequests
|
|
Dimensions:
|
|
- Name: TableName
|
|
Value: !Ref ShiftTable
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- API Gateway v2 (HTTP API) alarms on the implicit ServerlessHttpApi ---
|
|
# AWS::ApiGatewayV2 metric names: 4xx, 5xx, Latency; dimension ApiId.
|
|
ApiGateway4xxAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "ApiGateway-4xx-${ServerlessHttpApi}"
|
|
AlarmDescription: "Elevated 4xx responses on the Slack events HTTP API"
|
|
Namespace: AWS/ApiGateway
|
|
MetricName: 4xx
|
|
Dimensions:
|
|
- Name: ApiId
|
|
Value: !Ref ServerlessHttpApi
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 5
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
ApiGateway5xxAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "ApiGateway-5xx-${ServerlessHttpApi}"
|
|
AlarmDescription: "5xx responses on the Slack events HTTP API"
|
|
Namespace: AWS/ApiGateway
|
|
MetricName: 5xx
|
|
Dimensions:
|
|
- Name: ApiId
|
|
Value: !Ref ServerlessHttpApi
|
|
Statistic: Sum
|
|
Period: 300
|
|
EvaluationPeriods: 1
|
|
Threshold: 1
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# Latency p99 via ExtendedStatistic. ~3000ms target is PENDING ADAM SIGN-OFF
|
|
# alongside the Lambda Duration thresholds (Slack requires a fast 3s ack).
|
|
ApiGatewayLatencyAlarm:
|
|
Type: AWS::CloudWatch::Alarm
|
|
Properties:
|
|
AlarmName: !Sub "ApiGateway-Latency-${ServerlessHttpApi}"
|
|
AlarmDescription: "p99 latency on the Slack events HTTP API exceeded 3s"
|
|
Namespace: AWS/ApiGateway
|
|
MetricName: Latency
|
|
Dimensions:
|
|
- Name: ApiId
|
|
Value: !Ref ServerlessHttpApi
|
|
ExtendedStatistic: p99
|
|
Period: 300
|
|
EvaluationPeriods: 3
|
|
DatapointsToAlarm: 2
|
|
Threshold: 3000
|
|
ComparisonOperator: GreaterThanOrEqualToThreshold
|
|
TreatMissingData: notBreaching
|
|
AlarmActions:
|
|
- !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:site-alerts"
|
|
|
|
# --- CloudWatch Log Groups (explicit 60-day retention) ---
|
|
SlackBotLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${SlackBotFunction}"
|
|
RetentionInDays: 60
|
|
|
|
WeeklyPostLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${WeeklyPostFunction}"
|
|
RetentionInDays: 60
|
|
|
|
RosterSyncLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${RosterSyncFunction}"
|
|
RetentionInDays: 60
|
|
|
|
RingSchedulerLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${RingSchedulerFunction}"
|
|
RetentionInDays: 60
|
|
|
|
HolidayRouterLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${HolidayRouterFunction}"
|
|
RetentionInDays: 60
|
|
|
|
ReleaseNotifierLogGroup:
|
|
Type: AWS::Logs::LogGroup
|
|
Properties:
|
|
LogGroupName: !Sub "/aws/lambda/${ReleaseNotifierFunction}"
|
|
RetentionInDays: 60
|
|
|
|
Outputs:
|
|
SlackBotApiUrl:
|
|
Description: URL for Slack app Request URL configuration
|
|
Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/slack/events"
|
|
ShiftTableName:
|
|
Value: !Ref ShiftTable
|
|
SlackBotFunctionArn:
|
|
Value: !GetAtt SlackBotFunction.Arn
|
|
WeeklyPostFunctionArn:
|
|
Value: !GetAtt WeeklyPostFunction.Arn
|
|
RosterSyncFunctionArn:
|
|
Value: !GetAtt RosterSyncFunction.Arn
|
|
RingSchedulerFunctionArn:
|
|
Value: !GetAtt RingSchedulerFunction.Arn
|
|
HolidayRouterFunctionArn:
|
|
Value: !GetAtt HolidayRouterFunction.Arn
|
|
HolidaySchedulerExecutionRoleArn:
|
|
Description: Role EventBridge Scheduler assumes to invoke the holiday router; the slack-bot passes this when creating per-holiday schedules
|
|
Value: !GetAtt HolidaySchedulerExecutionRole.Arn
|
|
ReleaseNotifierFunctionArn:
|
|
Value: !GetAtt ReleaseNotifierFunction.Arn
|
|
ReleaseNotifyInvokeRoleArn:
|
|
Description: Set this as the RELEASE_NOTIFY_INVOKE_ROLE_ARN repo variable for release.yaml
|
|
Value: !GetAtt ReleaseNotifyInvokeRole.Arn
|