Add the exec-aide-listener-no-running-tasks CloudWatch alarm (RunningTaskCount Minimum < 1, eval 3 / datapoints 2, ALARM-only, NOT_BREACHING) so we page if the Slack Socket Mode listener has no running task and the bot goes dark. RunningTaskCount is only emitted in the ECS/ContainerInsights namespace, so this commit also enables Container Insights on the exec-aide cluster (containerInsightsV2: ENABLED). That is a cost/config change (extra CloudWatch ingestion/storage for the cluster) and is isolated to this commit pending Adam's sign-off; the rest of the alarm coverage does not depend on it. Also refreshes the README CDK Constructs + new Monitoring & Alarms section (and drops the stale 'reminder' Lambda reference removed in #54).
214 lines
8.5 KiB
TypeScript
214 lines
8.5 KiB
TypeScript
import { Construct } from 'constructs';
|
|
import * as cdk from 'aws-cdk-lib';
|
|
import * as ec2 from 'aws-cdk-lib/aws-ec2';
|
|
import * as ecs from 'aws-cdk-lib/aws-ecs';
|
|
import * as ecr from 'aws-cdk-lib/aws-ecr';
|
|
import * as ecrAssets from 'aws-cdk-lib/aws-ecr-assets';
|
|
import * as logs from 'aws-cdk-lib/aws-logs';
|
|
import * as iam from 'aws-cdk-lib/aws-iam';
|
|
import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
|
|
import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch';
|
|
import * as cloudwatchActions from 'aws-cdk-lib/aws-cloudwatch-actions';
|
|
import * as sns from 'aws-cdk-lib/aws-sns';
|
|
import * as path from 'path';
|
|
|
|
export interface SocketModeProps {
|
|
table: dynamodb.ITable;
|
|
conversationFnArn: string;
|
|
/**
|
|
* Shared site-alerts SNS topic for CloudWatch ALARM actions. Imported once at
|
|
* the stack level (sns.Topic.fromTopicArn) and injected here, mirroring how
|
|
* the table is wired into this construct.
|
|
*/
|
|
alarmTopic: sns.ITopic;
|
|
}
|
|
|
|
export class SocketModeConstruct extends Construct {
|
|
constructor(scope: Construct, id: string, props: SocketModeProps) {
|
|
super(scope, id);
|
|
|
|
const account = cdk.Stack.of(this).account;
|
|
const region = cdk.Stack.of(this).region;
|
|
|
|
// ── VPC ────────────────────────────────────────────────────
|
|
|
|
const vpc = new ec2.Vpc(this, 'Vpc', {
|
|
vpcName: 'exec-aide',
|
|
ipAddresses: ec2.IpAddresses.cidr('10.30.0.0/16'),
|
|
maxAzs: 1,
|
|
natGateways: 0,
|
|
subnetConfiguration: [
|
|
{
|
|
cidrMask: 24,
|
|
name: 'exec-aide-listener',
|
|
subnetType: ec2.SubnetType.PUBLIC,
|
|
},
|
|
],
|
|
});
|
|
|
|
const sg = new ec2.SecurityGroup(this, 'ListenerSG', {
|
|
vpc,
|
|
description: 'exec-aide listener - outbound only',
|
|
allowAllOutbound: true,
|
|
});
|
|
|
|
// ── ECR ────────────────────────────────────────────────────
|
|
|
|
new ecr.Repository(this, 'ListenerRepo', {
|
|
repositoryName: 'exec-aide-listener',
|
|
imageScanOnPush: true,
|
|
lifecycleRules: [
|
|
{
|
|
maxImageCount: 5,
|
|
description: 'Keep last 5 images',
|
|
},
|
|
],
|
|
removalPolicy: cdk.RemovalPolicy.RETAIN,
|
|
});
|
|
|
|
// ── ECS Cluster + Task Definition ──────────────────────────
|
|
|
|
const cluster = new ecs.Cluster(this, 'Cluster', {
|
|
clusterName: 'exec-aide',
|
|
vpc,
|
|
// NEEDS ADAM SIGN-OFF (cost/config): Container Insights is required for
|
|
// the RunningTaskCount alarm below (that metric lives in the
|
|
// ECS/ContainerInsights namespace; AWS/ECS does not publish it). Enabling
|
|
// it incurs additional CloudWatch ingestion/storage cost for this cluster.
|
|
containerInsightsV2: ecs.ContainerInsights.ENABLED,
|
|
});
|
|
|
|
const taskDef = new ecs.FargateTaskDefinition(this, 'TaskDef', {
|
|
family: 'exec-aide-listener',
|
|
cpu: 256,
|
|
memoryLimitMiB: 512,
|
|
runtimePlatform: {
|
|
cpuArchitecture: ecs.CpuArchitecture.ARM64,
|
|
operatingSystemFamily: ecs.OperatingSystemFamily.LINUX,
|
|
},
|
|
});
|
|
|
|
taskDef.addContainer('listener', {
|
|
image: ecs.ContainerImage.fromAsset(
|
|
path.join(__dirname, '../../listener'),
|
|
// Explicit platform: the task is ARM64, and without this an amd64 CI
|
|
// runner builds an amd64 image even with QEMU enabled — tasks then
|
|
// crash-loop with "exec format error" (50 failed starts on 2026-06-03).
|
|
{ platform: ecrAssets.Platform.LINUX_ARM64 },
|
|
),
|
|
essential: true,
|
|
environment: {
|
|
TABLE_NAME: props.table.tableName,
|
|
SECRET_SLACK: 'exec-aide/slack-credentials',
|
|
SSM_PREFIX: '/exec-aide',
|
|
CONVERSATION_FN_ARN: props.conversationFnArn,
|
|
},
|
|
logging: ecs.LogDrivers.awsLogs({
|
|
streamPrefix: 'listener',
|
|
logGroup: new logs.LogGroup(this, 'ListenerLogGroup', {
|
|
logGroupName: '/ecs/exec-aide-listener',
|
|
retention: logs.RetentionDays.TWO_MONTHS,
|
|
}),
|
|
}),
|
|
});
|
|
|
|
// ── Task role IAM ──────────────────────────────────────────
|
|
|
|
props.table.grantReadWriteData(taskDef.taskRole);
|
|
|
|
taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({
|
|
actions: ['secretsmanager:GetSecretValue'],
|
|
resources: [
|
|
`arn:aws:secretsmanager:${region}:${account}:secret:exec-aide/slack-credentials-*`,
|
|
],
|
|
}));
|
|
|
|
taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({
|
|
actions: ['ssm:GetParametersByPath', 'ssm:GetParameter'],
|
|
resources: [
|
|
`arn:aws:ssm:${region}:${account}:parameter/exec-aide`,
|
|
`arn:aws:ssm:${region}:${account}:parameter/exec-aide/*`,
|
|
],
|
|
}));
|
|
|
|
taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({
|
|
actions: ['lambda:InvokeFunction'],
|
|
resources: [props.conversationFnArn],
|
|
}));
|
|
|
|
// ── Fargate Service ────────────────────────────────────────
|
|
// Assigned to a const (no logical-id change vs. the prior anonymous
|
|
// construct — id 'Service' is unchanged) so the service-level CloudWatch
|
|
// metrics below can reference it.
|
|
const service = new ecs.FargateService(this, 'Service', {
|
|
serviceName: 'exec-aide-listener',
|
|
cluster,
|
|
taskDefinition: taskDef,
|
|
desiredCount: 1,
|
|
assignPublicIp: true,
|
|
securityGroups: [sg],
|
|
vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
|
|
});
|
|
|
|
// ── CloudWatch Alarms ──────────────────────────────────────
|
|
// ALARM-only (no OK / InsufficientData action); treatMissingData
|
|
// NOT_BREACHING. Shared site-alerts action injected via props.
|
|
// CPU and Memory come from AWS/ECS service metrics (no Container Insights
|
|
// required). The RunningTaskCount alarm — which DOES require Container
|
|
// Insights — is added in a separate, sign-off-gated commit.
|
|
const alarmAction = new cloudwatchActions.SnsAction(props.alarmTopic);
|
|
|
|
new cloudwatch.Alarm(this, 'ListenerCpuHigh', {
|
|
alarmName: 'exec-aide-listener-cpu-high',
|
|
alarmDescription: 'exec-aide-listener Fargate service CPU utilization is high',
|
|
metric: service.metricCpuUtilization({
|
|
period: cdk.Duration.minutes(5),
|
|
statistic: 'Average',
|
|
}),
|
|
threshold: 80,
|
|
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
evaluationPeriods: 3,
|
|
datapointsToAlarm: 2,
|
|
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
}).addAlarmAction(alarmAction);
|
|
|
|
new cloudwatch.Alarm(this, 'ListenerMemoryHigh', {
|
|
alarmName: 'exec-aide-listener-memory-high',
|
|
alarmDescription: 'exec-aide-listener Fargate service memory utilization is high',
|
|
metric: service.metricMemoryUtilization({
|
|
period: cdk.Duration.minutes(5),
|
|
statistic: 'Average',
|
|
}),
|
|
threshold: 80,
|
|
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
evaluationPeriods: 3,
|
|
datapointsToAlarm: 2,
|
|
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
}).addAlarmAction(alarmAction);
|
|
|
|
// NEEDS ADAM SIGN-OFF (cost/config): RunningTaskCount. The desiredCount is
|
|
// 1; this fires if the listener has no running task (Slack Socket Mode goes
|
|
// dark — DMs and @mentions stop being handled). RunningTaskCount is only
|
|
// emitted in the ECS/ContainerInsights namespace, which requires Container
|
|
// Insights to be enabled on the cluster (done above).
|
|
new cloudwatch.Alarm(this, 'ListenerNoRunningTasks', {
|
|
alarmName: 'exec-aide-listener-no-running-tasks',
|
|
alarmDescription: 'exec-aide-listener Fargate service has no running tasks — Slack Socket Mode is down',
|
|
metric: new cloudwatch.Metric({
|
|
namespace: 'ECS/ContainerInsights',
|
|
metricName: 'RunningTaskCount',
|
|
dimensionsMap: {
|
|
ClusterName: cluster.clusterName,
|
|
ServiceName: service.serviceName,
|
|
},
|
|
statistic: 'Minimum',
|
|
period: cdk.Duration.minutes(5),
|
|
}),
|
|
threshold: 1,
|
|
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
|
|
evaluationPeriods: 3,
|
|
datapointsToAlarm: 2,
|
|
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
}).addAlarmAction(alarmAction);
|
|
}
|
|
}
|