From 44ad13d8cd98404cded79be70ff97e91e3939a0e Mon Sep 17 00:00:00 2001
From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com>
Date: Wed, 17 Jun 2026 15:02:10 -0400
Subject: [PATCH] Add DLQ messages-present alarm for classifier (INFRA-57)
 (#26)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The classifier DLQ (apm-wo-analysis-classifier-dlq) had no alarm, so a
message landing there after Lambda exhausts async retries — a genuinely
dropped classifier run — surfaced nowhere.

Add a CloudWatch alarm on AWS/SQS ApproximateNumberOfMessagesVisible
(Maximum, threshold >0, 300s period, 1 evaluation period, notBreaching)
that pages the shared site-alerts SNS topic. Mirrors the
workorder-email-processor-dlq-messages alarm in procurement-ingest's
wo_stack and the payments-payroll-batch-dlq-messages alarm convention.
ALARM-only, no OK action, per the CloudWatch-alarm preference.
---
 cdk/stacks/pipeline_stack.py | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/cdk/stacks/pipeline_stack.py b/cdk/stacks/pipeline_stack.py
index 89bbe53..23df81e 100644
--- a/cdk/stacks/pipeline_stack.py
+++ b/cdk/stacks/pipeline_stack.py
@@ -29,6 +29,12 @@ from aws_cdk import (
 from aws_cdk import (
     aws_certificatemanager as acm,
 )
+from aws_cdk import (
+    aws_cloudwatch as cloudwatch,
+)
+from aws_cdk import (
+    aws_cloudwatch_actions as cw_actions,
+)
 from aws_cdk import (
     aws_glue as glue,
 )
@@ -56,6 +62,9 @@ from aws_cdk import (
 from aws_cdk import (
     aws_secretsmanager as secretsmanager,
 )
+from aws_cdk import (
+    aws_sns as sns,
+)
 from aws_cdk import (
     aws_sqs as sqs,
 )
@@ -238,6 +247,34 @@ class PipelineStack(Stack):
             retention_period=Duration.days(14),
             enforce_ssl=True,
         )
+
+        # DLQ messages-present alarm — a message only lands here after Lambda
+        # exhausts its async retries and gives up, i.e. a genuinely dropped run.
+        # MAXIMUM over a 5-min window so a single visible message pages even if it
+        # is later consumed/redriven. ALARM-only (no OK action, per the
+        # CloudWatch-alarm preference) to the shared site-alerts topic.
+        alarm_topic = sns.Topic.from_topic_arn(
+            self,
+            "SiteAlertsTopic",
+            f"arn:aws:sns:{self.region}:{self.account}:site-alerts",
+        )
+        classifier_dlq.metric_approximate_number_of_messages_visible(
+            period=Duration.minutes(5),
+            statistic="Maximum",
+        ).create_alarm(
+            self,
+            "ClassifierDlqMessagesAlarm",
+            alarm_name="apm-wo-analysis-classifier-dlq-messages",
+            alarm_description=(
+                "apm-wo-analysis-classifier DLQ has visible messages "
+                "(dropped classifier run)"
+            ),
+            threshold=0,
+            evaluation_periods=1,
+            comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+            treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
+        ).add_alarm_action(cw_actions.SnsAction(alarm_topic))
+
         self.classifier_fn = lambda_.Function(
             self,
             "Classifier",