From 29ad891290150eceb04953bf61c66a79e4312dd0 Mon Sep 17 00:00:00 2001
From: Adam Moussa <adam@seahavenind.com>
Date: Thu, 16 Jul 2026 15:15:13 -0400
Subject: [PATCH] Add PO fallback-rate alarm retuned for ~57 emails/day

The WO alarm's 15-min period and >=10-sample floor assume
~760/day and would be structurally dead at PO volume (a 15-min
period holds ~0.6 emails, so the floor is never met). Retune:
6-hour periods (~14.25 expected emails), IF((fb+tmpl)>=8,...)
volume floor so a single email can never breach a datapoint
(1/8 = 12.5% < 20%), threshold >20% against a ~1% expected
baseline, eval 4 / datapoints 2 (24h span) so noise self-clears
while total template drift pages within ~12h. No element-wise
MAX in the math expression (post-#102 rule); ALARM-only
SnsAction to site-alerts, NOT_BREACHING. Gated with
'npx cdk synth po-ingest'.

Also add template_parser.py to the bundling cp list -- without it
every deployed invocation would ImportError (unit tests cannot
catch an asset-bundling omission).

Refs: #99, #102
---
 cdk/po_stack.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/cdk/po_stack.py b/cdk/po_stack.py
index 18b50c2..c937b90 100644
--- a/cdk/po_stack.py
+++ b/cdk/po_stack.py
@@ -244,7 +244,7 @@ class PoIngestStack(Stack):
                         "-c",
                         "pip install --platform manylinux2014_aarch64 --only-binary=:all: "
                         "-r requirements.txt -t /asset-output && "
-                        "cp handler.py ses_auth.py /asset-output/",
+                        "cp handler.py ses_auth.py template_parser.py /asset-output/",
                     ],
                 ),
             ),
@@ -383,6 +383,69 @@ class PoIngestStack(Stack):
             treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
         ).add_alarm_action(cw_actions.SnsAction(alarm_topic))
 
+        # --- Template fallback-rate alarm: po-email-processor ---
+        # The processor tries a deterministic template parse first and only calls
+        # the Bedrock AI extractor on a miss/invalid. A sustained rise in the
+        # ai_fallback share signals Coupa template drift (coverage collapse).
+        # EMF metric Seahaven/PoIngest/ParseOutcome, dimensioned by ParseMethod
+        # (template|ai_fallback).
+        #
+        # RETUNED for PO volume (~57 emails/day ≈ 14.25 per 6h period) -- the WO
+        # alarm's 15-min period / >=10-sample floor assume ~760/day and would be
+        # structurally DEAD here (a 15-min period holds ~0.6 PO emails, so the
+        # floor is never met and the IF always takes the 0 branch):
+        #   * period 6h: a stable ~14-email denominator per datapoint.
+        #   * volume floor >=8: at the floor, one fallback email = 12.5% < 20%,
+        #     so a single email can NEVER breach a datapoint; a breach needs >=2
+        #     fallbacks in one 6h window (2/8 = 25%) or >=3 at typical volume
+        #     (3/14 ≈ 21%). Sparse overnight/weekend windows (<8 emails) take
+        #     the 0 branch -- non-breaching by design (accepted trade: a Friday-
+        #     evening drift may not page until weekend volume accrues).
+        #   * threshold >20%: expected baseline fallback ≈1% (comments 0.55% +
+        #     multi-line 0.18% + non-USD 0) -- far below the threshold.
+        #   * 2 of 4 datapoints (24h span): isolated noise self-clears, while
+        #     total template drift (100% fallback) pages within ~12h.
+        # Post-#102 rule: NO element-wise MAX(timeseries, scalar) in alarm math;
+        # the IF volume floor guarantees the non-zero denominator. Any change to
+        # this expression must be gated by `npx cdk synth po-ingest`.
+        fb_metric = cloudwatch.Metric(
+            namespace="Seahaven/PoIngest",
+            metric_name="ParseOutcome",
+            dimensions_map={"ParseMethod": "ai_fallback"},
+            statistic="Sum",
+            period=Duration.hours(6),
+        )
+        tmpl_metric = cloudwatch.Metric(
+            namespace="Seahaven/PoIngest",
+            metric_name="ParseOutcome",
+            dimensions_map={"ParseMethod": "template"},
+            statistic="Sum",
+            period=Duration.hours(6),
+        )
+        fallback_rate = cloudwatch.MathExpression(
+            expression=(
+                "IF((FILL(fb,0)+FILL(tmpl,0))>=8, "
+                "100*FILL(fb,0)/(FILL(fb,0)+FILL(tmpl,0)), 0)"
+            ),
+            using_metrics={"fb": fb_metric, "tmpl": tmpl_metric},
+            period=Duration.hours(6),
+            label="TemplateFallbackRatePct",
+        )
+        fallback_rate.create_alarm(
+            self,
+            "EmailProcessorTemplateFallbackRateAlarm",
+            alarm_name="po-email-processor-template-fallback-rate",
+            alarm_description=(
+                "po-email-processor deterministic-template coverage collapse: "
+                ">20% of parses fell back to the Bedrock AI extractor"
+            ),
+            threshold=20,
+            evaluation_periods=4,
+            datapoints_to_alarm=2,
+            comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+            treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
+        ).add_alarm_action(cw_actions.SnsAction(alarm_topic))
+
         # S3 event notification → Lambda
         email_bucket.add_event_notification(
             s3.EventType.OBJECT_CREATED,