mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-07 16:08:54 +00:00
Add PO fallback-rate alarm retuned for ~57 emails/day
The WO alarm's 15-min period and >=10-sample floor assume ~760/day and would be structurally dead at PO volume (a 15-min period holds ~0.6 emails, so the floor is never met). Retune: 6-hour periods (~14.25 expected emails), IF((fb+tmpl)>=8,...) volume floor so a single email can never breach a datapoint (1/8 = 12.5% < 20%), threshold >20% against a ~1% expected baseline, eval 4 / datapoints 2 (24h span) so noise self-clears while total template drift pages within ~12h. No element-wise MAX in the math expression (post-#102 rule); ALARM-only SnsAction to site-alerts, NOT_BREACHING. Gated with 'npx cdk synth po-ingest'. Also add template_parser.py to the bundling cp list -- without it every deployed invocation would ImportError (unit tests cannot catch an asset-bundling omission). Refs: #99, #102
This commit is contained in:
parent
3418c7c819
commit
29ad891290
1 changed files with 64 additions and 1 deletions
|
|
@ -244,7 +244,7 @@ class PoIngestStack(Stack):
|
|||
"-c",
|
||||
"pip install --platform manylinux2014_aarch64 --only-binary=:all: "
|
||||
"-r requirements.txt -t /asset-output && "
|
||||
"cp handler.py ses_auth.py /asset-output/",
|
||||
"cp handler.py ses_auth.py template_parser.py /asset-output/",
|
||||
],
|
||||
),
|
||||
),
|
||||
|
|
@ -383,6 +383,69 @@ class PoIngestStack(Stack):
|
|||
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
||||
|
||||
# --- Template fallback-rate alarm: po-email-processor ---
|
||||
# The processor tries a deterministic template parse first and only calls
|
||||
# the Bedrock AI extractor on a miss/invalid. A sustained rise in the
|
||||
# ai_fallback share signals Coupa template drift (coverage collapse).
|
||||
# EMF metric Seahaven/PoIngest/ParseOutcome, dimensioned by ParseMethod
|
||||
# (template|ai_fallback).
|
||||
#
|
||||
# RETUNED for PO volume (~57 emails/day ≈ 14.25 per 6h period) -- the WO
|
||||
# alarm's 15-min period / >=10-sample floor assume ~760/day and would be
|
||||
# structurally DEAD here (a 15-min period holds ~0.6 PO emails, so the
|
||||
# floor is never met and the IF always takes the 0 branch):
|
||||
# * period 6h: a stable ~14-email denominator per datapoint.
|
||||
# * volume floor >=8: at the floor, one fallback email = 12.5% < 20%,
|
||||
# so a single email can NEVER breach a datapoint; a breach needs >=2
|
||||
# fallbacks in one 6h window (2/8 = 25%) or >=3 at typical volume
|
||||
# (3/14 ≈ 21%). Sparse overnight/weekend windows (<8 emails) take
|
||||
# the 0 branch -- non-breaching by design (accepted trade: a Friday-
|
||||
# evening drift may not page until weekend volume accrues).
|
||||
# * threshold >20%: expected baseline fallback ≈1% (comments 0.55% +
|
||||
# multi-line 0.18% + non-USD 0) -- far below the threshold.
|
||||
# * 2 of 4 datapoints (24h span): isolated noise self-clears, while
|
||||
# total template drift (100% fallback) pages within ~12h.
|
||||
# Post-#102 rule: NO element-wise MAX(timeseries, scalar) in alarm math;
|
||||
# the IF volume floor guarantees the non-zero denominator. Any change to
|
||||
# this expression must be gated by `npx cdk synth po-ingest`.
|
||||
fb_metric = cloudwatch.Metric(
|
||||
namespace="Seahaven/PoIngest",
|
||||
metric_name="ParseOutcome",
|
||||
dimensions_map={"ParseMethod": "ai_fallback"},
|
||||
statistic="Sum",
|
||||
period=Duration.hours(6),
|
||||
)
|
||||
tmpl_metric = cloudwatch.Metric(
|
||||
namespace="Seahaven/PoIngest",
|
||||
metric_name="ParseOutcome",
|
||||
dimensions_map={"ParseMethod": "template"},
|
||||
statistic="Sum",
|
||||
period=Duration.hours(6),
|
||||
)
|
||||
fallback_rate = cloudwatch.MathExpression(
|
||||
expression=(
|
||||
"IF((FILL(fb,0)+FILL(tmpl,0))>=8, "
|
||||
"100*FILL(fb,0)/(FILL(fb,0)+FILL(tmpl,0)), 0)"
|
||||
),
|
||||
using_metrics={"fb": fb_metric, "tmpl": tmpl_metric},
|
||||
period=Duration.hours(6),
|
||||
label="TemplateFallbackRatePct",
|
||||
)
|
||||
fallback_rate.create_alarm(
|
||||
self,
|
||||
"EmailProcessorTemplateFallbackRateAlarm",
|
||||
alarm_name="po-email-processor-template-fallback-rate",
|
||||
alarm_description=(
|
||||
"po-email-processor deterministic-template coverage collapse: "
|
||||
">20% of parses fell back to the Bedrock AI extractor"
|
||||
),
|
||||
threshold=20,
|
||||
evaluation_periods=4,
|
||||
datapoints_to_alarm=2,
|
||||
comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
||||
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
||||
|
||||
# S3 event notification → Lambda
|
||||
email_bucket.add_event_notification(
|
||||
s3.EventType.OBJECT_CREATED,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue