mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-07 16:08:54 +00:00
Add PO fallback-rate alarm retuned for ~57 emails/day
The WO alarm's 15-min period and >=10-sample floor assume ~760/day and would be structurally dead at PO volume (a 15-min period holds ~0.6 emails, so the floor is never met). Retune: 6-hour periods (~14.25 expected emails), IF((fb+tmpl)>=8,...) volume floor so a single email can never breach a datapoint (1/8 = 12.5% < 20%), threshold >20% against a ~1% expected baseline, eval 4 / datapoints 2 (24h span) so noise self-clears while total template drift pages within ~12h. No element-wise MAX in the math expression (post-#102 rule); ALARM-only SnsAction to site-alerts, NOT_BREACHING. Gated with 'npx cdk synth po-ingest'. Also add template_parser.py to the bundling cp list -- without it every deployed invocation would ImportError (unit tests cannot catch an asset-bundling omission). Refs: #99, #102
This commit is contained in:
parent
3418c7c819
commit
29ad891290
1 changed files with 64 additions and 1 deletions
|
|
@ -244,7 +244,7 @@ class PoIngestStack(Stack):
|
||||||
"-c",
|
"-c",
|
||||||
"pip install --platform manylinux2014_aarch64 --only-binary=:all: "
|
"pip install --platform manylinux2014_aarch64 --only-binary=:all: "
|
||||||
"-r requirements.txt -t /asset-output && "
|
"-r requirements.txt -t /asset-output && "
|
||||||
"cp handler.py ses_auth.py /asset-output/",
|
"cp handler.py ses_auth.py template_parser.py /asset-output/",
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|
@ -383,6 +383,69 @@ class PoIngestStack(Stack):
|
||||||
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||||
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
||||||
|
|
||||||
|
# --- Template fallback-rate alarm: po-email-processor ---
|
||||||
|
# The processor tries a deterministic template parse first and only calls
|
||||||
|
# the Bedrock AI extractor on a miss/invalid. A sustained rise in the
|
||||||
|
# ai_fallback share signals Coupa template drift (coverage collapse).
|
||||||
|
# EMF metric Seahaven/PoIngest/ParseOutcome, dimensioned by ParseMethod
|
||||||
|
# (template|ai_fallback).
|
||||||
|
#
|
||||||
|
# RETUNED for PO volume (~57 emails/day ≈ 14.25 per 6h period) -- the WO
|
||||||
|
# alarm's 15-min period / >=10-sample floor assume ~760/day and would be
|
||||||
|
# structurally DEAD here (a 15-min period holds ~0.6 PO emails, so the
|
||||||
|
# floor is never met and the IF always takes the 0 branch):
|
||||||
|
# * period 6h: a stable ~14-email denominator per datapoint.
|
||||||
|
# * volume floor >=8: at the floor, one fallback email = 12.5% < 20%,
|
||||||
|
# so a single email can NEVER breach a datapoint; a breach needs >=2
|
||||||
|
# fallbacks in one 6h window (2/8 = 25%) or >=3 at typical volume
|
||||||
|
# (3/14 ≈ 21%). Sparse overnight/weekend windows (<8 emails) take
|
||||||
|
# the 0 branch -- non-breaching by design (accepted trade: a Friday-
|
||||||
|
# evening drift may not page until weekend volume accrues).
|
||||||
|
# * threshold >20%: expected baseline fallback ≈1% (comments 0.55% +
|
||||||
|
# multi-line 0.18% + non-USD 0) -- far below the threshold.
|
||||||
|
# * 2 of 4 datapoints (24h span): isolated noise self-clears, while
|
||||||
|
# total template drift (100% fallback) pages within ~12h.
|
||||||
|
# Post-#102 rule: NO element-wise MAX(timeseries, scalar) in alarm math;
|
||||||
|
# the IF volume floor guarantees the non-zero denominator. Any change to
|
||||||
|
# this expression must be gated by `npx cdk synth po-ingest`.
|
||||||
|
fb_metric = cloudwatch.Metric(
|
||||||
|
namespace="Seahaven/PoIngest",
|
||||||
|
metric_name="ParseOutcome",
|
||||||
|
dimensions_map={"ParseMethod": "ai_fallback"},
|
||||||
|
statistic="Sum",
|
||||||
|
period=Duration.hours(6),
|
||||||
|
)
|
||||||
|
tmpl_metric = cloudwatch.Metric(
|
||||||
|
namespace="Seahaven/PoIngest",
|
||||||
|
metric_name="ParseOutcome",
|
||||||
|
dimensions_map={"ParseMethod": "template"},
|
||||||
|
statistic="Sum",
|
||||||
|
period=Duration.hours(6),
|
||||||
|
)
|
||||||
|
fallback_rate = cloudwatch.MathExpression(
|
||||||
|
expression=(
|
||||||
|
"IF((FILL(fb,0)+FILL(tmpl,0))>=8, "
|
||||||
|
"100*FILL(fb,0)/(FILL(fb,0)+FILL(tmpl,0)), 0)"
|
||||||
|
),
|
||||||
|
using_metrics={"fb": fb_metric, "tmpl": tmpl_metric},
|
||||||
|
period=Duration.hours(6),
|
||||||
|
label="TemplateFallbackRatePct",
|
||||||
|
)
|
||||||
|
fallback_rate.create_alarm(
|
||||||
|
self,
|
||||||
|
"EmailProcessorTemplateFallbackRateAlarm",
|
||||||
|
alarm_name="po-email-processor-template-fallback-rate",
|
||||||
|
alarm_description=(
|
||||||
|
"po-email-processor deterministic-template coverage collapse: "
|
||||||
|
">20% of parses fell back to the Bedrock AI extractor"
|
||||||
|
),
|
||||||
|
threshold=20,
|
||||||
|
evaluation_periods=4,
|
||||||
|
datapoints_to_alarm=2,
|
||||||
|
comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
||||||
|
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||||
|
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
||||||
|
|
||||||
# S3 event notification → Lambda
|
# S3 event notification → Lambda
|
||||||
email_bucket.add_event_notification(
|
email_bucket.add_event_notification(
|
||||||
s3.EventType.OBJECT_CREATED,
|
s3.EventType.OBJECT_CREATED,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue