From e4fc32599fdf1149bf9b03727b3c02c2ff4294ea Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:39:11 -0400 Subject: [PATCH] fix(pipeline): page when classifier has no daily invocation --- cdk/stacks/pipeline_stack.py | 21 +++++++++++++++++++++ docs/RUNBOOK.md | 5 +++-- scripts/apm-wo-uploader.sh | 12 ++++++++---- tests/test_pipeline_synth.py | 22 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/cdk/stacks/pipeline_stack.py b/cdk/stacks/pipeline_stack.py index 23df81e..d9cdcf8 100644 --- a/cdk/stacks/pipeline_stack.py +++ b/cdk/stacks/pipeline_stack.py @@ -311,6 +311,27 @@ class PipelineStack(Stack): ), ) + # Daily-export freshness. The DLQ-depth alarm stays OK when the + # classifier is never invoked. Sum Invocations over 86400s < 1 pages + # a silent day. Missing data is the outage (no datapoint = no run), + # so BREACHING, not the house-default NOT_BREACHING. + self.classifier_fn.metric_invocations( + period=Duration.days(1), + statistic="Sum", + ).create_alarm( + self, + "ClassifierInvocationsAlarm", + alarm_name="apm-wo-analysis-classifier-invocations", + alarm_description=( + "apm-wo-analysis-classifier had fewer than 1 invocation in 24h " + "(daily export missing or S3 trigger broken)" + ), + threshold=1, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + # S3 trigger: any .xlsx/.csv landing under raw/ invokes the classifier. for suffix in (".xlsx", ".csv"): self.exports_bucket.add_event_notification( diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 60084ab..226aed1 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -18,9 +18,10 @@ alerts) aren't surfaced and the dashboard has no new snapshot. Recoverable by re-processing the export; not permanent data loss. ### Detection +- Alarm **`apm-wo-analysis-classifier-invocations`** (Sum Invocations over 86400 s `< 1`, `treat_missing_data=breaching`) — a silent day, including a missing export. - No **daily summary** in the WO Slack channel by the usual time (or no 3rd-escalation alert on a day one's expected). - **Grafana** shows no `dt = today` in the Snapshot Date dropdown / panels empty for today. -- **Messages in `apm-wo-analysis-classifier-dlq`** (SQS) — strongest signal the classifier failed. +- **Messages in `apm-wo-analysis-classifier-dlq`** (SQS) — strongest signal the classifier failed after it *did* run. - CloudWatch errors in `/aws/lambda/apm-wo-analysis-classifier` or `-slack-post`. ### Context @@ -83,7 +84,7 @@ The curated daily APM filter-view export (~350 WOs, `.xlsx`/`.csv`) reaches S3 b **Verify the agent:** ```bash launchctl list | grep apm-wo-uploader # present + last exit 0 -tail -f ~/apm-wo-drop/uploader.log # per-run logging (TBD: confirm log path) +tail -f ~/.local/log/apm-wo-uploader.log # per-run logging (outside WatchPaths) ``` **Common upstream issues:** agent unloaded (`launchctl load -w …plist`); script moved back into `~/Documents` (TCC blocks it — `LastExitStatus=32256`); `apm-wo-drop` diff --git a/scripts/apm-wo-uploader.sh b/scripts/apm-wo-uploader.sh index e009786..fa4cd73 100755 --- a/scripts/apm-wo-uploader.sh +++ b/scripts/apm-wo-uploader.sh @@ -14,15 +14,19 @@ set -euo pipefail DROP_DIR="$HOME/apm-wo-drop" UPLOADED_DIR="$DROP_DIR/uploaded" -LOG_FILE="$DROP_DIR/.upload.log" +# Log and lock MUST sit outside WatchPaths (~/apm-wo-drop). Writing them +# inside the drop folder retriggers launchd on every log line and lock +# mkdir/rmdir, which is what ballooned .upload.log to tens of MB. +STATE_DIR="$HOME/.local/log" +LOG_FILE="$STATE_DIR/apm-wo-uploader.log" +LOCK_DIR="$STATE_DIR/apm-wo-uploader.lock" BUCKET="apm-wo-analysis-exports-328440206208" PROFILE="${APM_WO_AWS_PROFILE:-apm-wo-drop}" -mkdir -p "$UPLOADED_DIR" +mkdir -p "$UPLOADED_DIR" "$STATE_DIR" exec >> "$LOG_FILE" 2>&1 # Single-flight: WatchPaths can fire several times for one save. -LOCK_DIR="$DROP_DIR/.upload.lock" if ! mkdir "$LOCK_DIR" 2>/dev/null; then echo "$(date '+%Y-%m-%dT%H:%M:%S') skipping — another run holds the lock" exit 0 @@ -50,7 +54,7 @@ for f in "$DROP_DIR"/*.xlsx "$DROP_DIR"/*.csv; do else echo "$(date '+%Y-%m-%dT%H:%M:%S') FAIL $name" failed_count=$((failed_count + 1)) - osascript -e "display notification \"Failed to upload $name — see .upload.log\" with title \"APM WO Uploader\" sound name \"Basso\"" 2>/dev/null || true + osascript -e "display notification \"Failed to upload $name — see $LOG_FILE\" with title \"APM WO Uploader\" sound name \"Basso\"" 2>/dev/null || true fi done diff --git a/tests/test_pipeline_synth.py b/tests/test_pipeline_synth.py index 641201c..ffdf70f 100644 --- a/tests/test_pipeline_synth.py +++ b/tests/test_pipeline_synth.py @@ -165,6 +165,28 @@ def test_classifier_has_dlq(): ) +def test_classifier_daily_invocation_alarm_treats_missing_as_breaching(): + # A silent day publishes no Invocations datapoint. NOT_BREACHING would + # hide the outage; BREACHING is the page. Dimension Value is a Ref to the + # function (function_name is set, but CDK still Refs the resource). + _template().has_resource_properties( + "AWS::CloudWatch::Alarm", + Match.object_like( + { + "AlarmName": "apm-wo-analysis-classifier-invocations", + "Namespace": "AWS/Lambda", + "MetricName": "Invocations", + "Statistic": "Sum", + "Period": 86400, + "Threshold": 1, + "ComparisonOperator": "LessThanThreshold", + "EvaluationPeriods": 1, + "TreatMissingData": "breaching", + } + ), + ) + + def test_interactions_stage_is_throttled(): # The public Slack interactions endpoint caps rate/burst. _template().has_resource_properties(