From 0d654edb359072c8de6e399d5362e0a99cfb7630 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:32:10 -0400 Subject: [PATCH] fix: Fix noisy CIS 4.1 unauthorized-API alarm; drop redundant billing alarm (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix noisy CIS 4.1 unauthorized-API alarm; drop redundant billing alarm CIS 4.1 (cis-UnauthorizedAPICalls) flapped OK<->ALARM 15 times in 30 days, all from benign AWS-service AccessDenied noise (CloudFormation deploy/drift describe-scans, AWS Config recorder). A single CFN run on 2026-07-07 emitted 100+ such denials in 15 min, tripping the alarm and burying the real CIS 4.1 security signal in email noise (alert fatigue). - Group both error codes so the exclusions apply to the whole filter (the old pattern leaked the UnauthorizedOperation branch past the exclusions due to && binding tighter than ||). - Exclude denials whose sourceIPAddress is an AWS service host (*.amazonaws.com) — AWS acting on our behalf, not a principal of concern. Real unauthorized calls from a console/CLI/attacker present a routable IP and are still counted. Validated against the trail log group: spike window 107 -> 4 matches, the 4 remaining all from a routable admin IP (genuine activity CIS should retain). - Keep the 3/3 evaluation as a backstop against one-off human fat-fingers. Billing: deleted the manually-created AWS-MonthlyBilling CloudWatch alarm ($50 threshold on EstimatedCharges, routed to site-alerts). It was unmanaged drift, permanently in ALARM, and fully redundant with the managed M-10 budget (seahaven-monthly-cost). README updated with rationale + restore command. * Address sh-security-review: scope CIS 4.1 exclusion to named benign sources The high-recall security review (detector fan-out + proof-or-kill verifier) confirmed a MEDIUM detection blind spot in the first revision: excluding all `*.amazonaws.com` source hosts would hide denials driven through ANY AWS service (SSM Automation, Step Functions, Lambda, etc.), which CloudTrail records with that service's host as sourceIPAddress — i.e. service-proxied privesc/recon attempts would evade CIS 4.1. Remediation: scope the exclusion to the specific benign sources that actually flap this account — `*cloudformation.amazonaws.com` (covers both cloudformation. and hooks.cloudformation.) and `config.amazonaws.com` — plus the pre-existing delivery.logs exclusion. Every other service-proxied denial is now retained. Residual (accepted, documented inline): CloudFormation/Config- proxied denials are still excluded — that path needs near-admin privilege (CreateStack + PassRole), successful changes still trip the other CIS 4.x alarms, and GuardDuty backstops. Validated on the live trail log group: spike window still 107 -> 4 matches (identical noise suppression), the 4 from a routable admin IP. tsc + synth clean. --- README.md | 15 +++++++++++++-- lib/cis-monitoring.ts | 42 +++++++++++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 688c86d..ebc88a8 100644 --- a/README.md +++ b/README.md @@ -177,8 +177,19 @@ aws ce update-cost-allocation-tags-status --cost-allocation-tags-status \ **L-8 (billing-metrics preference) is OUTSTANDING — console only.** Enabling the CloudWatch `EstimatedCharges` metric in us-east-1 requires turning on *Receive Billing Alerts* under Billing → Billing preferences; there is no public API/CLI. -The M-10 budget already provides cost alerting independent of that metric, so -this only affects the legacy `AWS-MonthlyBilling` CloudWatch alarm (L-8). +The M-10 budget (`seahaven-monthly-cost`, 80%/100% actual + 100% forecast) +provides cost alerting independent of that metric. + +> The legacy, manually-created `AWS-MonthlyBilling` CloudWatch alarm ($50 +> threshold on `EstimatedCharges`, routed to `site-alerts`) was **deleted +> 2026-07-07** as unmanaged drift: it was fully redundant with the M-10 budget, +> sat permanently in ALARM (spend has far exceeded $50/mo), and was never in +> IaC. Billing alerting is now solely the managed M-10 budget. To restore the +> old alarm if ever needed: `aws cloudwatch put-metric-alarm --alarm-name +> AWS-MonthlyBilling --namespace AWS/Billing --metric-name EstimatedCharges +> --dimensions Name=Currency,Value=USD --statistic Maximum --period 86400 +> --evaluation-periods 1 --threshold 50 --comparison-operator GreaterThanThreshold +> --alarm-actions arn:aws:sns:us-east-1:328440206208:site-alerts`. ### Monitoring + logging (audit Day 2) diff --git a/lib/cis-monitoring.ts b/lib/cis-monitoring.ts index 34b5740..7283ccf 100644 --- a/lib/cis-monitoring.ts +++ b/lib/cis-monitoring.ts @@ -39,15 +39,43 @@ const CIS_CONTROLS: CisControl[] = [ { id: "UnauthorizedApiCalls", metricName: "UnauthorizedAPICalls", + // The previous pattern relied on `&&` binding tighter than `||`, so the + // sourceIPAddress/HeadBucket exclusions applied ONLY to the AccessDenied + // branch, and the sole IP exclusion was delivery.logs.amazonaws.com. That + // left the filter counting the dominant source of benign noise: AccessDenied + // /*UnauthorizedOperation records generated by AWS services acting on our + // behalf — CloudFormation deploy/drift describe-scans (cloudformation. and + // hooks.cloudformation.amazonaws.com), the AWS Config recorder, etc. On + // 2026-07-07 a single CFN run emitted 100+ such denials in 15 minutes and + // flapped this alarm; it transitioned OK<->ALARM 15 times in 30 days, all + // benign, drowning the CIS 4.1 signal in email noise (alert fatigue). + // + // Fix: (1) group both error codes so the exclusions apply to the whole + // filter (not just the AccessDenied branch), and (2) drop only the SPECIFIC + // benign service sources that actually flap this account — CloudFormation + // (deploy/drift describe-scans, `cloudformation.` and `hooks.cloudformation.`) + // and the Config recorder (`config.`) — plus the pre-existing delivery.logs + // exclusion. + // + // SECURITY NOTE (sh-security-review 2026-07-07): an earlier revision excluded + // ALL `*.amazonaws.com` source hosts. That was rejected — a confirmed MEDIUM + // blind spot: denials driven through OTHER services (SSM Automation, Step + // Functions, Lambda, etc.) are recorded with that service's host as the + // sourceIPAddress, so a blanket exclusion would hide service-proxied + // privesc/recon attempts. Scoping to the named benign hosts keeps every other + // service-proxied denial in scope. Residual (accepted): denials proxied + // specifically through CloudFormation/Config are still excluded — that path + // requires near-admin privilege (cloudformation:CreateStack + iam:PassRole), + // any *successful* change still trips the other CIS 4.x alarms, and GuardDuty + // provides defence-in-depth. Direct console/CLI/credential denials always + // present a routable IP and are always counted. pattern: - '{ ($.errorCode = "*UnauthorizedOperation") || ($.errorCode = "AccessDenied*") && ($.sourceIPAddress != "delivery.logs.amazonaws.com") && ($.eventName != "HeadBucket") }', + '{ (($.errorCode = "*UnauthorizedOperation") || ($.errorCode = "AccessDenied*")) && ($.sourceIPAddress != "*cloudformation.amazonaws.com") && ($.sourceIPAddress != "config.amazonaws.com") && ($.sourceIPAddress != "delivery.logs.amazonaws.com") && ($.eventName != "HeadBucket") }', description: "CIS 4.1 — unauthorized API calls", - // This metric is high-volume: a single CloudFormation/CDK deploy can emit a - // burst of benign describe-API denials, and any one-off console fat-finger - // trips a 1/1 alarm and self-recovers, producing notification storms. Require - // 3 consecutive breaching 5-min periods so only *sustained* unauthorized - // activity (e.g. a misconfigured role failing every call) pages. Detection of - // a real persistent problem is preserved; transient bursts are filtered. + // Belt-and-suspenders after the service-noise exclusion above: still require + // 3 consecutive breaching 5-min periods so a one-off human fat-finger that + // self-recovers doesn't page, while sustained unauthorized activity (e.g. a + // misconfigured role failing every call) still trips within ~15 minutes. evaluationPeriods: 3, datapointsToAlarm: 3, },