diff --git a/README.md b/README.md index 98033c3..6954744 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Coupa PO emails are received at `amazon_po@int.seahaven.com`, parsed by Claude H |---|---|---| | `po-email-processor` | S3 ObjectCreated | Claude extraction + DynamoDB write | | `po-ingest-site-extractor` | DynamoDB Streams | Site code/address extraction -> `verified-sites` | -| `po-web-ui` | Function URL | HTML dashboard | +| `po-web-ui` | Manual invoke | HTML dashboard (public Function URL removed 2026-06-08, INFRA-74) | **Tables:** - `purchase-orders` (PK: `po_number`, Streams: NEW_IMAGE) — shared with seahaven-slack-bot (read-only; see Shared Resources) @@ -49,7 +49,7 @@ Amazon APM work order emails (from Hexagon EAM / HxGN SmartCloud) are received a | Function | Trigger | Purpose | |---|---|---| | `workorder-email-processor` | S3 ObjectCreated | Claude extraction + DynamoDB write | -| `workorder-web-ui` | Function URL | HTML dashboard | +| `workorder-web-ui` | Manual invoke | HTML dashboard (public Function URL removed 2026-06-08, INFRA-74) | **Tables:** - `WorkOrders` (PK: `work_order_id`, GSIs: `site-code-index`, `status-index`) @@ -67,6 +67,8 @@ All Lambdas: Python 3.12, ARM64, 60-day log retention. **SES:** Both stacks add rules to the shared `INBOUND_MAIL` receipt rule set on `int.seahaven.com`. +**Failure handling (INFRA-41):** Each email-processor is async-invoked (S3 → Lambda). Both have a CDK-managed SQS dead-letter queue (`dead_letter_queue=`, 14-day retention, SSL-enforced) so a failed parse is captured rather than silently dropped after Lambda's retries, plus an ALARM-only CloudWatch `Errors` alarm (Sum, threshold > 0) wired to the shared `site-alerts` SNS topic. + ## Shared Resources ### `purchase-orders` table (owned here) diff --git a/cdk/po_stack.py b/cdk/po_stack.py index cf0ea33..144d9e9 100644 --- a/cdk/po_stack.py +++ b/cdk/po_stack.py @@ -5,6 +5,8 @@ from aws_cdk import ( Duration, RemovalPolicy, Stack, + aws_cloudwatch as cloudwatch, + aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, aws_lambda as lambda_, aws_lambda_event_sources as lambda_event_sources, @@ -14,6 +16,8 @@ from aws_cdk import ( aws_ses as ses, aws_ses_actions as ses_actions, aws_secretsmanager as secretsmanager, + aws_sns as sns, + aws_sqs as sqs, ) from constructs import Construct @@ -58,6 +62,18 @@ class PoIngestStack(Stack): removal_policy=RemovalPolicy.RETAIN, ) + # --- DLQ for failed async invocations (INFRA-41 / audit H-8) --- + # SES → S3 → Lambda is async; without an OnFailure destination a failed + # parse (bad email, transient error) is silently dropped after Lambda's + # retries. CDK generates the queue name to avoid colliding with the + # interim CLI-created po-email-processor-dlq (removed post-deploy). + email_processor_dlq = sqs.Queue( + self, + "EmailProcessorDlq", + retention_period=Duration.days(14), + enforce_ssl=True, + ) + # --- Lambda function --- email_processor = lambda_.Function( self, @@ -82,6 +98,7 @@ class PoIngestStack(Stack): timeout=Duration.seconds(60), memory_size=256, log_retention=logs.RetentionDays.TWO_MONTHS, + dead_letter_queue=email_processor_dlq, environment={ "PO_TABLE": "purchase-orders", "ANTHROPIC_API_KEY_SECRET_ARN": anthropic_secret.secret_arn, @@ -93,6 +110,29 @@ class PoIngestStack(Stack): po_table.grant_read_write_data(email_processor) anthropic_secret.grant_read(email_processor) + # --- Errors alarm (INFRA-41 / audit H-8) --- + # ALARM-only (no OK action, per the CloudWatch-alarm preference) to the + # shared site-alerts topic (CMK alias/seahaven-alarm-topics lives on the + # topic). Any errored invocation in a 5-min window pages. + alarm_topic = sns.Topic.from_topic_arn( + self, + "SiteAlertsTopic", + f"arn:aws:sns:{self.region}:{self.account}:site-alerts", + ) + email_processor.metric_errors( + period=Duration.minutes(5), + statistic="Sum", + ).create_alarm( + self, + "EmailProcessorErrorsAlarm", + alarm_name="po-email-processor-errors", + alarm_description="po-email-processor async invocation errors", + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + # S3 event notification → Lambda email_bucket.add_event_notification( s3.EventType.OBJECT_CREATED, @@ -138,14 +178,10 @@ class PoIngestStack(Stack): po_table.grant_read_data(web_ui) - # Function URL for direct access - web_url = web_ui.add_function_url( - auth_type=lambda_.FunctionUrlAuthType.NONE, - ) - - cdk.CfnOutput( - self, "WebUIUrl", value=web_url.url, description="PO Dashboard URL" - ) + # Public Function URL removed 2026-06-08 (INFRA-74 / audit C-5): the + # unauthenticated FunctionUrlAuthType.NONE URL was deleted out-of-band + # via CLI. Removing the construct (and its auto-generated Principal:* + # invoke permission) reconciles IaC with the live state. # --- Verified sites table (extracted from PO ship-to addresses) --- verified_sites_table = dynamodb.Table( diff --git a/cdk/wo_stack.py b/cdk/wo_stack.py index 37355ac..4cba41b 100644 --- a/cdk/wo_stack.py +++ b/cdk/wo_stack.py @@ -5,6 +5,8 @@ from aws_cdk import ( Duration, RemovalPolicy, Stack, + aws_cloudwatch as cloudwatch, + aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, aws_lambda as lambda_, aws_logs as logs, @@ -13,6 +15,8 @@ from aws_cdk import ( aws_ses as ses, aws_ses_actions as ses_actions, aws_secretsmanager as secretsmanager, + aws_sns as sns, + aws_sqs as sqs, ) from constructs import Construct @@ -73,6 +77,18 @@ class WorkorderIngestStack(Stack): removal_policy=RemovalPolicy.RETAIN, ) + # --- DLQ for failed async invocations (INFRA-41 / audit H-8) --- + # SES → S3 → Lambda is async; without an OnFailure destination a failed + # parse (bad email, transient error) is silently dropped after Lambda's + # retries. CDK generates the queue name to avoid colliding with the + # interim CLI-created workorder-email-processor-dlq (removed post-deploy). + email_processor_dlq = sqs.Queue( + self, + "EmailProcessorDlq", + retention_period=Duration.days(14), + enforce_ssl=True, + ) + # --- Lambda function --- email_processor = lambda_.Function( self, @@ -97,6 +113,7 @@ class WorkorderIngestStack(Stack): timeout=Duration.seconds(60), memory_size=256, log_retention=logs.RetentionDays.TWO_MONTHS, + dead_letter_queue=email_processor_dlq, environment={ "WORK_ORDERS_TABLE": work_orders_table.table_name, "COMMENTS_TABLE": comments_table.table_name, @@ -110,6 +127,29 @@ class WorkorderIngestStack(Stack): comments_table.grant_read_write_data(email_processor) anthropic_secret.grant_read(email_processor) + # --- Errors alarm (INFRA-41 / audit H-8) --- + # ALARM-only (no OK action, per the CloudWatch-alarm preference) to the + # shared site-alerts topic (CMK alias/seahaven-alarm-topics lives on the + # topic). Any errored invocation in a 5-min window pages. + alarm_topic = sns.Topic.from_topic_arn( + self, + "SiteAlertsTopic", + f"arn:aws:sns:{self.region}:{self.account}:site-alerts", + ) + email_processor.metric_errors( + period=Duration.minutes(5), + statistic="Sum", + ).create_alarm( + self, + "EmailProcessorErrorsAlarm", + alarm_name="workorder-email-processor-errors", + alarm_description="workorder-email-processor async invocation errors", + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + # S3 event notification -> Lambda email_bucket.add_event_notification( s3.EventType.OBJECT_CREATED, @@ -156,11 +196,7 @@ class WorkorderIngestStack(Stack): work_orders_table.grant_read_data(web_ui) comments_table.grant_read_data(web_ui) - # Function URL for direct access - web_url = web_ui.add_function_url( - auth_type=lambda_.FunctionUrlAuthType.NONE, - ) - - cdk.CfnOutput( - self, "WebUIUrl", value=web_url.url, description="Work Order Dashboard URL" - ) + # Public Function URL removed 2026-06-08 (INFRA-74 / audit C-5): the + # unauthenticated FunctionUrlAuthType.NONE URL was deleted out-of-band + # via CLI. Removing the construct (and its auto-generated Principal:* + # invoke permission) reconciles IaC with the live state.