mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-09-30 09:33:15 +00:00
00d0d32337
Some checks are pending
Deploy / deploy (push) Waiting to run
First seahaven-prod deploy failed CREATE on the sender-auth MetricFilter: it imported /aws/lambda/<fn> by name, which pre-existed in mgmt but not in a fresh account. All 5 functions now get an explicit logs.LogGroup (TWO_MONTHS, RETAIN) via common.make_function_log_group, and the metric filter takes the construct so CFN orders it after the group exists. Also removes the deprecated LogRetention custom resource and its wildcard logs:PutRetentionPolicy role (CKV_AWS_111). Function roles keep AWSLambdaBasicExecutionRole (verified in the synthesized template), so log-write permissions are unchanged; cross-review's grant_write FIX was a false positive on that basis. Supersedes PR #84, which hardcoded the mgmt logs-CMK ARN and predates the common.py refactor. mgmt collision note: these CREATEs would collide with the pre-existing groups in mgmt; acceptable because the deploy secret now targets prod and mgmt is frozen pending decommission.
412 lines
19 KiB
Python
412 lines
19 KiB
Python
"""CDK stack for the work order email ingestion pipeline."""
|
|
|
|
import aws_cdk as cdk
|
|
from aws_cdk import (
|
|
Duration,
|
|
RemovalPolicy,
|
|
Stack,
|
|
aws_cloudwatch as cloudwatch,
|
|
aws_cloudwatch_actions as cw_actions,
|
|
aws_dynamodb as dynamodb,
|
|
aws_lambda as lambda_,
|
|
aws_s3 as s3,
|
|
aws_s3_notifications as s3n,
|
|
aws_ses as ses,
|
|
aws_ses_actions as ses_actions,
|
|
aws_secretsmanager as secretsmanager,
|
|
aws_sns as sns,
|
|
)
|
|
from constructs import Construct
|
|
|
|
import common
|
|
|
|
|
|
class WorkorderIngestStack(Stack):
|
|
def __init__(self, scope: Construct, construct_id: str, **kwargs):
|
|
super().__init__(scope, construct_id, **kwargs)
|
|
|
|
# --- Shared alarm SNS topic (site-alerts) ---
|
|
# Imported once near the top so every alarm in this stack reuses the same
|
|
# Topic construct instance (avoids duplicate logical IDs). ALARM-only
|
|
# SnsAction; no OK action, per the CloudWatch-alarm preference. The
|
|
# topic's CMK (alias/seahaven-alarm-topics) lives on the topic itself.
|
|
alarm_topic = sns.Topic.from_topic_arn(
|
|
self,
|
|
"SiteAlertsTopic",
|
|
f"arn:aws:sns:{self.region}:{self.account}:site-alerts",
|
|
)
|
|
|
|
# --- S3 bucket for raw emails ---
|
|
email_bucket = common.make_email_bucket(
|
|
self, "EmailBucket", "workorder-ingest-emails"
|
|
)
|
|
|
|
# --- DynamoDB tables ---
|
|
work_orders_table = dynamodb.Table(
|
|
self,
|
|
"WorkOrdersTable",
|
|
table_name="WorkOrders",
|
|
partition_key=dynamodb.Attribute(
|
|
name="work_order_id",
|
|
type=dynamodb.AttributeType.STRING,
|
|
),
|
|
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
|
|
removal_policy=RemovalPolicy.RETAIN,
|
|
)
|
|
# site-code-index and status-index GSIs removed 2026-06-03 (audit M-20):
|
|
# 0 reads in 30d against ~50k WCU each of write amplification. Re-add if
|
|
# a site-code or status query path ships.
|
|
|
|
comments_table = dynamodb.Table(
|
|
self,
|
|
"CommentsTable",
|
|
table_name="WorkOrderComments",
|
|
partition_key=dynamodb.Attribute(
|
|
name="work_order_id",
|
|
type=dynamodb.AttributeType.STRING,
|
|
),
|
|
sort_key=dynamodb.Attribute(
|
|
name="comment_id",
|
|
type=dynamodb.AttributeType.STRING,
|
|
),
|
|
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
|
|
removal_policy=RemovalPolicy.RETAIN,
|
|
)
|
|
|
|
# --- Anthropic API key secret removed (Bedrock migration) ---
|
|
# Parsing moved from the Anthropic API to the Bedrock inference profile
|
|
# us.anthropic.claude-haiku-4-5-20251001-v1:0, so no provider API key is
|
|
# needed. The old secret "workorder-ingest/anthropic-api-key" had
|
|
# RemovalPolicy.RETAIN, so it is ORPHANED (not deleted) by this change:
|
|
# delete it manually post-deploy and revoke the stored key at Anthropic.
|
|
|
|
# --- DLQ for failed async invocations (INFRA-41 / audit H-8) ---
|
|
# SES → S3 → Lambda is async; without an OnFailure destination a failed
|
|
# parse (bad email, transient error) is silently dropped after Lambda's
|
|
# retries. CDK generates the queue name to avoid colliding with the
|
|
# interim CLI-created workorder-email-processor-dlq (removed post-deploy).
|
|
email_processor_dlq = common.make_processor_dlq(self, "EmailProcessorDlq")
|
|
|
|
# --- Lambda function ---
|
|
email_processor_log_group = common.make_function_log_group(
|
|
self, "EmailProcessor", "workorder-email-processor"
|
|
)
|
|
email_processor = lambda_.Function(
|
|
self,
|
|
"EmailProcessor",
|
|
function_name="workorder-email-processor",
|
|
runtime=lambda_.Runtime.PYTHON_3_12,
|
|
architecture=lambda_.Architecture.ARM_64,
|
|
handler="handler.handler",
|
|
code=lambda_.Code.from_asset(
|
|
"../lambdas",
|
|
exclude=["**/__pycache__/**", "**/tests/**", "**/package/**"],
|
|
bundling=cdk.BundlingOptions(
|
|
image=lambda_.Runtime.PYTHON_3_12.bundling_image,
|
|
command=[
|
|
"bash",
|
|
"-c",
|
|
# pip step removed in Phase 7: requirements.txt is now empty
|
|
# (boto3 comes from the Lambda runtime), so nothing is installed
|
|
# and the manylinux pin has nothing to pin. cp-only is safe.
|
|
# shared/*.py ships the four modules extracted to
|
|
# lambdas/shared/ (Phase 3): ses_auth, web_ui_auth,
|
|
# email_parsing, emf. Flat cp keeps the bare-name
|
|
# imports (e.g. `from ses_auth import ...`) resolving
|
|
# unchanged in /asset-output.
|
|
"cp wo/email_processor/*.py /asset-output/ && "
|
|
"cp shared/*.py /asset-output/",
|
|
],
|
|
),
|
|
),
|
|
timeout=Duration.seconds(60),
|
|
memory_size=256,
|
|
log_group=email_processor_log_group,
|
|
dead_letter_queue=email_processor_dlq,
|
|
environment={
|
|
"WORK_ORDERS_TABLE": work_orders_table.table_name,
|
|
"COMMENTS_TABLE": comments_table.table_name,
|
|
"BEDROCK_MODEL_ID": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
|
# Fail-closed sender auth (INFRA-107): the handler only
|
|
# accepts mail whose SES-stamped Authentication-Results
|
|
# header carries dkim=pass for one of these domains. APM
|
|
# mail arrives via the apm@ Google Groups forward, which
|
|
# re-signs as seahaven.com (observed on live traffic
|
|
# 2026-07-15: "dkim=pass header.i=@seahaven.com"; the
|
|
# original hxgnsmartcloud.com signature does not survive
|
|
# the forward). Unset/empty ⇒ the handler rejects all mail.
|
|
"ALLOWED_DKIM_DOMAINS": "seahaven.com",
|
|
},
|
|
)
|
|
|
|
# Grant permissions
|
|
email_bucket.grant_read(email_processor)
|
|
work_orders_table.grant_read_write_data(email_processor)
|
|
comments_table.grant_read_write_data(email_processor)
|
|
|
|
# --- Bedrock InvokeModel grant ---
|
|
# The us.* inference profile can route cross-region, so the grant MUST
|
|
# cover both the inference-profile ARN AND the per-region foundation-model
|
|
# ARNs (empty account field) for every region the profile can reach
|
|
# (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at
|
|
# runtime whenever the profile routes to a region whose foundation-model
|
|
# ARN is not allowed.
|
|
email_processor.add_to_role_policy(common.make_bedrock_invoke_statement(self))
|
|
|
|
# NOTE: The pre-emptive grant_encrypt_decrypt on the shared DynamoDB CMK
|
|
# (alias/seahaven-dynamodb) was removed (security sweep 2026-06-17). The
|
|
# WorkOrders/WorkOrderComments tables are NOT SSE-KMS encrypted with that
|
|
# CMK, so the grant was unused for these tables yet handed
|
|
# wo-email-processor kms:Decrypt on the CMK that also protects the
|
|
# purchase-orders table (cross-stack decrypt reach). Re-add this grant only
|
|
# as part of the actual CMK migration of these tables (INFRA-6), at which
|
|
# point grant_read_write_data on the (then encrypted) tables would propagate
|
|
# the needed key permissions automatically.
|
|
|
|
# --- Standard per-Lambda alarms: workorder-email-processor ---
|
|
# errors (INFRA-41 / audit H-8), throttles, DLQ-visible-messages (dropped
|
|
# emails), and a p95 duration alarm (orphan adoption of the CLI
|
|
# Lambda-Duration-workorder-email-processor under <fn>-duration naming,
|
|
# 45000 ms = 75% of the 60s timeout, eval 3 / dp 2). p95 (NOT p99) is the
|
|
# WO-specific duration statistic. All ALARM-only to site-alerts.
|
|
common.add_standard_lambda_alarms(
|
|
self,
|
|
"EmailProcessor",
|
|
email_processor,
|
|
"workorder-email-processor",
|
|
alarm_topic,
|
|
duration_statistic="p95",
|
|
errors=True,
|
|
dlq=email_processor_dlq,
|
|
descriptions={
|
|
"errors": "workorder-email-processor async invocation errors",
|
|
"throttles": "workorder-email-processor invocation throttles",
|
|
"dlq": "workorder-email-processor DLQ has visible messages (dropped emails)",
|
|
"duration": "workorder-email-processor p95 duration approaching the 60s timeout",
|
|
},
|
|
)
|
|
|
|
# --- Sender-auth rejection alarm (INFRA-107) ---
|
|
# A rejected email (bad/unaligned DKIM verdict) returns normally, so it
|
|
# produces NO Lambda error, NO DLQ message and NO retry -- only a
|
|
# `sender_auth_rejected` warning log. The WO allowlist trusts dkim=pass
|
|
# for seahaven.com on the assumption the apm@ forward re-signs there; if
|
|
# that assumption is wrong (e.g. a Gmail auto-forward re-signs under a
|
|
# different domain), 100% of legitimate work-order mail is silently
|
|
# dropped. This metric filter + alarm turns those warnings into a paging
|
|
# signal so a false-reject storm surfaces instead of a silent outage.
|
|
common.add_sender_auth_rejected_alarm(
|
|
self,
|
|
"EmailProcessor",
|
|
"workorder-email-processor",
|
|
alarm_topic,
|
|
email_processor_log_group,
|
|
)
|
|
|
|
# --- Template fallback-rate alarm: workorder-email-processor ---
|
|
# The processor tries a deterministic template parse first and only calls
|
|
# the Bedrock AI extractor on a miss/invalid. A sustained rise in the
|
|
# ai_fallback share signals Hexagon template drift (coverage collapse).
|
|
# EMF metric Seahaven/WorkorderIngest/ParseOutcome, dimensioned by
|
|
# ParseMethod (template|ai_fallback). 15-min periods (deliberate deviation
|
|
# from the 5-min house style) accumulate a stable denominator at the low
|
|
# ~760/day volume; FILL(0) + a >=10-sample volume floor prevent
|
|
# low-volume false pages and INSUFFICIENT_DATA. ALARM-only SnsAction to
|
|
# site-alerts, no OK action, NOT_BREACHING -- matching the stack idiom.
|
|
# rejected_included=True: the WO expression folds ai_fallback_rejected
|
|
# (rej) into BOTH numerator and denominator -- a drift outage whose AI
|
|
# output also fails the gate must still count as fallback, otherwise it
|
|
# would LOWER the observed rate while silently dropping mail. (Contrast
|
|
# PO, which excludes rej to avoid a pre-call double-count.)
|
|
common.make_fallback_rate_alarm(
|
|
self,
|
|
"EmailProcessorTemplateFallbackRateAlarm",
|
|
namespace="Seahaven/WorkorderIngest",
|
|
alarm_topic=alarm_topic,
|
|
alarm_name="workorder-email-processor-template-fallback-rate",
|
|
alarm_description=(
|
|
"workorder-email-processor deterministic-template coverage "
|
|
"collapse: >15% of parses fell back to the Bedrock AI extractor"
|
|
),
|
|
rejected_included=True,
|
|
period=Duration.minutes(15),
|
|
threshold=15,
|
|
floor=10,
|
|
evaluation_periods=3,
|
|
datapoints_to_alarm=2,
|
|
)
|
|
|
|
# --- AI-fallback rejected alarm: workorder-email-processor ---
|
|
# A parse rejected by the validate_ai_fallback gate is dropped without
|
|
# error/retry/DLQ (fail closed), so like sender-auth rejections it
|
|
# needs its own pager or a sustained rejection condition (prompt-
|
|
# injection probing, or template drift whose AI output fails the gate)
|
|
# stays silent. Same sparse-arrival idiom as the sender-auth-rejected
|
|
# alarm: >=1 rejection per 5-min period, 2 of the last 6 periods (30
|
|
# min), so a lone probe self-clears but a burst pages within ~10 min.
|
|
# Coverage residual (matching the sender-auth-rejected sibling and
|
|
# knowingly accepted): rejections spaced >~25-30 min apart never place
|
|
# two breaching datapoints in one 30-min window, and the fallback-rate
|
|
# alarm dilutes them below 15% against normal template volume, so a
|
|
# *very* sparse silent-drop trickle is not paged by either alarm.
|
|
# EMF emits no datapoint in quiet periods (no metric-filter
|
|
# default_value here); NOT_BREACHING treats those gaps as OK.
|
|
cloudwatch.Metric(
|
|
namespace="Seahaven/WorkorderIngest",
|
|
metric_name="ParseOutcome",
|
|
dimensions_map={"ParseMethod": "ai_fallback_rejected"},
|
|
statistic="Sum",
|
|
period=Duration.minutes(5),
|
|
).create_alarm(
|
|
self,
|
|
"EmailProcessorAiFallbackRejectedAlarm",
|
|
alarm_name="workorder-email-processor-ai-fallback-rejected",
|
|
alarm_description=(
|
|
"workorder-email-processor is rejecting Bedrock AI-fallback "
|
|
"output at the validation gate (possible prompt-injection "
|
|
"probing or template drift silently dropping real mail)"
|
|
),
|
|
threshold=1,
|
|
evaluation_periods=6,
|
|
datapoints_to_alarm=2,
|
|
comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
|
|
|
|
# S3 event notification -> Lambda
|
|
email_bucket.add_event_notification(
|
|
s3.EventType.OBJECT_CREATED,
|
|
s3n.LambdaDestination(email_processor),
|
|
s3.NotificationKeyFilter(prefix="inbound/"),
|
|
)
|
|
|
|
# --- SES Receipt Rule ---
|
|
rule_set = ses.ReceiptRuleSet.from_receipt_rule_set_name(
|
|
self,
|
|
"ExistingRuleSet",
|
|
"INBOUND_MAIL",
|
|
)
|
|
|
|
rule_set.add_rule(
|
|
"WorkorderEmailRule",
|
|
recipients=["apm@int.seahaven.com"],
|
|
actions=[
|
|
ses_actions.S3(
|
|
bucket=email_bucket,
|
|
object_key_prefix="inbound/",
|
|
),
|
|
],
|
|
)
|
|
|
|
# --- Web UI auth token secret ---
|
|
# Shared secret for the web UI auth gate, stored in Secrets Manager and
|
|
# resolved at runtime so the token never appears in CloudFormation templates
|
|
# or Lambda environment variables. Create this secret before deploying
|
|
# either stack; both PO and WO stacks reference it by name.
|
|
web_ui_auth_secret = secretsmanager.Secret.from_secret_name_v2(
|
|
self,
|
|
"WebUiAuthToken",
|
|
"procurement-ingest/web-ui-auth-token",
|
|
)
|
|
|
|
# --- Web UI Lambda ---
|
|
web_ui_log_group = common.make_function_log_group(
|
|
self, "WebUI", "workorder-web-ui"
|
|
)
|
|
web_ui = lambda_.Function(
|
|
self,
|
|
"WebUI",
|
|
function_name="workorder-web-ui",
|
|
runtime=lambda_.Runtime.PYTHON_3_12,
|
|
architecture=lambda_.Architecture.ARM_64,
|
|
handler="handler.handler",
|
|
code=lambda_.Code.from_asset(
|
|
"../lambdas",
|
|
exclude=["**/__pycache__/**"],
|
|
bundling=cdk.BundlingOptions(
|
|
image=lambda_.Runtime.PYTHON_3_12.bundling_image,
|
|
command=[
|
|
"bash",
|
|
"-c",
|
|
# web_ui_auth.py is shared (lambdas/shared) and must land
|
|
# FLAT beside handler.py so
|
|
# `from web_ui_auth import is_authenticated` resolves at
|
|
# runtime. Only web_ui_auth is copied from shared/. WO
|
|
# baseline keeps __init__.py and requirements.txt, so the
|
|
# whole web_ui dir is copied; deployed file list becomes
|
|
# {__init__, handler, requirements.txt, web_ui_auth}.
|
|
# NOTE: the top-level `exclude=` on from_asset only
|
|
# filters the asset-hash fingerprint, NOT the directory
|
|
# Docker bundling actually mounts, so a local
|
|
# __pycache__ on disk at synth time WOULD otherwise leak
|
|
# into the bundled zip -- strip it explicitly post-cp
|
|
# instead of relying on exclude.
|
|
"cp -r wo/web_ui/. /asset-output/ && "
|
|
"cp shared/web_ui_auth.py /asset-output/ && "
|
|
"rm -rf /asset-output/__pycache__",
|
|
],
|
|
),
|
|
),
|
|
timeout=Duration.seconds(15),
|
|
memory_size=128,
|
|
log_group=web_ui_log_group,
|
|
environment={
|
|
"WORK_ORDERS_TABLE": work_orders_table.table_name,
|
|
"COMMENTS_TABLE": comments_table.table_name,
|
|
# Defense-in-depth shared secret for the web UI handler. The
|
|
# handler fails closed if this ARN is unset or the secret is
|
|
# missing, so any future invocation path cannot re-expose the
|
|
# WO DB unauthenticated. The secret value is fetched at runtime
|
|
# from Secrets Manager (not embedded in env vars or template).
|
|
"WEB_UI_AUTH_TOKEN_SECRET_ARN": web_ui_auth_secret.secret_arn,
|
|
},
|
|
)
|
|
|
|
work_orders_table.grant_read_data(web_ui)
|
|
comments_table.grant_read_data(web_ui)
|
|
web_ui_auth_secret.grant_read(web_ui)
|
|
|
|
# --- DynamoDB throttle + system-error alarms ---
|
|
# ThrottledRequests / SystemErrors emit at TableName + Operation only
|
|
# (verified against live CloudWatch: no TableName-only rollup exists, and
|
|
# metric_throttled_requests is deprecated/invalid in aws-cdk-lib 2.261.0).
|
|
# Each table currently has zero throttle/error datapoints, so the series
|
|
# only materialise on first occurrence — NOT_BREACHING keeps them OK until
|
|
# then.
|
|
common.add_ddb_alarms(
|
|
self, "WorkOrdersTable", work_orders_table, "WorkOrders", alarm_topic
|
|
)
|
|
common.add_ddb_alarms(
|
|
self, "WorkOrderComments", comments_table, "WorkOrderComments", alarm_topic
|
|
)
|
|
|
|
# --- Function ARN + consumed-table-name outputs (Phase 4, additive) ---
|
|
cdk.CfnOutput(
|
|
self,
|
|
"EmailProcessorFunctionArn",
|
|
value=email_processor.function_arn,
|
|
description="ARN of the workorder-email-processor Lambda",
|
|
)
|
|
cdk.CfnOutput(
|
|
self,
|
|
"WebUiFunctionArn",
|
|
value=web_ui.function_arn,
|
|
description="ARN of the workorder-web-ui Lambda",
|
|
)
|
|
cdk.CfnOutput(
|
|
self,
|
|
"WorkOrdersTableName",
|
|
value=work_orders_table.table_name,
|
|
description="WorkOrders DynamoDB table",
|
|
)
|
|
cdk.CfnOutput(
|
|
self,
|
|
"WorkOrderCommentsTableName",
|
|
value=comments_table.table_name,
|
|
description="WorkOrderComments DynamoDB table",
|
|
)
|
|
|
|
# Public Function URL removed 2026-06-08 (INFRA-74 / audit C-5): the
|
|
# unauthenticated FunctionUrlAuthType.NONE URL was deleted out-of-band
|
|
# via CLI. Removing the construct (and its auto-generated Principal:*
|
|
# invoke permission) reconciles IaC with the live state.
|