From a6d7e3468deb63848391711718b989c959ca2485 Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Wed, 15 Jul 2026 20:10:40 -0400 Subject: [PATCH] Grant Bedrock IAM, drop Anthropic secrets, add fallback alarm Both stacks moved their processors from the Anthropic API to the Bedrock inference profile us.anthropic.claude-haiku-4-5-20251001-v1:0. Grant each processor role bedrock:InvokeModel + bedrock:InvokeModelWithResponseStream on BOTH the inference-profile ARN AND the per-region foundation-model ARNs for us-east-1/us-east-2/us-west-2 (empty-account) -- the us.* profile routes cross-region, so a profile-only grant AccessDenies at runtime. Remove both anthropic-api-key Secret constructs, their grant_read, and the ANTHROPIC_API_KEY_SECRET_ARN env; add BEDROCK_MODEL_ID. The secrets had RemovalPolicy.RETAIN so they are orphaned, not deleted -- flagged in the README for manual post-deploy deletion and key revocation. Add the workorder-email-processor-template-fallback-rate alarm: a FILL(0) + >=10-sample volume-floor MathExpression over the EMF ParseOutcome metric (15-min periods) that pages when the AI-fallback share exceeds 15% sustained, catching Hexagon template drift. ALARM-only SnsAction to site-alerts, no OK action, NOT_BREACHING, matching the existing stack idiom. --- cdk/po_stack.py | 42 +++++++++++++++++------ cdk/wo_stack.py | 89 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 109 insertions(+), 22 deletions(-) diff --git a/cdk/po_stack.py b/cdk/po_stack.py index 4648842..2e9061a 100644 --- a/cdk/po_stack.py +++ b/cdk/po_stack.py @@ -8,6 +8,7 @@ from aws_cdk import ( aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, + aws_iam as iam, aws_kms as kms, aws_lambda as lambda_, aws_lambda_event_sources as lambda_event_sources, @@ -16,7 +17,6 @@ from aws_cdk import ( aws_s3_notifications as s3n, aws_ses as ses, aws_ses_actions as ses_actions, - aws_secretsmanager as secretsmanager, aws_sns as sns, aws_sqs as sqs, aws_ssm as ssm, @@ -136,14 +136,13 @@ class PoIngestStack(Stack): encryption_key=dynamodb_cmk, ) - # --- Secrets Manager for Anthropic API key --- - anthropic_secret = secretsmanager.Secret( - self, - "AnthropicApiKey", - secret_name="po-ingest/anthropic-api-key", - description="Anthropic API key for Coupa PO email parsing", - removal_policy=RemovalPolicy.RETAIN, - ) + # --- Anthropic API key secret removed (Bedrock migration) --- + # PO parsing stays fully AI but moved from the Anthropic API to the + # Bedrock inference profile us.anthropic.claude-haiku-4-5-20251001-v1:0, + # so no provider API key is needed. The old secret + # "po-ingest/anthropic-api-key" had RemovalPolicy.RETAIN, so it is + # ORPHANED (not deleted) by this change: delete it manually post-deploy + # and revoke the stored key at Anthropic. # --- DLQ for failed async invocations (INFRA-41 / audit H-8) --- # SES → S3 → Lambda is async; without an OnFailure destination a failed @@ -184,14 +183,35 @@ class PoIngestStack(Stack): dead_letter_queue=email_processor_dlq, environment={ "PO_TABLE": "purchase-orders", - "ANTHROPIC_API_KEY_SECRET_ARN": anthropic_secret.secret_arn, + "BEDROCK_MODEL_ID": "us.anthropic.claude-haiku-4-5-20251001-v1:0", }, ) # Grant permissions email_bucket.grant_read(email_processor) po_table.grant_read_write_data(email_processor) - anthropic_secret.grant_read(email_processor) + + # --- Bedrock InvokeModel grant --- + # The us.* inference profile can route cross-region, so the grant MUST + # cover both the inference-profile ARN AND the per-region foundation-model + # ARNs (empty account field) for every region the profile can reach + # (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at + # runtime whenever the profile routes to a region whose foundation-model + # ARN is not allowed. + email_processor.add_to_role_policy( + iam.PolicyStatement( + actions=[ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + resources=[ + "arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + ], + ) + ) # --- Errors alarm (INFRA-41 / audit H-8) --- # ALARM-only (no OK action, per the CloudWatch-alarm preference) to the diff --git a/cdk/wo_stack.py b/cdk/wo_stack.py index fc8d166..3081193 100644 --- a/cdk/wo_stack.py +++ b/cdk/wo_stack.py @@ -8,6 +8,7 @@ from aws_cdk import ( aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, + aws_iam as iam, aws_kms as kms, aws_lambda as lambda_, aws_logs as logs, @@ -15,7 +16,6 @@ from aws_cdk import ( aws_s3_notifications as s3n, aws_ses as ses, aws_ses_actions as ses_actions, - aws_secretsmanager as secretsmanager, aws_sns as sns, aws_sqs as sqs, aws_ssm as ssm, @@ -135,14 +135,12 @@ class WorkorderIngestStack(Stack): removal_policy=RemovalPolicy.RETAIN, ) - # --- Secrets Manager for Anthropic API key --- - anthropic_secret = secretsmanager.Secret( - self, - "AnthropicApiKey", - secret_name="workorder-ingest/anthropic-api-key", - description="Anthropic API key for work order email parsing", - removal_policy=RemovalPolicy.RETAIN, - ) + # --- Anthropic API key secret removed (Bedrock migration) --- + # Parsing moved from the Anthropic API to the Bedrock inference profile + # us.anthropic.claude-haiku-4-5-20251001-v1:0, so no provider API key is + # needed. The old secret "workorder-ingest/anthropic-api-key" had + # RemovalPolicy.RETAIN, so it is ORPHANED (not deleted) by this change: + # delete it manually post-deploy and revoke the stored key at Anthropic. # --- DLQ for failed async invocations (INFRA-41 / audit H-8) --- # SES → S3 → Lambda is async; without an OnFailure destination a failed @@ -184,7 +182,7 @@ class WorkorderIngestStack(Stack): environment={ "WORK_ORDERS_TABLE": work_orders_table.table_name, "COMMENTS_TABLE": comments_table.table_name, - "ANTHROPIC_API_KEY_SECRET_ARN": anthropic_secret.secret_arn, + "BEDROCK_MODEL_ID": "us.anthropic.claude-haiku-4-5-20251001-v1:0", }, ) @@ -192,7 +190,28 @@ class WorkorderIngestStack(Stack): email_bucket.grant_read(email_processor) work_orders_table.grant_read_write_data(email_processor) comments_table.grant_read_write_data(email_processor) - anthropic_secret.grant_read(email_processor) + + # --- Bedrock InvokeModel grant --- + # The us.* inference profile can route cross-region, so the grant MUST + # cover both the inference-profile ARN AND the per-region foundation-model + # ARNs (empty account field) for every region the profile can reach + # (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at + # runtime whenever the profile routes to a region whose foundation-model + # ARN is not allowed. + email_processor.add_to_role_policy( + iam.PolicyStatement( + actions=[ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + resources=[ + "arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + ], + ) + ) # Pre-emptive KMS grant on the shared DynamoDB CMK (alias/seahaven-dynamodb, # /seahaven/dynamodb/cmk-arn). The WorkOrders/WorkOrderComments tables are @@ -285,6 +304,54 @@ class WorkorderIngestStack(Stack): treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + # --- Template fallback-rate alarm: workorder-email-processor --- + # The processor tries a deterministic template parse first and only calls + # the Bedrock AI extractor on a miss/invalid. A sustained rise in the + # ai_fallback share signals Hexagon template drift (coverage collapse). + # EMF metric Seahaven/WorkorderIngest/ParseOutcome, dimensioned by + # ParseMethod (template|ai_fallback). 15-min periods (deliberate deviation + # from the 5-min house style) accumulate a stable denominator at the low + # ~760/day volume; FILL(0) + a >=10-sample volume floor prevent + # low-volume false pages and INSUFFICIENT_DATA. ALARM-only SnsAction to + # site-alerts, no OK action, NOT_BREACHING -- matching the stack idiom. + fb_metric = cloudwatch.Metric( + namespace="Seahaven/WorkorderIngest", + metric_name="ParseOutcome", + dimensions_map={"ParseMethod": "ai_fallback"}, + statistic="Sum", + period=Duration.minutes(15), + ) + tmpl_metric = cloudwatch.Metric( + namespace="Seahaven/WorkorderIngest", + metric_name="ParseOutcome", + dimensions_map={"ParseMethod": "template"}, + statistic="Sum", + period=Duration.minutes(15), + ) + fallback_rate = cloudwatch.MathExpression( + expression=( + "IF((FILL(fb,0)+FILL(tmpl,0))>=10, " + "100*FILL(fb,0)/MAX([FILL(fb,0)+FILL(tmpl,0),1]), 0)" + ), + using_metrics={"fb": fb_metric, "tmpl": tmpl_metric}, + period=Duration.minutes(15), + label="TemplateFallbackRatePct", + ) + fallback_rate.create_alarm( + self, + "EmailProcessorTemplateFallbackRateAlarm", + alarm_name="workorder-email-processor-template-fallback-rate", + alarm_description=( + "workorder-email-processor deterministic-template coverage " + "collapse: >15% of parses fell back to the Bedrock AI extractor" + ), + threshold=15, + evaluation_periods=3, + datapoints_to_alarm=2, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + # S3 event notification -> Lambda email_bucket.add_event_notification( s3.EventType.OBJECT_CREATED,