adam/procurement-ingest
1
0
Fork 0
mirror of https://github.com/Sea-Haven-Industries/procurement-ingest.git synced 2026-10-07 16:08:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Grant Bedrock IAM, drop Anthropic secrets, add fallback alarm

Browse source 
Both stacks moved their processors from the Anthropic API to the Bedrock
inference profile us.anthropic.claude-haiku-4-5-20251001-v1:0. Grant each
processor role bedrock:InvokeModel + bedrock:InvokeModelWithResponseStream
on BOTH the inference-profile ARN AND the per-region foundation-model
ARNs for us-east-1/us-east-2/us-west-2 (empty-account) -- the us.* profile
routes cross-region, so a profile-only grant AccessDenies at runtime.

Remove both anthropic-api-key Secret constructs, their grant_read, and
the ANTHROPIC_API_KEY_SECRET_ARN env; add BEDROCK_MODEL_ID. The secrets
had RemovalPolicy.RETAIN so they are orphaned, not deleted -- flagged in
the README for manual post-deploy deletion and key revocation.

Add the workorder-email-processor-template-fallback-rate alarm: a
FILL(0) + >=10-sample volume-floor MathExpression over the EMF
ParseOutcome metric (15-min periods) that pages when the AI-fallback
share exceeds 15% sustained, catching Hexagon template drift. ALARM-only
SnsAction to site-alerts, no OK action, NOT_BREACHING, matching the
existing stack idiom.
This commit is contained in:
Adam Moussa 2026-07-15 20:10:40 -04:00
parent 3151de92af
commit a6d7e3468d
No known key found for this signature in database
2 changed files with 109 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
cdk/po_stack.py
View file

@ -8,6 +8,7 @@ from aws_cdk import (
aws_cloudwatch as cloudwatch,
aws_cloudwatch_actions as cw_actions,
aws_dynamodb as dynamodb,
aws_iam as iam,
aws_kms as kms,
aws_lambda as lambda_,
aws_lambda_event_sources as lambda_event_sources,
@ -16,7 +17,6 @@ from aws_cdk import (
aws_s3_notifications as s3n,
aws_ses as ses,
aws_ses_actions as ses_actions,
aws_secretsmanager as secretsmanager,
aws_sns as sns,
aws_sqs as sqs,
aws_ssm as ssm,
@ -136,14 +136,13 @@ class PoIngestStack(Stack):
encryption_key=dynamodb_cmk,
)
# --- Secrets Manager for Anthropic API key ---
anthropic_secret = secretsmanager.Secret(
self,
"AnthropicApiKey",
secret_name="po-ingest/anthropic-api-key",
description="Anthropic API key for Coupa PO email parsing",
removal_policy=RemovalPolicy.RETAIN,
)
# --- Anthropic API key secret removed (Bedrock migration) ---
# PO parsing stays fully AI but moved from the Anthropic API to the
# Bedrock inference profile us.anthropic.claude-haiku-4-5-20251001-v1:0,
# so no provider API key is needed. The old secret
# "po-ingest/anthropic-api-key" had RemovalPolicy.RETAIN, so it is
# ORPHANED (not deleted) by this change: delete it manually post-deploy
# and revoke the stored key at Anthropic.
# --- DLQ for failed async invocations (INFRA-41 / audit H-8) ---
# SES → S3 → Lambda is async; without an OnFailure destination a failed
@ -184,14 +183,35 @@ class PoIngestStack(Stack):
dead_letter_queue=email_processor_dlq,
environment={
"PO_TABLE": "purchase-orders",
"ANTHROPIC_API_KEY_SECRET_ARN": anthropic_secret.secret_arn,
"BEDROCK_MODEL_ID": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
},
)
# Grant permissions
email_bucket.grant_read(email_processor)
po_table.grant_read_write_data(email_processor)
anthropic_secret.grant_read(email_processor)
# --- Bedrock InvokeModel grant ---
# The us.* inference profile can route cross-region, so the grant MUST
# cover both the inference-profile ARN AND the per-region foundation-model
# ARNs (empty account field) for every region the profile can reach
# (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at
# runtime whenever the profile routes to a region whose foundation-model
# ARN is not allowed.
email_processor.add_to_role_policy(
iam.PolicyStatement(
actions=[
"bedrock:InvokeModel",
"bedrock:InvokeModelWithResponseStream",
],
resources=[
"arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
],
)
)
# --- Errors alarm (INFRA-41 / audit H-8) ---
# ALARM-only (no OK action, per the CloudWatch-alarm preference) to the

89
cdk/wo_stack.py
View file

@ -8,6 +8,7 @@ from aws_cdk import (
aws_cloudwatch as cloudwatch,
aws_cloudwatch_actions as cw_actions,
aws_dynamodb as dynamodb,
aws_iam as iam,
aws_kms as kms,
aws_lambda as lambda_,
aws_logs as logs,
@ -15,7 +16,6 @@ from aws_cdk import (
aws_s3_notifications as s3n,
aws_ses as ses,
aws_ses_actions as ses_actions,
aws_secretsmanager as secretsmanager,
aws_sns as sns,
aws_sqs as sqs,
aws_ssm as ssm,
@ -135,14 +135,12 @@ class WorkorderIngestStack(Stack):
removal_policy=RemovalPolicy.RETAIN,
)
# --- Secrets Manager for Anthropic API key ---
anthropic_secret = secretsmanager.Secret(
self,
"AnthropicApiKey",
secret_name="workorder-ingest/anthropic-api-key",
description="Anthropic API key for work order email parsing",
removal_policy=RemovalPolicy.RETAIN,
)
# --- Anthropic API key secret removed (Bedrock migration) ---
# Parsing moved from the Anthropic API to the Bedrock inference profile
# us.anthropic.claude-haiku-4-5-20251001-v1:0, so no provider API key is
# needed. The old secret "workorder-ingest/anthropic-api-key" had
# RemovalPolicy.RETAIN, so it is ORPHANED (not deleted) by this change:
# delete it manually post-deploy and revoke the stored key at Anthropic.
# --- DLQ for failed async invocations (INFRA-41 / audit H-8) ---
# SES → S3 → Lambda is async; without an OnFailure destination a failed
@ -184,7 +182,7 @@ class WorkorderIngestStack(Stack):
environment={
"WORK_ORDERS_TABLE": work_orders_table.table_name,
"COMMENTS_TABLE": comments_table.table_name,
"ANTHROPIC_API_KEY_SECRET_ARN": anthropic_secret.secret_arn,
"BEDROCK_MODEL_ID": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
},
)
@ -192,7 +190,28 @@ class WorkorderIngestStack(Stack):
email_bucket.grant_read(email_processor)
work_orders_table.grant_read_write_data(email_processor)
comments_table.grant_read_write_data(email_processor)
anthropic_secret.grant_read(email_processor)
# --- Bedrock InvokeModel grant ---
# The us.* inference profile can route cross-region, so the grant MUST
# cover both the inference-profile ARN AND the per-region foundation-model
# ARNs (empty account field) for every region the profile can reach
# (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at
# runtime whenever the profile routes to a region whose foundation-model
# ARN is not allowed.
email_processor.add_to_role_policy(
iam.PolicyStatement(
actions=[
"bedrock:InvokeModel",
"bedrock:InvokeModelWithResponseStream",
],
resources=[
"arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0",
],
)
)
# Pre-emptive KMS grant on the shared DynamoDB CMK (alias/seahaven-dynamodb,
# /seahaven/dynamodb/cmk-arn). The WorkOrders/WorkOrderComments tables are
@ -285,6 +304,54 @@ class WorkorderIngestStack(Stack):
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
# --- Template fallback-rate alarm: workorder-email-processor ---
# The processor tries a deterministic template parse first and only calls
# the Bedrock AI extractor on a miss/invalid. A sustained rise in the
# ai_fallback share signals Hexagon template drift (coverage collapse).
# EMF metric Seahaven/WorkorderIngest/ParseOutcome, dimensioned by
# ParseMethod (template|ai_fallback). 15-min periods (deliberate deviation
# from the 5-min house style) accumulate a stable denominator at the low
# ~760/day volume; FILL(0) + a >=10-sample volume floor prevent
# low-volume false pages and INSUFFICIENT_DATA. ALARM-only SnsAction to
# site-alerts, no OK action, NOT_BREACHING -- matching the stack idiom.
fb_metric = cloudwatch.Metric(
namespace="Seahaven/WorkorderIngest",
metric_name="ParseOutcome",
dimensions_map={"ParseMethod": "ai_fallback"},
statistic="Sum",
period=Duration.minutes(15),
)
tmpl_metric = cloudwatch.Metric(
namespace="Seahaven/WorkorderIngest",
metric_name="ParseOutcome",
dimensions_map={"ParseMethod": "template"},
statistic="Sum",
period=Duration.minutes(15),
)
fallback_rate = cloudwatch.MathExpression(
expression=(
"IF((FILL(fb,0)+FILL(tmpl,0))>=10, "
"100*FILL(fb,0)/MAX([FILL(fb,0)+FILL(tmpl,0),1]), 0)"
),
using_metrics={"fb": fb_metric, "tmpl": tmpl_metric},
period=Duration.minutes(15),
label="TemplateFallbackRatePct",
)
fallback_rate.create_alarm(
self,
"EmailProcessorTemplateFallbackRateAlarm",
alarm_name="workorder-email-processor-template-fallback-rate",
alarm_description=(
"workorder-email-processor deterministic-template coverage "
"collapse: >15% of parses fell back to the Bedrock AI extractor"
),
threshold=15,
evaluation_periods=3,
datapoints_to_alarm=2,
comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING,
).add_alarm_action(cw_actions.SnsAction(alarm_topic))
# S3 event notification -> Lambda
email_bucket.add_event_notification(
s3.EventType.OBJECT_CREATED,