diff --git a/.claude/workflows/phase-4-cdk-common.js b/.claude/workflows/phase-4-cdk-common.js new file mode 100644 index 0000000..1fa35bf --- /dev/null +++ b/.claude/workflows/phase-4-cdk-common.js @@ -0,0 +1,684 @@ +export const meta = { + name: 'phase-4-cdk-common', + description: 'Phase 4 of the procurement-ingest refactor (docs/refactor-evaluation.md): collapse the 379 identical CDK lines into cdk/common.py as PLAIN FUNCTIONS taking (scope, id, ...) — called with the SAME Stack scope and the SAME construct ids the stacks use today, so every logical ID is byte-stable (Construct-subclass wrapping is forbidden: it would reparent the tree and attempt REPLACEMENT of the RETAIN-protected purchase-orders/WorkOrders tables and named buckets = data loss). Extracts add_ddb_alarms, add_sender_auth_rejected_alarm, add_standard_lambda_alarms, make_bedrock_invoke_statement (account/region-derived ARN, not hardcoded 328440206208), make_email_bucket, make_processor_dlq, make_fallback_rate_alarm — preserving every per-function alarm variance byte-for-byte. Same PR: account on both cdk.Environment, constructs== exact pin, stale-comment fix, CfnOutputs for five function ARNs + consumed table names. ZERO cdk diff on both stacks is the acceptance test. The make_bedrock_invoke_statement IAM PolicyStatement move triggers mandatory GPT-4.1 cross-family review even though semantics are identical. Committed locally, never pushed.', + phases: [ + { title: 'Setup', detail: 'verify Phases 0+1+2+3 on base, branch feature/phase-4-cdk-common', model: 'haiku' }, + { title: 'Recon', detail: '4 mappers: the 379 duplicated CDK lines + per-function alarm variance, the two inline Bedrock PolicyStatements, the fallback-rate + rejected alarm math (post-Phase-1), app.py/pins/outputs surface' }, + { title: 'Spec', detail: 'serial fable spec: pin common.py contents + every function signature, per-stack call-site rewrites, alarm-variance table, Bedrock IAM equivalence, fallback-rate call params, app/pins/CfnOutputs, zero-diff judging rules' }, + { title: 'Implement', detail: 'opus: cdk/common.py; opus: both stacks (rewire + CfnOutputs); sonnet: app.py + requirements pin + README — disjoint files', model: 'opus' }, + { title: 'Verify', detail: 'mechanical gates + zero-cdk-diff verifier (the load-bearing gate) + 3 fable lenses (logical-ID safety, alarm variance, IAM equivalence)' }, + { title: 'Fix', detail: 'opus fixer, full re-verify, max 3 rounds', model: 'opus' }, + { title: 'Package', detail: 'single commit via -F (no push); runs cross_review.py inline on the IAM diff', model: 'sonnet' }, + ], +} + +// ---------------------------------------------------------------- constants + +const REPO = '/Users/adammoussa/Documents/repositories/seahaven/procurement-ingest' +const BRANCH = 'feature/phase-4-cdk-common' +let _args = args +if (typeof _args === 'string') { + try { _args = JSON.parse(_args) } catch (e) { _args = null } +} +const BASE = (_args && _args.base) || 'main' + +const CONSTRAINTS = ` +PINNED BEHAVIORAL CONSTRAINTS (docs/refactor-evaluation.md Phase 4 — violating any is a build failure): +1. EXTRACT AS PLAIN FUNCTIONS taking (scope, id, ...), called with the SAME + scope (the Stack instance) and the SAME construct ids the stacks use + today -> 100% logical-ID-safe. NEVER wrap in Construct subclasses: a + subclass inserts a tree node, changes EVERY child logical ID, and would + attempt REPLACEMENT of the RETAIN-protected purchase-orders / WorkOrders + tables and the named buckets = DATA LOSS. This is THE load-bearing rule + of the phase — every other check exists to defend it. +2. New module cdk/common.py. Extract exactly: + - _DDB_ALARM_OPERATIONS + add_ddb_alarms + - add_sender_auth_rejected_alarm + - add_standard_lambda_alarms(scope, id_prefix, fn, name_prefix, topic, *, + duration_statistic, errors=True, dlq=None, descriptions=...) + - make_bedrock_invoke_statement (DERIVE the inference-profile ARN + the + per-region foundation-model ARNs from Stack.of(scope).account / + Stack.of(scope).region — NOT hardcoded 328440206208) + - make_email_bucket + - make_processor_dlq + - make_fallback_rate_alarm(namespace, rejected_included, period, threshold, + floor, evaluation_periods, datapoints_to_alarm) reproducing the + expression strings / FILL / labels BYTE-FOR-BYTE. +3. PER-FUNCTION ALARM VARIANCE — preserve EXACTLY, do NOT homogenize: + PO email-processor duration p99 vs wo-email-processor p95; po-web-ui + throttles+duration only; site_extractor no-DLQ; wo web_ui has ZERO alarms + (do NOT let the shared helper silently add any); every bespoke alarm + DESCRIPTION string is passed through verbatim. +4. FALLBACK-RATE RECONCILIATION (post-Phase-1): PO's template-fallback-rate + EXCLUDES the rejected series (byte-identical to today, a pre-call + double-count would result otherwise) -> call make_fallback_rate_alarm with + rejected_included=False; WO's INCLUDES rejected -> rejected_included=True. + The two REJECTED alarms are a DIFFERENT shape and are NOT + make_fallback_rate_alarm: PO's ai-fallback-rejected is a 6h count-floor + IF(FILL(rej,0)>=1,...) alarm (added in Phase 1); WO's is the 5-min / + 2-of-6 sparse idiom. Keep those as DISTINCT call sites (or a separate + dedicated helper) — do NOT force them through make_fallback_rate_alarm. + NO element-wise MAX anywhere (post-#102 rule). +5. Do NOT import stack-specific services (kms / ssm / event_sources) into + common.py — only the constructs the shared helpers actually need. +6. SAME PR, net-new & logical-ID-safe additions: + - add account='328440206208' to BOTH cdk.Environment calls + - pin constructs== to the exact installed version (not a floor >=) + - fix the stale "2.259.0" version comments + - add CfnOutputs for the FIVE function ARNs + the consumed table names. + These are additive; CfnOutputs and account are ID-safe. Verify none of + them perturbs an existing logical ID. +7. ZERO lambdas/ diff: git diff ${BASE}...HEAD -- lambdas/ must be EMPTY. + This phase is CDK-ONLY. tests/ may gain a cdk-diff / synth-only test for + the acceptance gate, but NO other tests/ change and NO lambdas/ change. +8. ZERO cdk diff on BOTH stacks is the acceptance test: npx cdk diff + po-ingest and npx cdk diff workorder-ingest must show ZERO resource + changes (no logical-ID, alarm, IAM, table, bucket, env, or metadata + delta beyond CDK-tooling noise). The CfnOutputs are the ONLY net-new + resources allowed to appear, and only as additions. +9. The wo artifact id is 'workorder-ingest' (the construct id / 2nd + positional arg), NOT 'WorkorderIngestStack' (that is stack_name). ALWAYS + drive synth/diff by the artifact id: npx cdk synth workorder-ingest, + npx cdk diff workorder-ingest. Using the stack_name selector fails. +10. NO cdk deploy, NO invoke, NO AWS mutation. Read-only AWS only if needed + (e.g. confirming deployed alarm names) — the diff gate is a pure local + synth-vs-synth comparison. +` + +const PREAMBLE = ` +You are one of several agents building refactor Phase 4 in the git repo at +${REPO} on branch ${BRANCH} (already checked out — do NOT switch branches, +do NOT create branches, do NOT commit, NEVER push, do NOT run cdk deploy or +touch AWS resources beyond read-only calls). +Authoritative spec: docs/refactor-evaluation.md, section "Phase 4". +Work ONLY in the files you are told you own; other agents are concurrently +editing other files in this same working tree. +${CONSTRAINTS} +Your final message is consumed by an orchestrator script, not a human — +return only the structured data requested. +` + +// ------------------------------------------------------------------ schemas + +const RECON = { + type: 'object', + required: ['summary', 'facts'], + properties: { + summary: { type: 'string' }, + facts: { type: 'array', items: { type: 'string' } }, + blockers: { type: 'array', items: { type: 'string' } }, + }, +} + +const SPEC = { + type: 'object', + required: ['commonModule', 'poStackEdits', 'woStackEdits', 'alarmVariance', 'bedrockStatement', 'fallbackRateCalls', 'appAndPins', 'diffRules', 'notes'], + properties: { + commonModule: { type: 'string', description: 'the full cdk/common.py: every function (add_ddb_alarms + _DDB_ALARM_OPERATIONS, add_sender_auth_rejected_alarm, add_standard_lambda_alarms, make_bedrock_invoke_statement, make_email_bucket, make_processor_dlq, make_fallback_rate_alarm) with its EXACT signature, and the exact imports it needs (no stack-specific kms/ssm/event_sources per constraint 5)' }, + poStackEdits: { type: 'string', description: 'cdk/po_stack.py: every inline block replaced by a common.* call, file:line, with the exact scope + construct-id + kwargs each call passes so the emitted resource is byte-identical; plus the PO CfnOutput additions (function ARNs + consumed table names)' }, + woStackEdits: { type: 'string', description: 'cdk/wo_stack.py: same — call-site rewrites file:line preserving construct ids, plus WO CfnOutput additions; explicitly note wo web_ui gets NO alarms (constraint 3)' }, + alarmVariance: { type: 'string', description: 'the per-function alarm-variance table proving each helper call reproduces exactly what the inline code emits today: PO p99 vs wo-email-processor p95, po-web-ui throttles+duration only, site_extractor no-DLQ, wo web_ui ZERO alarms, every bespoke description string mapped verbatim' }, + bedrockStatement: { type: 'string', description: 'make_bedrock_invoke_statement: the exact actions + resources, showing how the inference-profile ARN and per-region FM ARNs are derived from Stack.of(scope).account/.region, and PROVING the derived strings resolve in-account to the SAME ARNs the two inline PolicyStatements hardcode today' }, + fallbackRateCalls: { type: 'string', description: 'the make_fallback_rate_alarm call params for PO (rejected_included=False) and WO (rejected_included=True) reproducing the expression/FILL/label strings byte-for-byte; PLUS how the two DISTINCT rejected alarms stay distinct call sites (PO 6h count-floor IF(FILL(rej,0)>=1,...); WO 5-min/2-of-6 sparse) — NOT folded into make_fallback_rate_alarm, NO element-wise MAX' }, + appAndPins: { type: 'string', description: 'app.py account= additions to both cdk.Environment calls; the exact constructs== pin (installed version); the stale "2.259.0" comment fix locations + new text; confirmation none perturbs a logical ID' }, + diffRules: { type: 'string', description: 'exactly how Verify proves zero cdk diff: synth BASE and HEAD into separate temp dirs, diff the two stacks templates, ANY resource/logical-ID/property delta = FAIL, the ONLY allowed additions are the net-new CfnOutputs' }, + notes: { type: 'string' }, + }, +} + +const IMPL = { + type: 'object', + required: ['filesChanged', 'summary', 'checksRun'], + properties: { + filesChanged: { type: 'array', items: { type: 'string' } }, + summary: { type: 'string' }, + checksRun: { type: 'string' }, + blockers: { type: 'array', items: { type: 'string' } }, + }, +} + +const CHECKS = { + type: 'object', + required: ['passed', 'details'], + properties: { + passed: { type: 'boolean' }, + details: { type: 'string' }, + scopeViolations: { type: 'array', items: { type: 'string' } }, + }, +} + +const DIFF = { + type: 'object', + required: ['passed', 'poDiffVerdict', 'woDiffVerdict', 'details'], + properties: { + passed: { type: 'boolean' }, + poDiffVerdict: { type: 'string', description: 'po-ingest BASE-synth vs HEAD-synth: ZERO resource/logical-ID/property changes (CfnOutputs the only allowed net-new additions) — full template-diff evidence' }, + woDiffVerdict: { type: 'string', description: 'workorder-ingest (artifact id, NOT WorkorderIngestStack): same zero-change evidence' }, + details: { type: 'string' }, + }, +} + +const FINDINGS = { + type: 'object', + required: ['findings'], + properties: { + findings: { + type: 'array', + items: { + type: 'object', + required: ['title', 'severity', 'confirmed', 'evidence', 'fix'], + properties: { + title: { type: 'string' }, + severity: { enum: ['critical', 'high', 'medium', 'low'] }, + confirmed: { type: 'boolean' }, + evidence: { type: 'string' }, + fix: { type: 'string' }, + }, + }, + }, + }, +} + +// ------------------------------------------------------------------- setup + +phase('Setup') +const setup = await agent(` +In ${REPO}: +1. SEQUENCING GATE — Phases 0, 1, 2 AND 3 must all be on ${BASE} (Phase 4 + dedups BOTH cdk stacks, which Phases 1 (po_stack alarms), 2 (bundling + roots) and 3 (shared cp) all edited — building against a pre-Phase-3 + stack file guarantees a conflict and a wrong diff baseline). git fetch + origin, then pick the base ref: origin/${BASE} if that remote ref + exists, otherwise the local branch ${BASE} (a stacked local-only base is + expected and fine). Verify on the base ref: + (a) Phase 3: lambdas/shared/ exists + (git ls-tree -- lambdas/shared | head); + (b) Phase 2: BOTH cdk/po_stack.py and cdk/wo_stack.py contain + Code.from_asset("../lambdas") for the email processors + (git show :cdk/po_stack.py | grep -n '\\.\\./lambdas', same + for wo_stack.py); + (c) Phase 1: the po-email-processor-ai-fallback-rejected alarm / + EmailProcessorAiFallbackRejectedAlarm construct exists in po_stack.py + (git show :cdk/po_stack.py | grep -n 'ai-fallback-rejected\\|AiFallbackRejected'). + If Phase 3 is not on ${BASE}, STOP with a blocker (Phase 4 needs the + post-Phase-3 stack files as its zero-diff baseline). If any other phase + is missing, STOP with a blocker naming the unmet phase and do nothing + else. +2. Verify clean working tree (untracked .coverage / .claude/ / the local + lambdas/po/email_processor/package/ dir are fine; any OTHER dirt = + blocker, never stash or discard). +3. git checkout ${BASE}; then git pull --ff-only ONLY if the branch has an + upstream (a local-only base skips the pull — not a blocker); then + git checkout -b ${BRANCH} +4. gh pr list --state open --json number,title,headRefName (overlap check). +Return facts: HEAD sha, per-phase gate evidence, open PRs, blockers. +`, { label: 'setup:branch', model: 'haiku', schema: RECON }) + +if (!setup || (setup.blockers && setup.blockers.length)) { + return { aborted: 'setup blockers', blockers: setup ? setup.blockers : ['setup agent died'], facts: setup ? setup.facts : [] } +} +log(`Branch ${BRANCH} ready off ${BASE}. ${setup.summary}`) + +// ------------------------------------------------------------------- recon + +phase('Recon') +const recon = await parallel([ + () => agent(`${PREAMBLE} +Read-only recon of the ~379 duplicated CDK lines and the PER-FUNCTION ALARM +VARIANCE that MUST survive the dedup (constraint 3). In cdk/po_stack.py and +cdk/wo_stack.py, quote verbatim with file:line: +1. _DDB_ALARM_OPERATIONS + add_ddb_alarms (both copies — are they + byte-identical? diff them). +2. add_sender_auth_rejected_alarm (both copies). +3. Every add_standard_lambda_alarms-shaped block: for EACH function + (po email-processor, wo email-processor, po web_ui, wo web_ui, + po site_extractor) list the exact alarm set, the duration statistic + (prove PO email p99 vs wo email p95), whether throttles/errors/dlq + alarms are present, and QUOTE every bespoke alarm description string. + CRITICALLY: confirm wo web_ui has ZERO alarms today. +4. make_email_bucket / make_processor_dlq shaped blocks (both copies), and + which functions get a DLQ (site_extractor has none). +5. The exact construct ids (2nd positional arg to every alarm / bucket / + dlq / statement construct) — these are the logical-ID roots that must be + passed UNCHANGED into the shared helpers. +30-40 precise facts. Any block that is NOT actually identical between +stacks (genuine drift) is a blocker to report, not to silently reconcile.`, + { label: 'recon:duplicated-cdk', model: 'sonnet', phase: 'Recon', schema: RECON }), + + () => agent(`${PREAMBLE} +Read-only recon of the two inline Bedrock IAM PolicyStatements (the +make_bedrock_invoke_statement source — constraint 2, the cross-family-review +surface). In both stacks quote verbatim with file:line: the full +iam.PolicyStatement (effect, actions, resources, conditions). For EACH +resource ARN record whether the account (328440206208) and region are +hardcoded or referenced, and enumerate every inference-profile ARN and every +per-region foundation-model ARN. Determine the EXACT list of regions / model +ids baked into the resources so the derived form (Stack.of(scope).account / +.region) can be proven to resolve to the identical strings in-account. Note +which principal/role each statement is attached to and how (add_to_role_policy +vs inline policy). 15-25 facts.`, + { label: 'recon:bedrock-iam', model: 'sonnet', phase: 'Recon', schema: RECON }), + + () => agent(`${PREAMBLE} +Read-only recon of the fallback-rate + rejected alarm math AS IT STANDS +POST-PHASE-1 (constraint 4). In both stacks quote verbatim with file:line: +1. PO's template-fallback-rate alarm: the full metric-math expression + string(s), every FILL(), every label, the period/threshold/ + evaluation_periods/datapoints_to_alarm — and CONFIRM it EXCLUDES the + rejected series today (rejected_included=False). +2. WO's template-fallback-rate alarm: same, and CONFIRM it INCLUDES the + rejected series (rejected_included=True). +3. PO's ai-fallback-rejected alarm (added in Phase 1): confirm it is the + 6h count-floor IF(FILL(rej,0)>=1,...) shape — quote the expression. +4. WO's rejected alarm: confirm it is the 5-min / 2-of-6 sparse idiom — + quote it. These two are DIFFERENT shapes and must stay distinct call + sites, NOT folded into make_fallback_rate_alarm. +5. Confirm NO element-wise MAX exists anywhere in either expression + (post-#102 rule). +Return the exact parameter values each make_fallback_rate_alarm call must +carry so Verify can prove byte-identity. 15-25 facts.`, + { label: 'recon:fallback-alarms', model: 'sonnet', phase: 'Recon', schema: RECON }), + + () => agent(`${PREAMBLE} +Read-only recon of the app.py / pins / outputs surface (constraint 6): +1. cdk/app.py: quote both cdk.Environment(...) calls verbatim with line + numbers (region-only today; account must be added). +2. The constructs dependency pin: quote cdk/requirements.txt line(s) and + find the stale "2.259.0" version comment(s) wherever they live (app.py, + stacks, requirements — grep the whole cdk/ tree) with file:line and the + actual installed constructs / aws-cdk-lib versions. +3. The five Lambda function construct variables in the stacks whose ARNs + need CfnOutputs (po email-processor, po web_ui, po site_extractor, + wo email-processor, wo web_ui) and the table constructs whose names are + consumed (purchase-orders, WorkOrders, and any comments table) — quote + the construct handles + ids so the CfnOutputs reference them correctly. +4. Any existing CfnOutput in either stack (WO reportedly has zero). +5. Confirm the wo artifact id is 'workorder-ingest' (the 2nd positional + arg to the Stack constructor in app.py), distinct from + stack_name='WorkorderIngestStack' (constraint 9). +15-25 facts.`, + { label: 'recon:app-pins-outputs', model: 'haiku', phase: 'Recon', schema: RECON }), +]) + +const reconOk = recon.filter(Boolean) +const pack = reconOk.map(r => `## ${r.summary}\n${r.facts.join('\n')}`).join('\n\n') +const reconBlockers = reconOk.flatMap(r => r.blockers || []) + .filter(b => b && !/^\s*(none|n\/a)\b/i.test(b)) +log(`Recon complete: ${reconOk.length}/4 mappers, ${reconBlockers.length} advisory notes`) +// Recon "blockers" for this phase are advisory design-notes / intended +// constraint-2 & 6 work items (derive the Bedrock ARN from Stack.of(scope), +// add account= to the environments, exact-pin constructs, fix the stale +// aws-cdk-lib version comment, decide the one common sender-auth docstring), +// NOT stop conditions — main-loop verified. The real gate is the ZERO cdk diff +// acceptance test in Verify. Fold the notes into the spec context instead of +// aborting so the spec agent must address each. +const reconAdvisories = reconBlockers.length + ? `\n\nRECON ADVISORIES (recon flagged these; they are the intended constraint-2/6 work + a docstring choice, NOT drift that blocks dedup — resolve each per the constraints; the zero-cdk-diff test is the real acceptance gate):\n- ${reconBlockers.join('\n- ')}` + : '' + +// -------------------------------------------------------------------- spec + +phase('Spec') +const spec = await agent(`${PREAMBLE} +You are the SPEC agent — the single authority that pins every contested +decision BEFORE parallel implementation (parallel leaves cannot see each +other's choices). Using the recon pack below plus your own reads of the +actual files, produce the binding implementation spec: +- commonModule: the full cdk/common.py — every function per constraint 2 + with its EXACT signature (add_standard_lambda_alarms keyword-only params + exactly as the doc pins them), and ONLY the imports the helpers need + (constraint 5 — no kms/ssm/event_sources). Every function is a PLAIN + function taking (scope, id, ...) — NO Construct subclass anywhere + (constraint 1). +- poStackEdits / woStackEdits: for each stack, the exact call-site rewrites + (file:line) mapping each inline block to a common.* call, passing the + SAME scope (the Stack) and the SAME construct id it uses today so every + logical ID is byte-stable; plus the CfnOutput additions (five function + ARNs split across the two stacks + consumed table names). State + explicitly that wo web_ui receives NO alarm call (constraint 3). +- alarmVariance: the per-function variance table (constraint 3) proving each + helper call reproduces today's emitted alarms EXACTLY — PO p99 vs wo-email + p95, po-web-ui throttles+duration only, site_extractor no-DLQ, wo web_ui + zero alarms, every bespoke description string mapped verbatim. +- bedrockStatement: make_bedrock_invoke_statement's actions + resources and + the derivation of the inference-profile / per-region FM ARNs from + Stack.of(scope).account/.region, with a proof table showing each derived + string equals the inline hardcoded ARN in-account (this is the + cross-family-review surface — be exhaustive). +- fallbackRateCalls: the make_fallback_rate_alarm call params for PO + (rejected_included=False) and WO (rejected_included=True) reproducing the + expression/FILL/label strings byte-for-byte, PLUS the plan for keeping the + two DISTINCT rejected alarms as separate call sites (PO 6h count-floor, + WO 5-min/2-of-6) — NOT folded, NO element-wise MAX (constraint 4). +- appAndPins: app.py account= on both Environment calls; the exact + constructs== pin; the stale "2.259.0" comment fix (file:line + new text); + confirmation each is logical-ID-neutral. +- diffRules: exactly how Verify proves ZERO cdk diff — synth ${BASE} and + HEAD each into a separate temp dir, diff both stacks' templates, ANY + resource/logical-ID/property delta = FAIL, the ONLY allowed net-new is + the CfnOutputs; drive synth/diff by artifact id (po-ingest / + workorder-ingest, constraint 9). +Recon pack:\n${pack}${reconAdvisories}`, + { label: 'spec:pin-common', phase: 'Spec', schema: SPEC }) + +if (!spec) return { aborted: 'spec agent died — rerun workflow', reconBlockers } +const specBlock = `BINDING SPEC (from the spec agent — implement EXACTLY this):\n${JSON.stringify(spec, null, 2)}` +log('Spec pinned: common.py contents, per-stack rewrites, alarm variance, Bedrock IAM, fallback-rate calls, app/pins/outputs') + +// --------------------------------------------------------------- implement + +phase('Implement') +const impl = await parallel([ + () => agent(`${PREAMBLE} +YOU OWN: cdk/common.py ONLY (create it). Do not touch po_stack.py, +wo_stack.py, app.py, requirements.txt, README, tests, or anything under +lambdas/. +Task: author cdk/common.py per spec.commonModule EXACTLY — every function +(add_ddb_alarms + _DDB_ALARM_OPERATIONS, add_sender_auth_rejected_alarm, +add_standard_lambda_alarms with the pinned keyword-only signature, +make_bedrock_invoke_statement deriving ARNs from Stack.of(scope).account/ +.region, make_email_bucket, make_processor_dlq, make_fallback_rate_alarm) +as a PLAIN function taking (scope, id, ...) — NEVER a Construct subclass +(constraint 1). Import ONLY what the helpers need — no kms/ssm/ +event_sources (constraint 5). Match the fallback-rate expression/FILL/label +strings byte-for-byte (constraint 4). Do NOT put the two rejected alarms in +make_fallback_rate_alarm. +Run before returning: ruff check cdk/common.py && ruff format cdk/common.py +--check, plus a py_compile import smoke (python3 -c "import common" from +cdk/ with the CDK venv). Full synth is the stacks agent's job — but if you +can import common cleanly, report it. +${specBlock}`, + { label: 'impl:common-module', model: 'opus', phase: 'Implement', schema: IMPL }), + + () => agent(`${PREAMBLE} +YOU OWN: cdk/po_stack.py and cdk/wo_stack.py ONLY. Do not touch +cdk/common.py (another agent authors it), app.py, requirements.txt, README, +or lambdas/. +Task: apply spec.poStackEdits + spec.woStackEdits — replace each inline +block with the matching common.* call, passing the SAME scope (the Stack +instance, NOT a new Construct) and the SAME construct id used today so every +logical ID is byte-stable (constraint 1). Preserve every per-function alarm +variance (constraint 3): PO email p99 / wo email p95, po-web-ui throttles+ +duration only, site_extractor no-DLQ, wo web_ui gets NO alarm call, bespoke +descriptions passed verbatim. Wire the fallback-rate calls per +spec.fallbackRateCalls (PO rejected_included=False, WO True) and keep the +two rejected alarms as distinct call sites (constraint 4). Add the CfnOutputs +per spec (function ARNs + consumed table names) — additive, ID-safe. +Import from common (bare 'import common' / 'from common import ...' — cdk/ +is on sys.path via app.py's imports). Touch nothing else (tables, KMS, SSM, +event sources, the RETAIN policies stay byte-identical). +Run before returning: ruff check cdk && cd cdk && +npx cdk synth po-ingest -q -o /tmp/phase4-synth && +npx cdk synth workorder-ingest -q -o /tmp/phase4-synth (artifact-id +selectors, NOT stack_name — constraint 9). If cdk/common.py has not landed +yet the synth fails on the missing import — poll by re-running up to ~10 min +before reporting a blocker. Confirm both stacks synth and note that the +ONLY template delta vs ${BASE} is the net-new CfnOutputs (spot-check a +couple of alarm logical IDs are unchanged). +${specBlock}`, + { label: 'impl:cdk-stacks', model: 'opus', phase: 'Implement', schema: IMPL }), + + () => agent(`${PREAMBLE} +YOU OWN: cdk/app.py, cdk/requirements.txt and README.md ONLY. Do not touch +common.py, the stacks, tests, or lambdas/. +Task A: apply spec.appAndPins — add account='328440206208' to BOTH +cdk.Environment calls in app.py, pin constructs== to the exact installed +version in cdk/requirements.txt, and fix the stale "2.259.0" comment(s) at +the file:line spec.appAndPins gives (only those in files you own — if a +stale comment lives in a stack file, note it for the stacks agent, do NOT +edit their file). Every edit here must be logical-ID-neutral (account on +Environment does not change resource logical IDs; verify in your summary). +Task B: README — document cdk/common.py in the CDK/architecture section +(the shared plain-function helpers, the logical-ID-safety rule, the +account/region-derived Bedrock ARN, the new CfnOutputs), and note the +account is now explicit on both stacks. Match existing README style. If the +README documents the stacks' resource inventory, keep it accurate. +Run before returning: ruff check cdk (app.py) and a py_compile of app.py; +you cannot run the full synth without the stacks agent's edits — if you want +to smoke-test, poll cd cdk && npx cdk synth po-ingest -q up to ~10 min, but +a clean app.py parse is sufficient for your scope. +${specBlock}`, + { label: 'impl:app-pins-readme', model: 'sonnet', phase: 'Implement', schema: IMPL }), +]) + +const implOk = impl.filter(Boolean) +const implBlockers = implOk.flatMap(r => r.blockers || []) +log(`Implement complete: ${implOk.length}/3 agents, blockers: ${implBlockers.length}`) + +// ---------------------------------------------------- verify + fix loop + +const EXPECTED_SCOPE = [ + 'cdk/common.py', + 'cdk/po_stack.py', + 'cdk/wo_stack.py', + 'cdk/app.py', + 'cdk/requirements.txt', + 'README.md', + 'tests/', +] + +const mechanicalPrompt = `${PREAMBLE} +Independent re-verification — trust nothing self-reported. Run ALL gates, +quoting failures verbatim: +1. pytest -q --no-cov (repo root — all suites green; a synth-only cdk-diff + test may now exist under tests/) +2. ruff check . && ruff format --check . +3. cd cdk && npx cdk synth po-ingest -q && npx cdk synth workorder-ingest -q + (artifact-id selectors, NOT stack_name — constraint 9) +4. ZERO-CODE invariant (constraint 7): git diff ${BASE}...HEAD -- lambdas/ + must output NOTHING. +5. NO CONSTRUCT SUBCLASS (constraint 1): grep cdk/common.py for + 'class .*Construct' / 'class .*(Construct)' — there must be NONE; every + extracted symbol is a plain 'def'. Report any subclass as a hard FAIL. +6. cdk/common.py imports NO kms/ssm/event_sources (constraint 5) — grep and + confirm. +7. Both cdk.Environment calls in app.py carry account='328440206208'; + constructs== is an exact pin (not >=); no "2.259.0" stale comment + remains (grep the cdk/ tree). +8. CfnOutputs: five function ARNs + the consumed table names are present + across the two stacks (grep CfnOutput). +9. git status --porcelain scope check: every modified/added path under + ${EXPECTED_SCOPE.join(', ')} (untracked .coverage/.claude/package/ + tolerated). No lambdas/ or non-cdk-diff tests/ change. +passed=true only if all green. YOU MAY NOT edit files.` + +const diffPrompt = `${PREAMBLE} +You are the ZERO-CDK-DIFF verifier — the load-bearing gate of this phase +(the doc names it the acceptance test). Everything is local synth-vs-synth. +1. From a clean worktree state, synth the BASE templates: check out (via + git worktree add or git stash-free 'git show'-based synth — prefer + 'git worktree add /tmp/phase4-base ${BASE}' so HEAD is untouched), then + in that BASE tree cd cdk && npx cdk synth po-ingest -q -o + /tmp/phase4-diff-base && npx cdk synth workorder-ingest -q -o + /tmp/phase4-diff-base. +2. Synth the HEAD templates: in the working tree cd cdk && npx cdk synth + po-ingest -q -o /tmp/phase4-diff-head && npx cdk synth workorder-ingest + -q -o /tmp/phase4-diff-head. (Both by ARTIFACT ID, constraint 9.) +3. Diff the two CloudFormation templates per stack + (/tmp/phase4-diff-base/.template.json vs + /tmp/phase4-diff-head/.template.json). Normalize only CDK-tooling + noise (the CDKMetadata Analytics string, asset-hash-derived S3Key values + that were ALSO equal on BASE). Then judge: + - ZERO logical-ID changes (no renamed/removed/added Resources except the + net-new CfnOutputs). + - ZERO alarm property deltas (thresholds, statistics p99/p95, expression + strings, FILL, labels, evaluation_periods, datapoints_to_alarm). + - ZERO IAM deltas: the Bedrock PolicyStatement actions/resources must be + byte-identical after the account/region derivation resolves in-account. + - ZERO DynamoDB table / bucket / DLQ / env / runtime deltas. + - The ONLY allowed net-new is the Outputs block (the five function ARNs + + consumed table names). + ANY delta outside that allowance = FAIL. Paste the actual per-stack + template diff (or 'identical' with the normalized-noise list). +4. Cross-check with the live tool: cd cdk && npx cdk diff po-ingest ; + npx cdk diff workorder-ingest against the deployed state must also show + only the CfnOutput additions (network permitting; if AWS creds are + read-only-absent, the BASE-vs-HEAD template diff in steps 1-3 is + authoritative). +5. Clean up any /tmp worktrees you created (git worktree remove). +passed=true only if BOTH stacks show zero resource change beyond the +CfnOutput additions.` + +const lenses = [ + { key: 'logical-id-safety', prompt: `${PREAMBLE} +ADVERSARIAL REVIEW — logical-ID-safety lens (the load-bearing rule, +constraint 1). Try to prove a construct-id or tree-shape change slipped in. +(1) Read cdk/common.py: is EVERY extracted symbol a plain 'def' taking +(scope, id, ...)? Any 'class X(Construct)' / Construct subclass / nested +Construct is an automatic CRITICAL — a subclass reparents the tree and +would attempt REPLACEMENT of the RETAIN-protected purchase-orders / +WorkOrders tables and the named buckets. (2) For every rewritten call site +in both stacks, prove the scope passed is the Stack instance (self), NOT a +new intermediate construct, and the construct id string is IDENTICAL to the +BASE inline id (git diff ${BASE} the id strings). (3) Synth BASE and HEAD +and diff the full Resources logical-ID SET — it must be identical (only +Outputs added). Any renamed/removed logical ID = confirmed CRITICAL. +(4) Confirm the RETAIN removal policies on the tables and the bucket +names/policies are byte-identical post-refactor. confirmed=true only with a +concrete logical-ID delta or a subclass-smell file:line.` }, + { key: 'alarm-variance', prompt: `${PREAMBLE} +ADVERSARIAL REVIEW — alarm-variance lens (constraint 3). The shared helpers +must NOT homogenize the deliberate per-function differences. Read +cdk/common.py + every helper call site + the synthesized templates' alarms. +Prove each of these survived EXACTLY: (1) PO email-processor duration alarm +uses p99, wo-email-processor uses p95 — quote both from the synthesized +templates. (2) po-web-ui has ONLY throttles+duration alarms (no errors/dlq +beyond what it had). (3) site_extractor has NO DLQ alarm. (4) wo web_ui has +ZERO alarms — prove the shared helper did NOT silently add any (search the +wo template for any alarm whose dimensions point at the wo web_ui +function). (5) Every bespoke alarm description string is byte-identical to +BASE (diff the AlarmDescription fields). (6) Fallback-rate: PO excludes the +rejected series (rejected_included=False), WO includes it; the two rejected +alarms kept their distinct shapes (PO 6h count-floor, WO 5-min/2-of-6); NO +element-wise MAX anywhere. confirmed=true only with a template-level diff +showing a homogenized or dropped alarm.` }, + { key: 'iam-equivalence', prompt: `${PREAMBLE} +ADVERSARIAL REVIEW — IAM-equivalence lens (the cross-family-review surface). +make_bedrock_invoke_statement moved the PolicyStatement construction and +swapped hardcoded 328440206208 for Stack.of(scope).account/.region. Prove +the produced IAM is IDENTICAL. (1) Synth both stacks and extract the Bedrock +PolicyStatement from each template; diff actions and resources against +${BASE}'s synthesized statements — the resolved ARN strings (account + +region substituted) must be byte-identical in-account. (2) Confirm the +resources still enumerate the SAME inference-profile ARN and the SAME +per-region foundation-model ARNs (no region dropped/added, no wildcard +broadening). (3) Confirm effect/conditions/principal attachment unchanged +and the statement is attached to the SAME role. (4) Flag any broadening +(e.g. a Ref/Sub that resolves to a wildcard, or Stack.region producing a +different region than the hardcoded one) as confirmed HIGH. confirmed=true +only with the two synthesized statements diffed.` }, +] + +let round = 0 +let checks = null +let diff = null +let confirmed = [] +while (round < 3) { + phase('Verify') + const results = await parallel([ + () => agent(mechanicalPrompt, { label: `verify:mechanical-r${round}`, model: 'sonnet', phase: 'Verify', schema: CHECKS }), + () => agent(diffPrompt, { label: `verify:cdk-diff-r${round}`, model: 'opus', phase: 'Verify', schema: DIFF }), + ...lenses.map(l => () => + agent(l.prompt, { label: `verify:${l.key}-r${round}`, phase: 'Verify', schema: FINDINGS })), + ]) + checks = results[0] + diff = results[1] + confirmed = results.slice(2).filter(Boolean) + .flatMap(r => r.findings || []) + .filter(f => f.confirmed && f.severity !== 'low') + const green = checks && checks.passed && diff && diff.passed + log(`Verify round ${round}: mechanical ${checks && checks.passed ? 'GREEN' : 'RED'}, cdk-diff ${diff && diff.passed ? 'GREEN' : 'RED'}, confirmed findings: ${confirmed.length}`) + if (green && confirmed.length === 0) break + + round += 1 + if (round >= 3) break + phase('Fix') + await agent(`${PREAMBLE} +You are the fix agent — you may edit files under: ${EXPECTED_SCOPE.join(', ')}. +Fix EVERY item below minimally; the binding spec and 10 pinned constraints +still hold (a finding that conflicts with a constraint is reported, not +"fixed" — the constraint wins, esp. constraint 1's plain-function / +no-Construct-subclass rule, constraint 3's alarm variance, and constraint 4's +distinct rejected alarms / no element-wise MAX). Re-run the specific failing +gate per fix (the zero-cdk-diff check is authoritative — a fix that +introduces ANY logical-ID or property delta is worse than the finding). +MECHANICAL:\n${checks ? checks.details : '(agent died — rerun all gates)'} +CDK-DIFF:\n${diff ? diff.details : '(agent died — rerun all)'} +CONFIRMED FINDINGS:\n${JSON.stringify(confirmed, null, 2)} +${specBlock}`, + { label: `fix:round-${round}`, model: 'opus', phase: 'Fix', schema: IMPL }) +} + +const verifyClean = checks && checks.passed && diff && diff.passed && confirmed.length === 0 +if (!verifyClean) { + return { + status: 'NEEDS ATTENTION — verify not clean after 3 rounds; branch left uncommitted', + branch: BRANCH, + mechanical: checks, + cdkDiff: diff, + unresolvedFindings: confirmed, + implBlockers, + reconBlockers, + spec, + } +} + +// ----------------------------------------------------------------- package + +phase('Package') +const commit = await agent(`${PREAMBLE.replace('do NOT commit, ', '')} +YOU are the commit agent: +1. MANDATORY GPT-4.1 CROSS-FAMILY REVIEW (the doc mandates it for the + make_bedrock_invoke_statement IAM PolicyStatement move, even though + semantics are identical). Capture the Bedrock-statement diff first: + git diff ${BASE}...HEAD -- cdk/common.py cdk/po_stack.py cdk/wo_stack.py + (isolate the make_bedrock_invoke_statement + its two call sites), then + RUN inline: + python3 ~/Documents/repositories/seahaven/security-review/cross_review.py + "Review this CDK IAM PolicyStatement move for breaking changes: the two + inline Bedrock invoke PolicyStatements (hardcoded account 328440206208) + were consolidated into make_bedrock_invoke_statement in cdk/common.py, + deriving the inference-profile + per-region foundation-model ARNs from + Stack.of(scope).account/.region. Confirm the produced actions/resources + are byte-identical in-account and no privilege broadening. " + Record the verdict verbatim in your summary. If cross_review.py is + unavailable, DO NOT block the local commit but flag the review as + OUTSTANDING in your summary (it must be run before merge). +2. Read ~/Documents/repositories/seahaven/engineering-handbook/commit-messages.md + and follow it exactly. +3. git add only paths under: ${EXPECTED_SCOPE.join(', ')} and + .claude/workflows/phase-4-cdk-common.js. NOT .coverage, NOT package/, + NOT cdk.out. Verify the staged set with git status. +4. ONE commit; write the message to /tmp/phase4-commit-msg.txt and use + git commit -F /tmp/phase4-commit-msg.txt (backticks in -m get eaten by + zsh). Suggested subject: + "feat: extract cdk/common.py — dedup 379 CDK lines as logical-ID-safe plain helpers (refactor phase 4)" + Body: the plain-function (scope, id, ...) approach and WHY no Construct + subclass (RETAIN-table replacement), the account/region-derived Bedrock + ARN, the zero-cdk-diff acceptance evidence for both stacks, the + account=/constructs pin/stale-comment/CfnOutput net-new additions, and + the cross_review.py verdict one-liner. NO AI attribution / Co-Authored-By + lines. +5. Do NOT push. Return commit sha + shortstat + the cross_review.py verdict + in summary.`, + { label: 'package:commit', model: 'sonnet', phase: 'Package', schema: IMPL }) + +return { + status: 'BUILT — committed locally, NOT pushed', + branch: BRANCH, + base: BASE, + commit: commit ? commit.summary : 'commit agent died — commit manually', + spec: { commonModule: spec.commonModule, bedrockStatement: spec.bedrockStatement, fallbackRateCalls: spec.fallbackRateCalls, appAndPins: spec.appAndPins, notes: spec.notes }, + diffEvidence: diff ? { po: diff.poDiffVerdict, wo: diff.woDiffVerdict } : null, + implementation: implOk.map(r => r.summary), + filesChanged: implOk.flatMap(r => r.filesChanged), + verifyRounds: round + 1, + blockers: implBlockers.concat(reconBlockers), + outstandingGates: [ + 'MANDATORY cross-family GPT-4.1 review (cross_review.py) on the make_bedrock_invoke_statement IAM PolicyStatement move — the Package agent runs it inline and records the verdict; confirm that verdict is clean (or re-run) before merge. If cross_review.py was unavailable at commit time, this review is OUTSTANDING — do not merge without it.', + '/sh-security-review NOT required (pure CDK refactor — no untrusted-input/auth-logic change; the pre-push scanners still run as the unattended backstop)', + 'push + PR + gh pr checks green', + 'deploy-then-merge with cdk diff zero-change on BOTH stacks as the live acceptance signal: deploy from branch, confirm cdk diff po-ingest / cdk diff workorder-ingest show only the CfnOutput additions, both stacks reach UPDATE_COMPLETE, all alarms still OK, THEN merge (a no-op resource redeploy is itself the verification signal). Drive synth/diff by artifact id workorder-ingest, NOT stack_name WorkorderIngestStack (constraint 9).', + 'update the Confluence "AWS Architecture Map" if the new CfnOutputs / account-explicit envs change the documented resource inventory', + ], +} diff --git a/README.md b/README.md index 61ba9ab..9bd9227 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Amazon APM work order emails (from Hexagon EAM / HxGN SmartCloud) are received a ## Architecture -**IaC:** AWS CDK (Python), two stacks in one app, region `us-east-1`. +**IaC:** AWS CDK (Python), two stacks in one app, region `us-east-1`. The `cdk.Environment` is deliberately **account-agnostic** (region-only, no `account=`): the stacks deploy to whichever account the deploy credentials target (`328440206208` today), and every account-derived template value — bucket names, `Lambda::Permission` source account, the site-alerts SNS action ARN, the Bedrock ARN below — renders as the CloudFormation `AWS::AccountId` pseudo-parameter rather than a literal. Pinning `account=` was evaluated in Phase 4 and rejected: it would resolve those tokens to literals, and against the deployed (account-agnostic) templates CloudFormation flags the `RemovalPolicy.RETAIN` email buckets as requiring replacement — a data-loss risk — for no functional gain. All Lambdas: Python 3.12, ARM64, 60-day log retention. @@ -98,6 +98,18 @@ All Lambdas: Python 3.12, ARM64, 60-day log retention. **SES:** Both stacks add rules to the shared `INBOUND_MAIL` receipt rule set on `int.seahaven.com`. +### Shared CDK helpers (`cdk/common.py`, Phase 4) + +The ~379 lines that `po_stack.py` and `wo_stack.py` both defined identically (DynamoDB alarms, the sender-auth-rejected metric filter + alarm, the standard per-Lambda alarm set, the Bedrock `InvokeModel` grant, the raw-email bucket, the async-invoke DLQ, and the template-fallback-rate math alarm) are collapsed into `cdk/common.py`. + +**Logical-ID-safety rule (load-bearing).** Every helper is a **plain function** taking `(scope, id, ...)` — never a `Construct` subclass. Each stack calls a helper with its **own Stack instance as `scope`** and the **exact same literal construct id** it used inline before the extraction, so every synthesized logical ID is byte-stable. A `Construct` subclass would insert an extra tree node, reparent every child's logical ID, and CloudFormation would attempt to **replace** the `RemovalPolicy.RETAIN`-protected `purchase-orders` / `WorkOrders` / `WorkOrderComments` tables and the named S3 buckets — a data-loss event. Nothing in `common.py` subclasses `Construct`, and it imports only the CDK constructs its helpers touch (`dynamodb`, `cloudwatch`/`cw_actions`, `iam`, `logs`, `s3`, `sqs` — no `kms`, `ssm`, or Lambda event-source imports). + +Extracted helpers: `add_ddb_alarms`, `add_sender_auth_rejected_alarm`, `add_standard_lambda_alarms` (bespoke `descriptions=` dict passed through verbatim per call site — no alarm text is generated or homogenized), `make_bedrock_invoke_statement`, `make_email_bucket`, `make_processor_dlq`, `make_fallback_rate_alarm`. Per-function alarm variance is preserved exactly through call-site arguments, not baked into the helpers: PO's email-processor duration alarm uses `p99`, WO's uses `p95`; `po-web-ui` gets throttles + duration only (no errors, no DLQ); `po-ingest-site-extractor` has no DLQ alarm (it's a DynamoDB-stream consumer, not async-invoked); `workorder-web-ui` gets **zero** alarms — the helper is simply never called for it, so it cannot silently add any. The two "AI-fallback-rejected" alarms (PO's 6-hour count-floor `IF(FILL(rej,0)>=1,...)`, WO's 5-minute/2-of-6 sparse idiom) are a different shape from `make_fallback_rate_alarm` and stay as distinct inline call sites in each stack rather than being forced through the shared helper. + +**Bedrock ARN, now account/region-derived.** `make_bedrock_invoke_statement(scope)` builds the inference-profile ARN and the `us-east-1` foundation-model ARN from `Stack.of(scope).account` / `.region` instead of the previous hardcoded `328440206208`/`us-east-1` literals (the `us-east-2`/`us-west-2` foundation-model ARNs stay literal — they're fixed cross-region reach targets, not the stack's own region). Because the environment is account-agnostic (above), `stack.account` is the `AWS::AccountId` pseudo-parameter, so the derived ARN synthesizes as an `Fn::Sub`/`Ref` that **resolves at deploy time to the same ARN** the hardcoded literal named in-account. Versus the deployed stack this is a benign **in-place** IAM policy update (IAM policies never require replacement) — and it makes the grant account-portable instead of pinned to the management account. This IAM PolicyStatement change is the surface the mandatory GPT-4.1 cross-family review covers. + +**New `CfnOutput`s.** Each stack now exports its Lambda function ARNs and the DynamoDB table names it owns/consumes, all net-new/additive (no existing output changes): `po-ingest` — `EmailProcessorFunctionArn`, `WebUiFunctionArn`, `SiteExtractorFunctionArn`, `PurchaseOrdersTableName`, `PendingSiteReviewTableName` (the existing `VerifiedSitesTableName` output is unchanged); `workorder-ingest` — `EmailProcessorFunctionArn`, `WebUiFunctionArn`, `WorkOrdersTableName`, `WorkOrderCommentsTableName`. + ### Sender authentication (INFRA-107) The `From` header and any `Authentication-Results` header inside the raw MIME are attacker-forgeable, so neither is trusted. Instead, both email processors authenticate the sender against the verdicts SES itself stamps at delivery time, failing closed. The authenticator is **single-sourced** at `lambdas/shared/ses_auth.py` (Phase 3) — previously duplicated byte-for-byte in each pipeline's `email_processor/` dir and kept in sync by a fixture-hygiene test; now one copy, so a future hardening fix to the fail-closed logic lands **once** instead of needing two identical edits. The CDK bundling `cp`s it flat beside each handler so the handlers' unchanged `from ses_auth import authenticate_inbound_email` still resolves at runtime (see [`lambdas/shared/`](#deploy-pipeline-guards-phase-0)). The gate: @@ -340,7 +352,10 @@ python scripts/backfill_sites.py ``` cdk/ - app.py # Two stacks: po-ingest + WorkorderIngestStack + app.py # Two stacks: po-ingest + WorkorderIngestStack (region-only env) + common.py # Phase 4: shared plain-function CDK helpers (alarms, Bedrock grant, + # email bucket, processor DLQ, fallback-rate alarm) -- called with + # each stack's own scope + literal construct ids, logical-ID-safe po_stack.py # Purchase order pipeline resources wo_stack.py # Work order pipeline resources lambdas/ # Phase 2: shared Code.from_asset("../lambdas") bundling root for diff --git a/cdk/common.py b/cdk/common.py new file mode 100644 index 0000000..5dec302 --- /dev/null +++ b/cdk/common.py @@ -0,0 +1,331 @@ +"""Shared CDK helpers for the procurement-ingest stacks (Phase 4). + +Plain free functions extracted from po_stack.py / wo_stack.py. Each takes the +same Stack `scope` and the same literal construct `id` the stacks passed inline, +so every synthesized logical ID is byte-stable. NOTHING here is a Construct +subclass: a subclass would insert a tree node and reparent/replace the +RETAIN-protected tables and named buckets. +""" + +from aws_cdk import ( + Duration, + RemovalPolicy, + Stack, + aws_cloudwatch as cloudwatch, + aws_cloudwatch_actions as cw_actions, + aws_dynamodb as dynamodb, + aws_iam as iam, + aws_logs as logs, + aws_s3 as s3, + aws_sqs as sqs, +) + +# --- DynamoDB alarm operations (moved verbatim from both stacks) --- +_DDB_ALARM_OPERATIONS = [ + dynamodb.Operation.GET_ITEM, + dynamodb.Operation.BATCH_GET_ITEM, + dynamodb.Operation.QUERY, + dynamodb.Operation.SCAN, + dynamodb.Operation.PUT_ITEM, + dynamodb.Operation.UPDATE_ITEM, + dynamodb.Operation.DELETE_ITEM, + dynamodb.Operation.BATCH_WRITE_ITEM, +] + +# CloudWatch namespace for the log-derived sender-authentication metrics. +_SENDER_AUTH_METRIC_NAMESPACE = "Seahaven/ProcurementIngest" + + +def add_ddb_alarms(scope, id_prefix, table, alarm_name_prefix, alarm_topic): + """Add throttle + system-error alarms for a DynamoDB table. + + Both fire on any non-zero datapoint in a 5-min window. ALARM-only SnsAction + to site-alerts (no OK action); TreatMissingData NOT_BREACHING. + """ + table.metric_throttled_requests_for_operations( + operations=_DDB_ALARM_OPERATIONS, + period=Duration.minutes(5), + statistic="Sum", + ).create_alarm( + scope, + f"{id_prefix}ThrottlesAlarm", + alarm_name=f"{alarm_name_prefix}-throttles", + alarm_description=f"{alarm_name_prefix} DynamoDB throttled requests", + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + + table.metric_system_errors_for_operations( + operations=_DDB_ALARM_OPERATIONS, + period=Duration.minutes(5), + statistic="Sum", + ).create_alarm( + scope, + f"{id_prefix}SystemErrorsAlarm", + alarm_name=f"{alarm_name_prefix}-system-errors", + alarm_description=f"{alarm_name_prefix} DynamoDB server-side (5xx) errors", + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + + +def add_sender_auth_rejected_alarm(scope, id_prefix, function_name, alarm_topic): + """Metric-filter + alarm on ``sender_auth_rejected`` warnings (INFRA-107). + + A rejected inbound email is skipped without erroring the invocation, so it + is invisible to the Errors/Throttles/DLQ alarms. This turns the structured + warning log into a CloudWatch metric and pages when rejections spike -- + catching a silent false-reject storm (allowlist wrong, signing-domain + drift, SES header-format change) that would otherwise discard legitimate + mail while the pipeline reports healthy. + + ALARM-only SnsAction to site-alerts; no OK action. The metric filter reads + the function's own log group (imported by the deterministic + ``/aws/lambda/`` name, created by the function's log_retention). A plain + substring pattern is used because Lambda prefixes each line with its own + level/timestamp/request-id, so the JSON payload is not a standalone JSON + log event a `{$.event=...}` pattern could match. + """ + metric_name = f"{function_name}-sender-auth-rejected" + logs.MetricFilter( + scope, + f"{id_prefix}SenderAuthRejectedFilter", + log_group=logs.LogGroup.from_log_group_name( + scope, + f"{id_prefix}LogGroup", + f"/aws/lambda/{function_name}", + ), + filter_pattern=logs.FilterPattern.literal('"sender_auth_rejected"'), + metric_namespace=_SENDER_AUTH_METRIC_NAMESPACE, + metric_name=metric_name, + metric_value="1", + default_value=0, + ) + + cloudwatch.Metric( + namespace=_SENDER_AUTH_METRIC_NAMESPACE, + metric_name=metric_name, + period=Duration.minutes(5), + statistic="Sum", + ).create_alarm( + scope, + f"{id_prefix}SenderAuthRejectedAlarm", + alarm_name=f"{function_name}-sender-auth-rejected", + alarm_description=( + f"{function_name} rejected inbound mail on sender authentication " + "(possible allowlist/DKIM-domain drift silently dropping real mail)" + ), + threshold=1, + evaluation_periods=6, + datapoints_to_alarm=2, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + + +def add_standard_lambda_alarms( + scope, + id_prefix, + fn, + name_prefix, + topic, + *, + duration_statistic, + errors=True, + dlq=None, + descriptions, +): + """Standard per-Lambda alarm set: errors (optional), throttles, dlq + (optional), duration. Every alarm bespoke-described via `descriptions` + (keys: errors/throttles/dlq/duration passed through VERBATIM). Construct + ids are f"{id_prefix}Alarm", alarm names f"{name_prefix}-", + exactly the inline literals. Order of creation is irrelevant to logical IDs + (ids are explicit) so the fixed errors->throttles->dlq->duration order here + reproduces both PO (dlq before duration) and WO (duration before dlq) + templates identically. + """ + if errors: + fn.metric_errors(period=Duration.minutes(5), statistic="Sum").create_alarm( + scope, + f"{id_prefix}ErrorsAlarm", + alarm_name=f"{name_prefix}-errors", + alarm_description=descriptions["errors"], + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(topic)) + + fn.metric_throttles(period=Duration.minutes(5), statistic="Sum").create_alarm( + scope, + f"{id_prefix}ThrottlesAlarm", + alarm_name=f"{name_prefix}-throttles", + alarm_description=descriptions["throttles"], + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(topic)) + + if dlq is not None: + dlq.metric_approximate_number_of_messages_visible( + period=Duration.minutes(5), + statistic="Maximum", + ).create_alarm( + scope, + f"{id_prefix}DlqMessagesAlarm", + alarm_name=f"{name_prefix}-dlq-messages", + alarm_description=descriptions["dlq"], + threshold=0, + evaluation_periods=1, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(topic)) + + fn.metric_duration( + period=Duration.minutes(5), + statistic=duration_statistic, + ).create_alarm( + scope, + f"{id_prefix}DurationAlarm", + alarm_name=f"{name_prefix}-duration", + alarm_description=descriptions["duration"], + threshold=45000, + evaluation_periods=3, + datapoints_to_alarm=2, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(topic)) + + +def make_bedrock_invoke_statement(scope): + """Return the Bedrock InvokeModel PolicyStatement for the email processor. + + The us.* inference profile can route cross-region, so the grant covers the + inference-profile ARN plus the per-region foundation-model ARNs + (us-east-1/us-east-2/us-west-2). ARN #1 account and ARN #1/#2 region are + DERIVED from Stack.of(scope).account/.region (not hardcoded 328440206208); + ARN #3/#4 regions (us-east-2/us-west-2) are cross-region reach targets, NOT + the stack's own region, so they stay literal. Caller attaches via + fn.add_to_role_policy(...) on the SAME function -> logical-ID-safe. + """ + stack = Stack.of(scope) + account = stack.account + region = stack.region + return iam.PolicyStatement( + actions=[ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + resources=[ + f"arn:aws:bedrock:{region}:{account}:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0", + f"arn:aws:bedrock:{region}::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + "arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", + ], + ) + + +def make_email_bucket(scope, id, name_prefix): + """Raw-email S3 bucket: BLOCK_ALL public, RETAIN, 90-day expiration. + bucket_name = f"{name_prefix}-{account}" (account derived from the stack), + reproducing f"po-ingest-emails-{self.account}" / + f"workorder-ingest-emails-{self.account}" exactly. + """ + return s3.Bucket( + scope, + id, + bucket_name=f"{name_prefix}-{Stack.of(scope).account}", + block_public_access=s3.BlockPublicAccess.BLOCK_ALL, + removal_policy=RemovalPolicy.RETAIN, + lifecycle_rules=[ + s3.LifecycleRule(expiration=Duration.days(90)), + ], + ) + + +def make_processor_dlq(scope, id): + """Async-invoke DLQ: 14-day retention, enforce_ssl. CDK-generated name.""" + return sqs.Queue( + scope, + id, + retention_period=Duration.days(14), + enforce_ssl=True, + ) + + +def make_fallback_rate_alarm( + scope, + id, + *, + namespace, + alarm_topic, + alarm_name, + alarm_description, + rejected_included, + period, + threshold, + floor, + evaluation_periods, + datapoints_to_alarm, +): + """Template-fallback-rate MathExpression alarm. + + rejected_included=False (PO): numerator FILL(fb,0), denom fb+tmpl. + rejected_included=True (WO): numerator (fb+rej), denom fb+rej+tmpl. + Expression string, FILL, label reproduced BYTE-FOR-BYTE. GREATER_THAN, + NOT_BREACHING. NO element-wise MAX (post-#102). The two DISTINCT rejected + alarms are NOT built here. + """ + fb_metric = cloudwatch.Metric( + namespace=namespace, + metric_name="ParseOutcome", + dimensions_map={"ParseMethod": "ai_fallback"}, + statistic="Sum", + period=period, + ) + tmpl_metric = cloudwatch.Metric( + namespace=namespace, + metric_name="ParseOutcome", + dimensions_map={"ParseMethod": "template"}, + statistic="Sum", + period=period, + ) + if rejected_included: + fb_rej_metric = cloudwatch.Metric( + namespace=namespace, + metric_name="ParseOutcome", + dimensions_map={"ParseMethod": "ai_fallback_rejected"}, + statistic="Sum", + period=period, + ) + using_metrics = {"fb": fb_metric, "rej": fb_rej_metric, "tmpl": tmpl_metric} + sum_terms = "FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0)" + numerator = "(FILL(fb,0)+FILL(rej,0))" + else: + using_metrics = {"fb": fb_metric, "tmpl": tmpl_metric} + sum_terms = "FILL(fb,0)+FILL(tmpl,0)" + numerator = "FILL(fb,0)" + expression = f"IF(({sum_terms})>={floor}, 100*{numerator}/({sum_terms}), 0)" + + fallback_rate = cloudwatch.MathExpression( + expression=expression, + using_metrics=using_metrics, + period=period, + label="TemplateFallbackRatePct", + ) + fallback_rate.create_alarm( + scope, + id, + alarm_name=alarm_name, + alarm_description=alarm_description, + threshold=threshold, + evaluation_periods=evaluation_periods, + datapoints_to_alarm=datapoints_to_alarm, + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, + treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, + ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) diff --git a/cdk/po_stack.py b/cdk/po_stack.py index a63a4ba..1cfb51c 100644 --- a/cdk/po_stack.py +++ b/cdk/po_stack.py @@ -8,7 +8,6 @@ from aws_cdk import ( aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, - aws_iam as iam, aws_kms as kms, aws_lambda as lambda_, aws_lambda_event_sources as lambda_event_sources, @@ -19,136 +18,11 @@ from aws_cdk import ( aws_ses_actions as ses_actions, aws_secretsmanager as secretsmanager, aws_sns as sns, - aws_sqs as sqs, aws_ssm as ssm, ) from constructs import Construct -# Operations these tables actually issue (PutItem/UpdateItem/DeleteItem writes, -# GetItem/Query/BatchGetItem reads). DynamoDB emits ThrottledRequests/SystemErrors -# keyed by TableName + Operation only, so the CDK *_for_operations helpers (which -# render a SUM MathExpression across these per-operation metrics) are the correct, -# non-deprecated way to roll a table up to a single alarmable series. -_DDB_ALARM_OPERATIONS = [ - dynamodb.Operation.GET_ITEM, - dynamodb.Operation.BATCH_GET_ITEM, - dynamodb.Operation.QUERY, - dynamodb.Operation.SCAN, - dynamodb.Operation.PUT_ITEM, - dynamodb.Operation.UPDATE_ITEM, - dynamodb.Operation.DELETE_ITEM, - dynamodb.Operation.BATCH_WRITE_ITEM, -] - - -def _add_ddb_alarms(scope, id_prefix, table, alarm_name_prefix, alarm_topic): - """Add throttle + system-error alarms for a DynamoDB table. - - Both fire on any non-zero datapoint in a 5-min window. ALARM-only SnsAction - to site-alerts (no OK action); TreatMissingData NOT_BREACHING. - """ - table.metric_throttled_requests_for_operations( - operations=_DDB_ALARM_OPERATIONS, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}ThrottlesAlarm", - alarm_name=f"{alarm_name_prefix}-throttles", - alarm_description=f"{alarm_name_prefix} DynamoDB throttled requests", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - table.metric_system_errors_for_operations( - operations=_DDB_ALARM_OPERATIONS, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}SystemErrorsAlarm", - alarm_name=f"{alarm_name_prefix}-system-errors", - alarm_description=f"{alarm_name_prefix} DynamoDB server-side (5xx) errors", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - -# CloudWatch namespace for the log-derived sender-authentication metrics. -_SENDER_AUTH_METRIC_NAMESPACE = "Seahaven/ProcurementIngest" - - -def _add_sender_auth_rejected_alarm(scope, id_prefix, function_name, alarm_topic): - """Metric-filter + alarm on ``sender_auth_rejected`` warnings (INFRA-107). - - A rejected inbound email is skipped without erroring the invocation, so it - is invisible to the Errors/Throttles/DLQ alarms. This turns the structured - warning log into a CloudWatch metric and pages when rejections spike -- - catching a silent false-reject storm (allowlist wrong, signing-domain - drift, SES header-format change) that would otherwise discard legitimate - mail while the pipeline reports healthy. - - ALARM-only SnsAction to site-alerts; no OK action. The metric filter reads - the function's own log group (imported by the deterministic - ``/aws/lambda/`` name, created by the function's log_retention). A plain - substring pattern is used because Lambda prefixes each line with its own - level/timestamp/request-id, so the JSON payload is not a standalone JSON - log event a `{$.event=...}` pattern could match. - """ - metric_name = f"{function_name}-sender-auth-rejected" - logs.MetricFilter( - scope, - f"{id_prefix}SenderAuthRejectedFilter", - log_group=logs.LogGroup.from_log_group_name( - scope, - f"{id_prefix}LogGroup", - f"/aws/lambda/{function_name}", - ), - filter_pattern=logs.FilterPattern.literal('"sender_auth_rejected"'), - metric_namespace=_SENDER_AUTH_METRIC_NAMESPACE, - metric_name=metric_name, - metric_value="1", - default_value=0, - ) - - # Fire on a *sustained* reject condition rather than a volume spike. The - # earlier Sum>=3-over-15-min threshold had a blind spot that is exactly the - # failure this alarm exists to catch: a low-traffic pipeline in total - # drift outage (allowlist wrong / signing-domain changed) may only produce - # a trickle of rejects -- one every few minutes -- that never sums to 3 in - # any window, so the outage never pages. Instead: >=1 reject per 5-min - # period, alarming when 2 of the last 6 periods breach (evaluation_periods=6 - # / datapoints_to_alarm=2). Six periods (30 min) with only 2 required - # datapoints closes the sparse-outage residual: even rejections >10-15 min - # apart can still place two breaching datapoints in a single 30-min - # evaluation window. A single stray spoof probe (one lone period) is - # tolerated and self-clears, but a sustained reject condition trips even - # at very low arrival rates. default_value=0 on the metric filter keeps the - # series continuous so NOT_BREACHING only applies before the first datapoint - # ever arrives. - cloudwatch.Metric( - namespace=_SENDER_AUTH_METRIC_NAMESPACE, - metric_name=metric_name, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}SenderAuthRejectedAlarm", - alarm_name=f"{function_name}-sender-auth-rejected", - alarm_description=( - f"{function_name} rejected inbound mail on sender authentication " - "(possible allowlist/DKIM-domain drift silently dropping real mail)" - ), - threshold=1, - evaluation_periods=6, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) +import common class PoIngestStack(Stack): @@ -167,16 +41,7 @@ class PoIngestStack(Stack): ) # --- S3 bucket for raw emails --- - email_bucket = s3.Bucket( - self, - "EmailBucket", - bucket_name=f"po-ingest-emails-{self.account}", - block_public_access=s3.BlockPublicAccess.BLOCK_ALL, - removal_policy=RemovalPolicy.RETAIN, - lifecycle_rules=[ - s3.LifecycleRule(expiration=Duration.days(90)), - ], - ) + email_bucket = common.make_email_bucket(self, "EmailBucket", "po-ingest-emails") # --- Shared customer-managed CMK for sensitive DynamoDB tables --- # Owned by the account-baseline app (alias/seahaven-dynamodb, INFRA-95 / @@ -223,12 +88,7 @@ class PoIngestStack(Stack): # parse (bad email, transient error) is silently dropped after Lambda's # retries. CDK generates the queue name to avoid colliding with the # interim CLI-created po-email-processor-dlq (removed post-deploy). - email_processor_dlq = sqs.Queue( - self, - "EmailProcessorDlq", - retention_period=Duration.days(14), - enforce_ssl=True, - ) + email_processor_dlq = common.make_processor_dlq(self, "EmailProcessorDlq") # --- Lambda function --- email_processor = lambda_.Function( @@ -301,37 +161,29 @@ class PoIngestStack(Stack): # (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at # runtime whenever the profile routes to a region whose foundation-model # ARN is not allowed. - email_processor.add_to_role_policy( - iam.PolicyStatement( - actions=[ - "bedrock:InvokeModel", - "bedrock:InvokeModelWithResponseStream", - ], - resources=[ - "arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - ], - ) - ) + email_processor.add_to_role_policy(common.make_bedrock_invoke_statement(self)) - # --- Errors alarm (INFRA-41 / audit H-8) --- - # ALARM-only (no OK action, per the CloudWatch-alarm preference) to the - # shared site-alerts topic. Any errored invocation in a 5-min window pages. - email_processor.metric_errors( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( + # --- Standard per-Lambda alarms: po-email-processor --- + # errors (INFRA-41 / audit H-8), throttles, DLQ-messages (dropped PO + # emails), and a p99 duration alarm (orphan adoption of the CLI + # Lambda-Duration-po-email-processor under -duration naming, 45000 ms + # = 75% of the 60s timeout, eval 3 / dp 2). All ALARM-only to site-alerts. + common.add_standard_lambda_alarms( self, - "EmailProcessorErrorsAlarm", - alarm_name="po-email-processor-errors", - alarm_description="po-email-processor async invocation errors", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + "EmailProcessor", + email_processor, + "po-email-processor", + alarm_topic, + duration_statistic="p99", + errors=True, + dlq=email_processor_dlq, + descriptions={ + "errors": "po-email-processor async invocation errors", + "throttles": "po-email-processor invocation throttles", + "dlq": "po-email-processor DLQ has messages (dropped PO emails)", + "duration": "po-email-processor p99 duration approaching the 60s timeout", + }, + ) # --- Sender-auth rejection alarm (INFRA-107) --- # A rejected email (bad/unaligned DKIM verdict) returns normally, so it @@ -343,70 +195,10 @@ class PoIngestStack(Stack): # A CloudWatch Logs metric filter turns those warnings into a metric so a # false-reject storm pages instead of vanishing. default_value=0 keeps the # series populated (alarm stays OK, never INSUFFICIENT_DATA) between events. - _add_sender_auth_rejected_alarm( + common.add_sender_auth_rejected_alarm( self, "EmailProcessor", "po-email-processor", alarm_topic ) - # --- Throttles alarm: po-email-processor --- - # Any throttled invocation (concurrency cap hit) in a 5-min window pages. - # ALARM-only to site-alerts; no OK action; NOT_BREACHING when no data. - email_processor.metric_throttles( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - self, - "EmailProcessorThrottlesAlarm", - alarm_name="po-email-processor-throttles", - alarm_description="po-email-processor invocation throttles", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- DLQ messages-present alarm --- - # Pages when any message lands in the EmailProcessorDlq: a message here - # means a PO email was permanently dropped after Lambda exhausted its - # async retries. Maximum over a single 5-min window > 0 fires; missing - # data (no messages metric emitted) is not breaching. Reuses the shared - # site-alerts topic, ALARM-only, like the errors alarm above. The metric - # helper derives the QueueName dimension from the queue construct, so the - # alarm tracks the CDK-generated queue name without hardcoding it. - email_processor_dlq.metric_approximate_number_of_messages_visible( - period=Duration.minutes(5), - statistic="Maximum", - ).create_alarm( - self, - "EmailProcessorDlqMessagesAlarm", - alarm_name="po-email-processor-dlq-messages", - alarm_description="po-email-processor DLQ has messages (dropped PO emails)", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- Duration alarm: po-email-processor (orphan adoption) --- - # Adopts the orphaned CLI alarm Lambda-Duration-po-email-processor under - # the repo's -duration naming (NEW logical name → no deploy collision; - # delete the orphan post-deploy). p99 / 45000 ms - # (75% of the 60s timeout) / eval 3 of 3 — tighter than the orphan's - # Maximum>=48000 / 1-of-1. - email_processor.metric_duration( - period=Duration.minutes(5), - statistic="p99", - ).create_alarm( - self, - "EmailProcessorDurationAlarm", - alarm_name="po-email-processor-duration", - alarm_description="po-email-processor p99 duration approaching the 60s timeout", - threshold=45000, - evaluation_periods=3, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - # --- Template fallback-rate alarm: po-email-processor --- # The processor tries a deterministic template parse first and only calls # the Bedrock AI extractor on a miss/invalid. A sustained rise in the @@ -452,43 +244,23 @@ class PoIngestStack(Stack): # falsely page this template-drift alarm on top of the dedicated # rejected alarm below. The rejected series gets its own alarm # instead (EmailProcessorAiFallbackRejectedAlarm, below). - fb_metric = cloudwatch.Metric( - namespace="Seahaven/PoIngest", - metric_name="ParseOutcome", - dimensions_map={"ParseMethod": "ai_fallback"}, - statistic="Sum", - period=Duration.hours(6), - ) - tmpl_metric = cloudwatch.Metric( - namespace="Seahaven/PoIngest", - metric_name="ParseOutcome", - dimensions_map={"ParseMethod": "template"}, - statistic="Sum", - period=Duration.hours(6), - ) - fallback_rate = cloudwatch.MathExpression( - expression=( - "IF((FILL(fb,0)+FILL(tmpl,0))>=8, " - "100*FILL(fb,0)/(FILL(fb,0)+FILL(tmpl,0)), 0)" - ), - using_metrics={"fb": fb_metric, "tmpl": tmpl_metric}, - period=Duration.hours(6), - label="TemplateFallbackRatePct", - ) - fallback_rate.create_alarm( + common.make_fallback_rate_alarm( self, "EmailProcessorTemplateFallbackRateAlarm", + namespace="Seahaven/PoIngest", + alarm_topic=alarm_topic, alarm_name="po-email-processor-template-fallback-rate", alarm_description=( "po-email-processor deterministic-template coverage collapse: " ">20% of parses fell back to the Bedrock AI extractor" ), + rejected_included=False, + period=Duration.hours(6), threshold=20, + floor=8, evaluation_periods=4, datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + ) # --- AI-fallback rejected alarm: po-email-processor (Phase 1) --- # The validate_ai_fallback gate (template_parser.py) fail-closes Bedrock @@ -652,38 +424,24 @@ class PoIngestStack(Stack): po_table.grant_read_data(web_ui) web_ui_auth_secret.grant_read(web_ui) - # --- Throttles alarm: po-web-ui --- - web_ui.metric_throttles( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( + # --- Standard per-Lambda alarms: po-web-ui --- + # Throttles + p99 duration only (no errors alarm, no DLQ -- web_ui is a + # synchronous read path with no async DLQ). p99 / 45000 ms (75% of the + # 60s timeout) / eval 3, datapoints 2. + common.add_standard_lambda_alarms( self, - "WebUiThrottlesAlarm", - alarm_name="po-web-ui-throttles", - alarm_description="po-web-ui invocation throttles", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- Duration alarm: po-web-ui --- - # Net-new (no orphan exists for this function). - # p99 / 45000 ms (75% of the 60s timeout) / eval 3, datapoints 2. - web_ui.metric_duration( - period=Duration.minutes(5), - statistic="p99", - ).create_alarm( - self, - "WebUiDurationAlarm", - alarm_name="po-web-ui-duration", - alarm_description="po-web-ui p99 duration approaching the 60s timeout", - threshold=45000, - evaluation_periods=3, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + "WebUi", + web_ui, + "po-web-ui", + alarm_topic, + duration_statistic="p99", + errors=False, + dlq=None, + descriptions={ + "throttles": "po-web-ui invocation throttles", + "duration": "po-web-ui p99 duration approaching the 60s timeout", + }, + ) # Public Function URL removed 2026-06-08 (INFRA-74 / audit C-5): the # unauthenticated FunctionUrlAuthType.NONE URL was deleted out-of-band @@ -745,56 +503,27 @@ class PoIngestStack(Stack): ) ) - # --- Errors alarm: po-ingest-site-extractor --- + # --- Standard per-Lambda alarms: po-ingest-site-extractor --- + # errors + throttles + p99 duration. NO DLQ alarm: site_extractor is a + # DynamoEventSource stream consumer with no async DLQ attached (dlq=None). # Stream-consumer errors retry per the event-source config, but a - # persistent failure stalls the verified-sites pipeline. ALARM-only to - # site-alerts; no OK action; NOT_BREACHING when no data. - site_extractor.metric_errors( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( + # persistent failure stalls the verified-sites pipeline. p99 / 45000 ms + # (75% of the 60s timeout) / eval 3, datapoints 2. + common.add_standard_lambda_alarms( self, - "SiteExtractorErrorsAlarm", - alarm_name="po-ingest-site-extractor-errors", - alarm_description="po-ingest-site-extractor invocation errors", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- Throttles alarm: po-ingest-site-extractor --- - site_extractor.metric_throttles( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - self, - "SiteExtractorThrottlesAlarm", - alarm_name="po-ingest-site-extractor-throttles", - alarm_description="po-ingest-site-extractor invocation throttles", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- Duration alarm: po-ingest-site-extractor --- - # Net-new (no orphan exists for this function). - # p99 / 45000 ms (75% of the 60s timeout) / eval 3, datapoints 2. - site_extractor.metric_duration( - period=Duration.minutes(5), - statistic="p99", - ).create_alarm( - self, - "SiteExtractorDurationAlarm", - alarm_name="po-ingest-site-extractor-duration", - alarm_description="po-ingest-site-extractor p99 duration approaching the 60s timeout", - threshold=45000, - evaluation_periods=3, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + "SiteExtractor", + site_extractor, + "po-ingest-site-extractor", + alarm_topic, + duration_statistic="p99", + errors=True, + dlq=None, + descriptions={ + "errors": "po-ingest-site-extractor invocation errors", + "throttles": "po-ingest-site-extractor invocation throttles", + "duration": "po-ingest-site-extractor p99 duration approaching the 60s timeout", + }, + ) cdk.CfnOutput( self, @@ -822,24 +551,56 @@ class PoIngestStack(Stack): # --- DynamoDB throttle + system-error alarms --- # ThrottledRequests / SystemErrors emit at TableName + Operation only # (verified against live CloudWatch: no TableName-only rollup exists, and - # metric_throttled_requests is deprecated/invalid in aws-cdk-lib 2.259.0). + # metric_throttled_requests is deprecated/invalid in aws-cdk-lib 2.261.0). # Each table currently has zero throttle/error datapoints, so the series # only materialise on first occurrence — NOT_BREACHING keeps them OK until # then. - _add_ddb_alarms( + common.add_ddb_alarms( self, "PurchaseOrdersTable", po_table, "purchase-orders", alarm_topic ) - _add_ddb_alarms( + common.add_ddb_alarms( self, "VerifiedSitesTable", verified_sites_table, "verified-sites", alarm_topic, ) - _add_ddb_alarms( + common.add_ddb_alarms( self, "PendingSiteReviewTable", pending_review_table, "pending-site-review", alarm_topic, ) + + # --- Function ARN + consumed-table-name outputs (Phase 4, additive) --- + cdk.CfnOutput( + self, + "EmailProcessorFunctionArn", + value=email_processor.function_arn, + description="ARN of the po-email-processor Lambda", + ) + cdk.CfnOutput( + self, + "WebUiFunctionArn", + value=web_ui.function_arn, + description="ARN of the po-web-ui Lambda", + ) + cdk.CfnOutput( + self, + "SiteExtractorFunctionArn", + value=site_extractor.function_arn, + description="ARN of the po-ingest-site-extractor Lambda", + ) + cdk.CfnOutput( + self, + "PurchaseOrdersTableName", + value=po_table.table_name, + description="purchase-orders DynamoDB table consumed by this stack", + ) + cdk.CfnOutput( + self, + "PendingSiteReviewTableName", + value=pending_review_table.table_name, + description="pending-site-review DynamoDB table", + ) diff --git a/cdk/requirements.txt b/cdk/requirements.txt index 786e564..4e6455d 100644 --- a/cdk/requirements.txt +++ b/cdk/requirements.txt @@ -1,2 +1,2 @@ aws-cdk-lib==2.261.0 -constructs>=10.6.0 +constructs==10.6.0 diff --git a/cdk/wo_stack.py b/cdk/wo_stack.py index e5b37e7..28756a2 100644 --- a/cdk/wo_stack.py +++ b/cdk/wo_stack.py @@ -8,7 +8,6 @@ from aws_cdk import ( aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cw_actions, aws_dynamodb as dynamodb, - aws_iam as iam, aws_lambda as lambda_, aws_logs as logs, aws_s3 as s3, @@ -17,138 +16,10 @@ from aws_cdk import ( aws_ses_actions as ses_actions, aws_secretsmanager as secretsmanager, aws_sns as sns, - aws_sqs as sqs, ) from constructs import Construct -# Operations these tables actually issue (PutItem/UpdateItem/DeleteItem writes, -# GetItem/Query/BatchGetItem reads). DynamoDB emits ThrottledRequests/SystemErrors -# keyed by TableName + Operation only, so the CDK *_for_operations helpers (which -# render a SUM MathExpression across these per-operation metrics) are the correct, -# non-deprecated way to roll a table up to a single alarmable series. -_DDB_ALARM_OPERATIONS = [ - dynamodb.Operation.GET_ITEM, - dynamodb.Operation.BATCH_GET_ITEM, - dynamodb.Operation.QUERY, - dynamodb.Operation.SCAN, - dynamodb.Operation.PUT_ITEM, - dynamodb.Operation.UPDATE_ITEM, - dynamodb.Operation.DELETE_ITEM, - dynamodb.Operation.BATCH_WRITE_ITEM, -] - - -def _add_ddb_alarms(scope, id_prefix, table, alarm_name_prefix, alarm_topic): - """Add throttle + system-error alarms for a DynamoDB table. - - Both fire on any non-zero datapoint in a 5-min window. ALARM-only SnsAction - to site-alerts (no OK action); TreatMissingData NOT_BREACHING. - """ - table.metric_throttled_requests_for_operations( - operations=_DDB_ALARM_OPERATIONS, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}ThrottlesAlarm", - alarm_name=f"{alarm_name_prefix}-throttles", - alarm_description=f"{alarm_name_prefix} DynamoDB throttled requests", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - table.metric_system_errors_for_operations( - operations=_DDB_ALARM_OPERATIONS, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}SystemErrorsAlarm", - alarm_name=f"{alarm_name_prefix}-system-errors", - alarm_description=f"{alarm_name_prefix} DynamoDB server-side (5xx) errors", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - -# CloudWatch namespace for the log-derived sender-authentication metrics. -_SENDER_AUTH_METRIC_NAMESPACE = "Seahaven/ProcurementIngest" - - -def _add_sender_auth_rejected_alarm(scope, id_prefix, function_name, alarm_topic): - """Metric-filter + alarm on ``sender_auth_rejected`` warnings (INFRA-107). - - A rejected inbound email is skipped without erroring the invocation, so it - is invisible to the Errors/Throttles/DLQ alarms. This turns the structured - warning log into a CloudWatch metric and pages when rejections spike -- - catching a silent false-reject storm (allowlist wrong, signing-domain - drift, SES header-format change) that would otherwise discard legitimate - mail while the pipeline reports healthy. This is the safety net for the WO - allowlist domain assumption (seahaven.com) -- if the real Gmail-forward - re-signing domain differs, this alarm surfaces it instead of a silent - work-order outage. - - ALARM-only SnsAction to site-alerts; no OK action. The metric filter reads - the function's own log group (imported by the deterministic - ``/aws/lambda/`` name, created by the function's log_retention). A plain - substring pattern is used because Lambda prefixes each line with its own - level/timestamp/request-id, so the JSON payload is not a standalone JSON - log event a `{$.event=...}` pattern could match. - """ - metric_name = f"{function_name}-sender-auth-rejected" - logs.MetricFilter( - scope, - f"{id_prefix}SenderAuthRejectedFilter", - log_group=logs.LogGroup.from_log_group_name( - scope, - f"{id_prefix}LogGroup", - f"/aws/lambda/{function_name}", - ), - filter_pattern=logs.FilterPattern.literal('"sender_auth_rejected"'), - metric_namespace=_SENDER_AUTH_METRIC_NAMESPACE, - metric_name=metric_name, - metric_value="1", - default_value=0, - ) - - # Fire on a *sustained* reject condition rather than a volume spike. The - # earlier Sum>=3-over-15-min threshold had a blind spot that is exactly the - # failure this alarm exists to catch: a low-traffic pipeline in total - # drift outage (allowlist wrong / signing-domain changed) may only produce - # a trickle of rejects -- one every few minutes -- that never sums to 3 in - # any window, so the outage never pages. Instead: >=1 reject per 5-min - # period, alarming when 2 of the last 6 periods breach (evaluation_periods=6 - # / datapoints_to_alarm=2). Six periods (30 min) with only 2 required - # datapoints closes the sparse-outage residual: even rejections >10-15 min - # apart can still place two breaching datapoints in a single 30-min - # evaluation window. A single stray spoof probe (one lone period) is - # tolerated and self-clears, but a sustained reject condition trips even - # at very low arrival rates. default_value=0 on the metric filter keeps the - # series continuous so NOT_BREACHING only applies before the first datapoint - # ever arrives. - cloudwatch.Metric( - namespace=_SENDER_AUTH_METRIC_NAMESPACE, - metric_name=metric_name, - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - scope, - f"{id_prefix}SenderAuthRejectedAlarm", - alarm_name=f"{function_name}-sender-auth-rejected", - alarm_description=( - f"{function_name} rejected inbound mail on sender authentication " - "(possible allowlist/DKIM-domain drift silently dropping real mail)" - ), - threshold=1, - evaluation_periods=6, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) +import common class WorkorderIngestStack(Stack): @@ -167,15 +38,8 @@ class WorkorderIngestStack(Stack): ) # --- S3 bucket for raw emails --- - email_bucket = s3.Bucket( - self, - "EmailBucket", - bucket_name=f"workorder-ingest-emails-{self.account}", - block_public_access=s3.BlockPublicAccess.BLOCK_ALL, - removal_policy=RemovalPolicy.RETAIN, - lifecycle_rules=[ - s3.LifecycleRule(expiration=Duration.days(90)), - ], + email_bucket = common.make_email_bucket( + self, "EmailBucket", "workorder-ingest-emails" ) # --- DynamoDB tables --- @@ -222,12 +86,7 @@ class WorkorderIngestStack(Stack): # parse (bad email, transient error) is silently dropped after Lambda's # retries. CDK generates the queue name to avoid colliding with the # interim CLI-created workorder-email-processor-dlq (removed post-deploy). - email_processor_dlq = sqs.Queue( - self, - "EmailProcessorDlq", - retention_period=Duration.days(14), - enforce_ssl=True, - ) + email_processor_dlq = common.make_processor_dlq(self, "EmailProcessorDlq") # --- Lambda function --- email_processor = lambda_.Function( @@ -290,20 +149,7 @@ class WorkorderIngestStack(Stack): # (us-east-1/us-east-2/us-west-2). A profile-only grant AccessDenies at # runtime whenever the profile routes to a region whose foundation-model # ARN is not allowed. - email_processor.add_to_role_policy( - iam.PolicyStatement( - actions=[ - "bedrock:InvokeModel", - "bedrock:InvokeModelWithResponseStream", - ], - resources=[ - "arn:aws:bedrock:us-east-1:328440206208:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-east-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - "arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0", - ], - ) - ) + email_processor.add_to_role_policy(common.make_bedrock_invoke_statement(self)) # NOTE: The pre-emptive grant_encrypt_decrypt on the shared DynamoDB CMK # (alias/seahaven-dynamodb) was removed (security sweep 2026-06-17). The @@ -315,22 +161,28 @@ class WorkorderIngestStack(Stack): # point grant_read_write_data on the (then encrypted) tables would propagate # the needed key permissions automatically. - # --- Errors alarm (INFRA-41 / audit H-8) --- - # ALARM-only (no OK action, per the CloudWatch-alarm preference) to the - # shared site-alerts topic. Any errored invocation in a 5-min window pages. - email_processor.metric_errors( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( + # --- Standard per-Lambda alarms: workorder-email-processor --- + # errors (INFRA-41 / audit H-8), throttles, DLQ-visible-messages (dropped + # emails), and a p95 duration alarm (orphan adoption of the CLI + # Lambda-Duration-workorder-email-processor under -duration naming, + # 45000 ms = 75% of the 60s timeout, eval 3 / dp 2). p95 (NOT p99) is the + # WO-specific duration statistic. All ALARM-only to site-alerts. + common.add_standard_lambda_alarms( self, - "EmailProcessorErrorsAlarm", - alarm_name="workorder-email-processor-errors", - alarm_description="workorder-email-processor async invocation errors", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + "EmailProcessor", + email_processor, + "workorder-email-processor", + alarm_topic, + duration_statistic="p95", + errors=True, + dlq=email_processor_dlq, + descriptions={ + "errors": "workorder-email-processor async invocation errors", + "throttles": "workorder-email-processor invocation throttles", + "dlq": "workorder-email-processor DLQ has visible messages (dropped emails)", + "duration": "workorder-email-processor p95 duration approaching the 60s timeout", + }, + ) # --- Sender-auth rejection alarm (INFRA-107) --- # A rejected email (bad/unaligned DKIM verdict) returns normally, so it @@ -341,68 +193,10 @@ class WorkorderIngestStack(Stack): # different domain), 100% of legitimate work-order mail is silently # dropped. This metric filter + alarm turns those warnings into a paging # signal so a false-reject storm surfaces instead of a silent outage. - _add_sender_auth_rejected_alarm( + common.add_sender_auth_rejected_alarm( self, "EmailProcessor", "workorder-email-processor", alarm_topic ) - # --- Throttles alarm: workorder-email-processor --- - # Any throttled invocation (concurrency cap hit) in a 5-min window pages. - # ALARM-only to site-alerts; no OK action; NOT_BREACHING when no data. - email_processor.metric_throttles( - period=Duration.minutes(5), - statistic="Sum", - ).create_alarm( - self, - "EmailProcessorThrottlesAlarm", - alarm_name="workorder-email-processor-throttles", - alarm_description="workorder-email-processor invocation throttles", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- Duration alarm: workorder-email-processor (orphan adoption) --- - # Adopts the orphaned CLI alarm Lambda-Duration-workorder-email-processor - # under the repo's -duration naming (NEW logical name → no deploy - # collision; delete the orphan post-deploy). p95 / - # 45000 ms (75% of the 60s timeout) / eval 3 of which 2 datapoints — - # tighter than the orphan's Maximum>=48000 / 1-of-1. - email_processor.metric_duration( - period=Duration.minutes(5), - statistic="p95", - ).create_alarm( - self, - "EmailProcessorDurationAlarm", - alarm_name="workorder-email-processor-duration", - alarm_description="workorder-email-processor p95 duration approaching the 60s timeout", - threshold=45000, - evaluation_periods=3, - datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - - # --- DLQ messages-present alarm (INFRA-41 / audit H-8) --- - # The errors alarm above fires on any errored invocation, but a message - # only lands in the DLQ after Lambda exhausts its async retries and gives - # up — i.e. a genuinely dropped email. ALARM-only (no OK action) to the - # same shared site-alerts topic. MAXIMUM over a 5-min window so a single - # visible message pages even if it is later consumed/redriven. - email_processor_dlq.metric_approximate_number_of_messages_visible( - period=Duration.minutes(5), - statistic="Maximum", - ).create_alarm( - self, - "EmailProcessorDlqMessagesAlarm", - alarm_name="workorder-email-processor-dlq-messages", - alarm_description="workorder-email-processor DLQ has visible messages (dropped emails)", - threshold=0, - evaluation_periods=1, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) - # --- Template fallback-rate alarm: workorder-email-processor --- # The processor tries a deterministic template parse first and only calls # the Bedrock AI extractor on a miss/invalid. A sustained rise in the @@ -413,64 +207,28 @@ class WorkorderIngestStack(Stack): # ~760/day volume; FILL(0) + a >=10-sample volume floor prevent # low-volume false pages and INSUFFICIENT_DATA. ALARM-only SnsAction to # site-alerts, no OK action, NOT_BREACHING -- matching the stack idiom. - fb_metric = cloudwatch.Metric( - namespace="Seahaven/WorkorderIngest", - metric_name="ParseOutcome", - dimensions_map={"ParseMethod": "ai_fallback"}, - statistic="Sum", - period=Duration.minutes(15), - ) - # AI-fallback parses REJECTED by the validate_ai_fallback gate emit - # ParseMethod=ai_fallback_rejected (and nothing else), so they must - # count as fallback here too -- otherwise a drift outage whose AI - # output also fails the gate would LOWER the observed fallback rate - # while silently dropping mail. - fb_rej_metric = cloudwatch.Metric( - namespace="Seahaven/WorkorderIngest", - metric_name="ParseOutcome", - dimensions_map={"ParseMethod": "ai_fallback_rejected"}, - statistic="Sum", - period=Duration.minutes(15), - ) - tmpl_metric = cloudwatch.Metric( - namespace="Seahaven/WorkorderIngest", - metric_name="ParseOutcome", - dimensions_map={"ParseMethod": "template"}, - statistic="Sum", - period=Duration.minutes(15), - ) - fallback_rate = cloudwatch.MathExpression( - expression=( - # The IF volume floor (>=10) already guarantees the denominator - # is non-zero in the true branch, so divide directly. (An earlier - # MAX([...,1]) divide-by-zero guard used array syntax CloudWatch - # rejects at deploy: "Unsupported operand type(s) for MAX".) - "IF((FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0))>=10, " - "100*(FILL(fb,0)+FILL(rej,0))" - "/(FILL(fb,0)+FILL(rej,0)+FILL(tmpl,0)), 0)" - ), - using_metrics={ - "fb": fb_metric, - "rej": fb_rej_metric, - "tmpl": tmpl_metric, - }, - period=Duration.minutes(15), - label="TemplateFallbackRatePct", - ) - fallback_rate.create_alarm( + # rejected_included=True: the WO expression folds ai_fallback_rejected + # (rej) into BOTH numerator and denominator -- a drift outage whose AI + # output also fails the gate must still count as fallback, otherwise it + # would LOWER the observed rate while silently dropping mail. (Contrast + # PO, which excludes rej to avoid a pre-call double-count.) + common.make_fallback_rate_alarm( self, "EmailProcessorTemplateFallbackRateAlarm", + namespace="Seahaven/WorkorderIngest", + alarm_topic=alarm_topic, alarm_name="workorder-email-processor-template-fallback-rate", alarm_description=( "workorder-email-processor deterministic-template coverage " "collapse: >15% of parses fell back to the Bedrock AI extractor" ), + rejected_included=True, + period=Duration.minutes(15), threshold=15, + floor=10, evaluation_periods=3, datapoints_to_alarm=2, - comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD, - treat_missing_data=cloudwatch.TreatMissingData.NOT_BREACHING, - ).add_alarm_action(cw_actions.SnsAction(alarm_topic)) + ) # --- AI-fallback rejected alarm: workorder-email-processor --- # A parse rejected by the validate_ai_fallback gate is dropped without @@ -602,17 +360,43 @@ class WorkorderIngestStack(Stack): # --- DynamoDB throttle + system-error alarms --- # ThrottledRequests / SystemErrors emit at TableName + Operation only # (verified against live CloudWatch: no TableName-only rollup exists, and - # metric_throttled_requests is deprecated/invalid in aws-cdk-lib 2.259.0). + # metric_throttled_requests is deprecated/invalid in aws-cdk-lib 2.261.0). # Each table currently has zero throttle/error datapoints, so the series # only materialise on first occurrence — NOT_BREACHING keeps them OK until # then. - _add_ddb_alarms( + common.add_ddb_alarms( self, "WorkOrdersTable", work_orders_table, "WorkOrders", alarm_topic ) - _add_ddb_alarms( + common.add_ddb_alarms( self, "WorkOrderComments", comments_table, "WorkOrderComments", alarm_topic ) + # --- Function ARN + consumed-table-name outputs (Phase 4, additive) --- + cdk.CfnOutput( + self, + "EmailProcessorFunctionArn", + value=email_processor.function_arn, + description="ARN of the workorder-email-processor Lambda", + ) + cdk.CfnOutput( + self, + "WebUiFunctionArn", + value=web_ui.function_arn, + description="ARN of the workorder-web-ui Lambda", + ) + cdk.CfnOutput( + self, + "WorkOrdersTableName", + value=work_orders_table.table_name, + description="WorkOrders DynamoDB table", + ) + cdk.CfnOutput( + self, + "WorkOrderCommentsTableName", + value=comments_table.table_name, + description="WorkOrderComments DynamoDB table", + ) + # Public Function URL removed 2026-06-08 (INFRA-74 / audit C-5): the # unauthenticated FunctionUrlAuthType.NONE URL was deleted out-of-band # via CLI. Removing the construct (and its auto-generated Principal:*