From df03f3497fc30b934e41af0997d374e96e4709a5 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:47:01 -0400 Subject: [PATCH] feat: widen email-processor asset roots to lambdas/ with scoped globs + excludes (refactor phase 2) (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both email-processor Code.from_asset calls now bundle from lambdas/ instead of their per-function subdirectory, so Phase 3's shared/ module is reachable from the asset root once it lands. The bundling commands were rewritten for the new cwd (pip install -r / email_processor/requirements.txt -t /asset-output && cp / email_processor/*.py /asset-output/), preserving the ARM64 --platform manylinux2014_aarch64 --only-binary=:all: pin exactly — its removal shipped x86 wheels into the ARM64 function and caused a 100% outage (PR #34). All five from_asset calls (both email processors, po web_ui, po site_extractor, wo web_ui) now exclude **/__pycache__/**; the two widened ones also exclude **/tests/** and **/package/**. Without the package/ exclude, the stale untracked 44 MB lambdas/po/email_processor/package/ dir (local-only, never present in CI) would diverge local vs CI asset hashes and force spurious redeploys — from_asset doesn't honor .gitignore. That dir is left in place; deleting it is Adam's call. WO's prod zip shrinks as deliberate cleanup, not a byte-identical match to PO: the old `cp -r .` shipped tests/ (real scrubbed .eml fixtures), __pycache__/, and requirements.txt into production. The acceptance bar for WO is runtime-imported module set unchanged + smoke, not a byte-identical zip; PO keeps the byte-identical first-party file set guarantee. tests/test_bundle_consistency.py is updated in the same change to recognize the scoped `cp po/email_processor/*.py` (resp. wo) glob as the new unconditionally-safe shape, without loosening the allowlist-revert detection, the detection-logic mutation test, or the PO_EXPECTED_TOP_LEVEL_MODULES exact-set pin. No code moved under lambdas/ in this change (git diff main...HEAD -- lambdas/ is empty); only CDK asset wiring and its tests changed. --- .claude/workflows/phase-2-bundling-root.js | 482 +++++++++++++++++++++ README.md | 25 +- cdk/po_stack.py | 15 +- cdk/wo_stack.py | 11 +- tests/test_bundle_consistency.py | 151 ++++++- 5 files changed, 652 insertions(+), 32 deletions(-) create mode 100644 .claude/workflows/phase-2-bundling-root.js diff --git a/.claude/workflows/phase-2-bundling-root.js b/.claude/workflows/phase-2-bundling-root.js new file mode 100644 index 0000000..acc13cc --- /dev/null +++ b/.claude/workflows/phase-2-bundling-root.js @@ -0,0 +1,482 @@ +export const meta = { + name: 'phase-2-bundling-root', + description: 'Phase 2 of the procurement-ingest refactor (docs/refactor-evaluation.md): widen both email-processor asset roots to ../lambdas (making lambdas/shared/ reachable for Phase 3) with scoped cp globs, add exclude lists (from_asset ignores .gitignore — the stale 44 MB package/ dir would poison asset hashes), and add __pycache__ excludes to the three plain from_asset calls. ZERO code change under lambdas/ — the workflow proves bundle parity by diffing the locally-synthed staged asset against the currently-deployed zip. Committed locally, never pushed.', + phases: [ + { title: 'Setup', detail: 'verify Phases 0+1 merged into base, branch feature/phase-2-bundling-root', model: 'haiku' }, + { title: 'Recon', detail: '3 mappers: all five from_asset sites, deployed zip file lists (read-only AWS), AST-test shapes' }, + { title: 'Spec', detail: 'serial opus spec: exact new bundling commands, exclude lists, and the parity-comparison rules', model: 'opus' }, + { title: 'Implement', detail: 'sonnet per stack file + sonnet for AST-test/README updates — disjoint files', model: 'sonnet' }, + { title: 'Verify', detail: 'mechanical gates + staged-vs-deployed parity + determinism + 2 fable lenses' }, + { title: 'Fix', detail: 'opus fixer, full re-verify, max 3 rounds', model: 'opus' }, + { title: 'Package', detail: 'single commit via -F (no push)', model: 'sonnet' }, + ], +} + +// ---------------------------------------------------------------- constants + +const REPO = '/Users/adammoussa/Documents/repositories/seahaven/procurement-ingest' +const BRANCH = 'feature/phase-2-bundling-root' +const BASE = (args && args.base) || 'main' + +const CONSTRAINTS = ` +PINNED BEHAVIORAL CONSTRAINTS (docs/refactor-evaluation.md Phase 2 — violating any is a build failure): +1. ZERO diff under lambdas/: git diff ${BASE}...HEAD -- lambdas/ must be + EMPTY. This phase moves NO code — only cdk/, tests/test_bundle_consistency.py, + README.md (and docs/ if needed) may change. +2. Both email processors: Code.from_asset("../lambdas") with the bundling + command rewritten for the new cwd (bundling cwd IS the asset root): + pip install ... -r po/email_processor/requirements.txt -t /asset-output && + cp po/email_processor/*.py /asset-output/ (resp. wo/email_processor). + The -r path change is REQUIRED. PRESERVE the pip + "--platform manylinux2014_aarch64 --only-binary=:all:" pin exactly — its + removal shipped x86 wheels into the ARM64 function and caused a 100% + outage (PR #34); it is non-negotiable. +3. Both widened from_asset calls get + exclude=['**/__pycache__/**', '**/tests/**', '**/package/**']. + from_asset does NOT honor .gitignore; without the package/ exclude the + stale untracked 44 MB lambdas/po/email_processor/package/ dir diverges + LOCAL vs CI asset hashes (CI never has it) and forces spurious redeploys. +4. WO prod-zip shrinkage is DELIBERATE cleanup, not an accident to fix: the + current 'cp -r .' ships tests/ (real scrubbed .eml fixtures), __pycache__, + and requirements.txt in the production zip. The acceptance criterion for + WO is "runtime-imported module set unchanged + smoke", NOT byte-identical + zip. Byte-identical first-party file set applies to PO ONLY. Document the + shrinkage explicitly (list every file class that drops out). +5. The three plain from_asset calls (po web_ui, po site_extractor, wo web_ui + — locate them, line numbers have shifted since the doc) each gain + exclude=['**/__pycache__/**'] so local __pycache__ stops making their + asset hashes nondeterministic. Nothing else about them changes. +6. tests/test_bundle_consistency.py must be updated IN THE SAME CHANGE — it + deliberately change-detects both bundling commands and will fail red on + the new shapes. Update it to recognize the scoped glob + 'cp po/email_processor/*.py' (resp. wo) as the new unconditionally-safe + shape, WITHOUT loosening it: the allowlist-revert detection, the + detection-logic mutation test, and the PO_EXPECTED_TOP_LEVEL_MODULES + exact-set pin must all keep their teeth. A bare-substring match that any + commented-out glob satisfies is a regression. +7. cdk diff on BOTH stacks must show ONLY the two functions' Code/asset + property changes (+ CDK metadata). No logical-ID changes, no alarm, IAM, + table, env, or function-config deltas of any kind. +8. Do NOT delete lambdas/po/email_processor/package/ (untracked, local-only; + deletion is Adam's call — with the exclude in place it is hash-neutral). + Do NOT touch requirements.txt contents (Phase 7 scope). +9. AWS access is READ-ONLY (get-function, downloading the deployed zip via + its presigned URL). NEVER cdk deploy, never invoke, never mutate. +` + +const PREAMBLE = ` +You are one of several agents building refactor Phase 2 in the git repo at +${REPO} on branch ${BRANCH} (already checked out — do NOT switch branches, +do NOT create branches, do NOT commit, NEVER push). +Authoritative spec: docs/refactor-evaluation.md, section "Phase 2". +Work ONLY in the files you are told you own; other agents are concurrently +editing other files in this same working tree. +${CONSTRAINTS} +Your final message is consumed by an orchestrator script, not a human — +return only the structured data requested. +` + +// ------------------------------------------------------------------ schemas + +const RECON = { + type: 'object', + required: ['summary', 'facts'], + properties: { + summary: { type: 'string' }, + facts: { type: 'array', items: { type: 'string' } }, + blockers: { type: 'array', items: { type: 'string' } }, + }, +} + +const SPEC = { + type: 'object', + required: ['poBundling', 'woBundling', 'plainAssetEdits', 'astTestChanges', 'parityRules', 'notes'], + properties: { + poBundling: { type: 'string', description: 'the complete new po_stack.py from_asset block: asset path, full command string, exclude list — exact code' }, + woBundling: { type: 'string', description: 'same for wo_stack.py' }, + plainAssetEdits: { type: 'string', description: 'the three plain from_asset calls: current file:line + exact exclude addition each' }, + astTestChanges: { type: 'string', description: 'exactly how test_bundle_consistency.py recognizes the new scoped-glob shapes without losing revert detection; which assertions/regexes change and to what' }, + parityRules: { type: 'string', description: 'the staged-asset vs deployed-zip comparison rules: strict first-party .py set equality for PO; WO expected-shrinkage list + runtime-module retention; how dependency version drift is tolerated' }, + notes: { type: 'string' }, + }, +} + +const IMPL = { + type: 'object', + required: ['filesChanged', 'summary', 'checksRun'], + properties: { + filesChanged: { type: 'array', items: { type: 'string' } }, + summary: { type: 'string' }, + checksRun: { type: 'string' }, + blockers: { type: 'array', items: { type: 'string' } }, + }, +} + +const CHECKS = { + type: 'object', + required: ['passed', 'details'], + properties: { + passed: { type: 'boolean' }, + details: { type: 'string' }, + scopeViolations: { type: 'array', items: { type: 'string' } }, + }, +} + +const PARITY = { + type: 'object', + required: ['passed', 'poVerdict', 'woVerdict', 'details'], + properties: { + passed: { type: 'boolean' }, + poVerdict: { type: 'string', description: 'PO staged vs deployed: identical first-party set? full evidence' }, + woVerdict: { type: 'string', description: 'WO: runtime modules retained + exact shrinkage list' }, + determinism: { type: 'string', description: 'repeat-synth + injected-__pycache__ hash results' }, + details: { type: 'string' }, + }, +} + +const FINDINGS = { + type: 'object', + required: ['findings'], + properties: { + findings: { + type: 'array', + items: { + type: 'object', + required: ['title', 'severity', 'confirmed', 'evidence', 'fix'], + properties: { + title: { type: 'string' }, + severity: { enum: ['critical', 'high', 'medium', 'low'] }, + confirmed: { type: 'boolean' }, + evidence: { type: 'string' }, + fix: { type: 'string' }, + }, + }, + }, + }, +} + +// ------------------------------------------------------------------- setup + +phase('Setup') +const setup = await agent(` +In ${REPO}: +1. SEQUENCING GATE — Phases 0 AND 1 must be merged into ${BASE} (Phase 1 + also edits cdk/po_stack.py; building Phase 2 against a pre-Phase-1 stack + file guarantees a conflict). git fetch origin, then verify on + origin/${BASE}: (a) the healthcheck branch exists in both handlers and + tests/test_bundle_consistency.py exists (Phase 0); (b) validate_ai_fallback + exists in the PO pipeline and po_stack.py contains the + ai-fallback-rejected alarm (Phase 1). If either is missing, STOP with a + blocker naming the unmerged phase and do nothing else. +2. Verify clean tree (untracked .coverage/.claude/ fine; other dirt = blocker, + never stash or discard). +3. git checkout ${BASE} && git pull --ff-only && git checkout -b ${BRANCH} +4. gh pr list --state open --json number,title,headRefName +Return: HEAD sha, gate evidence, open PRs, blockers. +`, { label: 'setup:branch', model: 'haiku', schema: RECON }) + +if (!setup || (setup.blockers && setup.blockers.length)) { + return { aborted: 'setup blockers', blockers: setup ? setup.blockers : ['setup agent died'], facts: setup ? setup.facts : [] } +} +log(`Branch ${BRANCH} ready off ${BASE}. ${setup.summary}`) + +// ------------------------------------------------------------------- recon + +phase('Recon') +const recon = await parallel([ + () => agent(`${PREAMBLE} +Read-only recon of ALL FIVE Code.from_asset sites in cdk/po_stack.py and +cdk/wo_stack.py: for each, quote verbatim with current file:line — the asset +path, full bundling command (if bundled) or plain form, and any existing +exclude. Also: both email_processor requirements.txt paths + contents (the +pip -r path must be rewritten); the exact set of top-level .py files in +lambdas/po/email_processor and lambdas/wo/email_processor (non-recursive); +every subdirectory of each (tests/, package/, __pycache__ presence); and +whether lambdas/ contains anything else a widened ../lambdas asset root +would stage (shared/ existing yet? stray files at lambdas/ top level?). +15-25 precise facts.`, + { label: 'recon:asset-sites', model: 'sonnet', phase: 'Recon', schema: RECON }), + + () => agent(`${PREAMBLE} +Read-only AWS recon (region us-east-1, READ-ONLY): for po-email-processor +and workorder-email-processor run aws lambda get-function, download each +Code.Location presigned zip to a scratch dir, and produce the COMPLETE file +list of each deployed zip (unzip -l), separating (a) first-party top-level +.py files, (b) pip-installed dependency dirs (name + version from +.dist-info), (c) everything else (tests/, fixtures, __pycache__, +requirements.txt — expected in the WO zip today). Also record each +function's CodeSha256. This is the parity baseline the new bundles are +judged against. Return the categorized lists as facts.`, + { label: 'recon:deployed-zips', model: 'sonnet', phase: 'Recon', schema: RECON }), + + () => agent(`${PREAMBLE} +Read-only recon of tests/test_bundle_consistency.py: quote the exact +regexes/assertions that will change — the PO executed-glob pin, the WO +recursive-copy pin, _bundling_ships_all's two unconditionally-safe shapes, +_extract_bundling_command's exactly-one-command demand (both stacks still +have exactly one bundled function after Phase 2 — confirm), the +PO_EXPECTED_TOP_LEVEL_MODULES pin, and the mutation test. State which +assertions fail red against the Phase 2 command shapes (expected: PO glob +pin and WO cp -r pin both fail). 8-15 facts.`, + { label: 'recon:ast-test', model: 'haiku', phase: 'Recon', schema: RECON }), +]) + +const reconOk = recon.filter(Boolean) +const pack = reconOk.map(r => `## ${r.summary}\n${r.facts.join('\n')}`).join('\n\n') +const reconBlockers = reconOk.flatMap(r => r.blockers || []) +log(`Recon complete: ${reconOk.length}/3 mappers, ${reconBlockers.length} blockers`) + +// -------------------------------------------------------------------- spec + +phase('Spec') +const spec = await agent(`${PREAMBLE} +You are the SPEC agent — pin every exact string before parallel +implementation. Using the recon pack and your own reads, produce: +- poBundling / woBundling: the complete replacement from_asset blocks as + exact code — asset path "../lambdas", full bash -c command (pip line with + rewritten -r path and the preserved manylinux2014_aarch64 pin, then the + scoped non-recursive glob cp), exclude list per constraint 3. Mind the + bundling cwd semantics: the container mounts the ASSET ROOT (../lambdas) + as the working dir, so all paths are relative to lambdas/. +- plainAssetEdits: the three plain from_asset calls with exact exclude + additions. +- astTestChanges: precise edits to test_bundle_consistency.py — the new + scoped-glob regex (must match 'cp po/email_processor/*.py /asset-output/' + as executed, still comment-strip-proof, still reject narrowed or + commented-out variants), what replaces the WO cp -r pin, and confirmation + the mutation test + exact-set pin survive unchanged in spirit. +- parityRules: exactly how Verify judges the new bundles against the + deployed-zip baseline from recon: PO = first-party top-level .py set must + be IDENTICAL; dependency packages compared by NAME (version drift from the + floor-pinned boto3 is tolerated and noted, not failed); nothing new may + appear. WO = every first-party module in the CURRENT deployed zip that the + handler imports (cross-check the AST sibling list) must be retained; + expected drop list = tests/ + fixtures + __pycache__ + requirements.txt + and NOTHING else may silently vanish; nothing new may appear. +Recon pack:\n${pack}`, + { label: 'spec:pin-strings', model: 'opus', phase: 'Spec', schema: SPEC }) + +if (!spec) return { aborted: 'spec agent died — rerun workflow', reconBlockers } +const specBlock = `BINDING SPEC (implement EXACTLY this):\n${JSON.stringify(spec, null, 2)}` +log('Spec pinned: bundling commands, excludes, AST-test edits, parity rules') + +// --------------------------------------------------------------- implement + +phase('Implement') +const impl = await parallel([ + () => agent(`${PREAMBLE} +YOU OWN: cdk/po_stack.py ONLY. Apply spec.poBundling (email processor +from_asset: root, command, exclude) and the po web_ui + site_extractor +exclude additions from spec.plainAssetEdits. Touch NOTHING else in the file +(alarms, IAM, tables are all off-limits). Preserve/adapt the bundling +warning comment so it stays true. +Run before returning: ruff check cdk && cd cdk && npx cdk synth po-ingest -q +(this runs Docker bundling — confirm the staged asset in cdk.out contains +exactly the expected files and note the asset hash in your summary). +${specBlock}`, + { label: 'impl:po-stack', model: 'sonnet', phase: 'Implement', schema: IMPL }), + + () => agent(`${PREAMBLE} +YOU OWN: cdk/wo_stack.py ONLY. Apply spec.woBundling and the wo web_ui +exclude from spec.plainAssetEdits. Touch nothing else. +Run before returning: ruff check cdk && cd cdk && npx cdk synth +workorder-ingest -q (artifact-id selector, NOT stack_name) — confirm staged +asset contents + hash in your summary. +${specBlock}`, + { label: 'impl:wo-stack', model: 'sonnet', phase: 'Implement', schema: IMPL }), + + () => agent(`${PREAMBLE} +YOU OWN: tests/test_bundle_consistency.py and README.md ONLY. +Task A: apply spec.astTestChanges. The test must PASS against the new stack +files and still FAIL against: a reverted filename allowlist missing a +sibling, a commented-out glob, and a narrowed glob (add/adjust the mutation +tests to cover the new shapes — e.g. 'cp po/email_processor/handler.py' +alone must not satisfy the scoped-glob pin). +Task B: README — update the Deploy-Pipeline Guards bundling paragraph and +the CI/CD + repo-layout sections for the widened ../lambdas asset root; +document the deliberate WO prod-zip shrinkage (what dropped and why that is +cleanup, per constraint 4) and the exclude rationale (from_asset ignores +.gitignore; package/ hash poisoning). +Run before returning: pytest tests/test_bundle_consistency.py -q --no-cov +(must be green against the OTHER agents' stack edits — if they have not +landed yet, poll by re-running up to ~10 min before reporting a blocker), +ruff check tests. +${specBlock}`, + { label: 'impl:ast-test-readme', model: 'sonnet', phase: 'Implement', schema: IMPL }), +]) + +const implOk = impl.filter(Boolean) +const implBlockers = implOk.flatMap(r => r.blockers || []) +log(`Implement complete: ${implOk.length}/3 agents, blockers: ${implBlockers.length}`) + +// ---------------------------------------------------- verify + fix loop + +const EXPECTED_SCOPE = [ + 'cdk/po_stack.py', + 'cdk/wo_stack.py', + 'tests/test_bundle_consistency.py', + 'README.md', +] + +const mechanicalPrompt = `${PREAMBLE} +Independent re-verification — trust nothing self-reported. Run ALL gates, +quoting failures verbatim: +1. pytest -q --no-cov (repo root) +2. ruff check . && ruff format --check . +3. cd cdk && npx cdk synth po-ingest -q && npx cdk synth workorder-ingest -q +4. cd cdk && npx cdk diff po-ingest ; npx cdk diff workorder-ingest — + the ONLY resource deltas allowed are the two email-processor functions' + Code/S3Key (asset hash) changes + CDK metadata. Any alarm/IAM/env/config/ + logical-ID delta = FAIL (constraint 7). Paste the diff summaries. +5. ZERO-CODE-MOVE invariant: git diff ${BASE}...HEAD -- lambdas/ must output + NOTHING. +6. git status scope: every change under ${EXPECTED_SCOPE.join(', ')} only + (untracked .coverage/.claude/ tolerated). +passed=true only if all green. YOU MAY NOT edit files.` + +const parityPrompt = `${PREAMBLE} +You are the ARTIFACT-PARITY verifier — the load-bearing gate of this phase. +Everything is local synth + read-only AWS. +1. cd cdk && npx cdk synth po-ingest -q -o /tmp/phase2-synth && npx cdk + synth workorder-ingest -q -o /tmp/phase2-synth. Locate each email + processor's staged bundled asset dir in /tmp/phase2-synth (the asset.* + dir containing handler.py). +2. Re-download both deployed zips (aws lambda get-function Code.Location, + region us-east-1) fresh — do not trust a recon cache. +3. Judge per the spec parityRules: PO first-party top-level .py set staged + vs deployed IDENTICAL (list both sets); dependency packages by name with + version drift noted; NOTHING new. WO: every AST-derived handler sibling + retained; the drop list is EXACTLY tests//fixtures/__pycache__/ + requirements.txt-class files — enumerate every dropped path class and + every added path; anything outside the expected classes = FAIL. +4. DETERMINISM: run the po-ingest synth twice into two fresh -o dirs — + asset hashes must be identical. Then create a throwaway + __pycache__/junk.pyc under lambdas/po/web_ui/ AND a dummy file under + lambdas/po/email_processor/package/ (create the dir if absent, remember + to DELETE everything you created afterwards), re-synth, and confirm NO + asset hash changed (proves the excludes + the package/ hash-poisoning + fix actually work). Report before/after hashes. +5. Sanity: the staged PO asset must NOT contain web_ui/site_extractor + sources, tests/, package/, or any .eml — the widened root stages more + context but the cp glob + excludes must keep the OUTPUT clean. +passed=true only if every check holds.` + +const lenses = [ + { key: 'bundling-semantics', prompt: `${PREAMBLE} +ADVERSARIAL REVIEW — bundling-semantics lens. Read the full cdk diff +(git diff ${BASE}) plus aws-cdk-lib's documented from_asset/BundlingOptions +semantics and try to REFUTE correctness: (1) is the bundling container cwd +really the widened asset root, making 'pip install -r +po/email_processor/requirements.txt' and 'cp po/email_processor/*.py' +resolve correctly — or does an image workdir default break it? (2) do the +exclude patterns ('**/__pycache__/**' etc.) apply to ASSET STAGING (hash +input) and not merely the docker mount — i.e. does the package/ exclude +actually stop local-vs-CI hash divergence? (3) does exclude affect what the +bundling container can SEE, and could excluding tests/ break the pip +install or cp step? (4) non-recursive scoped glob: could bash glob +expansion inside the container (sh vs bash, nullglob) change behavior vs +the Phase 0 top-level glob? (5) does widening the asset root change the +ASSET HASH INPUTS such that unrelated wo/ file edits now redeploy the PO +function (and vice versa) — if so, state the blast radius plainly so it is +documented, not discovered. confirmed=true only with concrete evidence +(CDK docs/source or a reproduced synth).` }, + { key: 'guard-integrity', prompt: `${PREAMBLE} +ADVERSARIAL REVIEW — guard-integrity lens. The Phase 0 guards must come +through Phase 2 with their teeth intact. (1) Attack the updated +test_bundle_consistency.py: does the new scoped-glob regex accept a +commented-out glob, a narrowed single-file cp, a glob for the WRONG +pipeline dir (cp wo/email_processor/*.py in po_stack), or a cp missing +/asset-output? Does the WO assertion still reject a narrowed recursive +copy? Run the test against hand-mutated command strings to prove each +rejection (scratch copies, not repo edits). (2) Does +PO_EXPECTED_TOP_LEVEL_MODULES still bound what ships (the glob is now +scoped — is the pin still checking the right directory)? (3) The smoke +script + healthcheck are untouched by this diff — confirm zero diff to +scripts/ and lambdas/. (4) README claims about bundling must match the new +reality — no stale Phase 0 text contradicting the widened root. +confirmed=true only with file:line evidence.` }, +] + +let round = 0 +let checks = null +let parity = null +let confirmed = [] +while (round < 3) { + phase('Verify') + const results = await parallel([ + () => agent(mechanicalPrompt, { label: `verify:mechanical-r${round}`, model: 'sonnet', phase: 'Verify', schema: CHECKS }), + () => agent(parityPrompt, { label: `verify:parity-r${round}`, model: 'opus', phase: 'Verify', schema: PARITY }), + ...lenses.map(l => () => + agent(l.prompt, { label: `verify:${l.key}-r${round}`, phase: 'Verify', schema: FINDINGS })), + ]) + checks = results[0] + parity = results[1] + confirmed = results.slice(2).filter(Boolean) + .flatMap(r => r.findings || []) + .filter(f => f.confirmed && f.severity !== 'low') + const green = checks && checks.passed && parity && parity.passed + log(`Verify round ${round}: mechanical ${checks && checks.passed ? 'GREEN' : 'RED'}, parity ${parity && parity.passed ? 'GREEN' : 'RED'}, confirmed findings: ${confirmed.length}`) + if (green && confirmed.length === 0) break + + round += 1 + if (round >= 3) break + phase('Fix') + await agent(`${PREAMBLE} +You are the fix agent — you may edit files under: ${EXPECTED_SCOPE.join(', ')}. +Fix EVERY item minimally; the binding spec and 9 pinned constraints hold (a +finding that conflicts with a constraint is reported, not "fixed"). Re-run +the specific failing gate per fix. +MECHANICAL:\n${checks ? checks.details : '(agent died — rerun all)'} +PARITY:\n${parity ? parity.details : '(agent died — rerun all)'} +CONFIRMED FINDINGS:\n${JSON.stringify(confirmed, null, 2)} +${specBlock}`, + { label: `fix:round-${round}`, model: 'opus', phase: 'Fix', schema: IMPL }) +} + +const verifyClean = checks && checks.passed && parity && parity.passed && confirmed.length === 0 +if (!verifyClean) { + return { + status: 'NEEDS ATTENTION — verify not clean after 3 rounds; branch left uncommitted', + branch: BRANCH, + mechanical: checks, + parity, + unresolvedFindings: confirmed, + implBlockers, + reconBlockers, + } +} + +// ----------------------------------------------------------------- package + +phase('Package') +const commit = await agent(`${PREAMBLE.replace('do NOT commit, ', '')} +YOU are the commit agent: +1. Read ~/Documents/repositories/seahaven/engineering-handbook/commit-messages.md. +2. git add only: ${EXPECTED_SCOPE.join(', ')} and + .claude/workflows/phase-2-bundling-root.js. NOT .coverage. Verify staged + set with git status. +3. ONE commit via git commit -F /tmp/phase2-commit-msg.txt (write the message + file first; backticks in -m get eaten by zsh). Suggested subject: + "feat: widen email-processor asset roots to lambdas/ with scoped globs + excludes (refactor phase 2)" + Body: why (shared/ reachability for Phase 3), the WO shrinkage list, the + package/ hash-poisoning exclude, parity evidence one-liner. + NO AI attribution / Co-Authored-By lines. +4. Do NOT push. Return commit sha + shortstat.`, + { label: 'package:commit', model: 'sonnet', phase: 'Package', schema: IMPL }) + +return { + status: 'BUILT — committed locally, NOT pushed', + branch: BRANCH, + base: BASE, + commit: commit ? commit.summary : 'commit agent died — commit manually', + parityEvidence: parity ? { po: parity.poVerdict, wo: parity.woVerdict, determinism: parity.determinism } : null, + implementation: implOk.map(r => r.summary), + filesChanged: implOk.flatMap(r => r.filesChanged), + verifyRounds: round + 1, + blockers: implBlockers.concat(reconBlockers), + outstandingGates: [ + 'push + PR + gh pr checks green (no /sh-security-review required — no untrusted-input/auth/IAM surface; pre-push scanners still run; cross-family review not required — no IAM or handler-signature change)', + 'deploy-then-merge: deploy from branch, then the LIVE gate — aws lambda get-function zip file-list diff (PO first-party set identical; WO shrinkage exactly as documented), smoke green, one real PO + WO email each with ParseMethod=template, alarms green, THEN merge', + 'optional cleanup for Adam (hash-neutral once excludes are deployed): delete the untracked 44 MB lambdas/po/email_processor/package/ dir (doc Phase 7 item, endorsed to do with/after Phase 2)', + ], +} diff --git a/README.md b/README.md index a221309..8fb8937 100644 --- a/README.md +++ b/README.md @@ -172,9 +172,22 @@ This placement is deliberate, not incidental: real mail always arrives as an S3 The script runs `set -euo pipefail` and exits non-zero on any invoke failure, any `FunctionError`, or a payload mismatch on either function, failing the deploy job. -**PO bundling: glob replaces the hand-maintained allowlist.** `cdk/po_stack.py`'s asset bundling command now ships PO's Lambda source with a non-recursive glob, `cp ./*.py /asset-output/`, instead of a hand-maintained list of filenames (`cp handler.py ses_auth.py template_parser.py derived_fields.py /asset-output/`). The glob is functionally identical for today's file set — non-recursive, so `tests/` and other subdirectories are still excluded — but structurally eliminates the failure mode that shipped a broken bundle twice (PR #105 omitted `template_parser.py`; PR #2 nearly omitted `derived_fields.py`): a new sibling module the handler imports now ships automatically instead of requiring someone to remember to add it to the list. WO's bundling (`cdk/wo_stack.py`) already used a recursive `cp -r` of the whole source dir and is unaffected. +**PO bundling: glob replaces the hand-maintained allowlist.** `cdk/po_stack.py`'s asset bundling command ships PO's Lambda source with a non-recursive glob instead of a hand-maintained list of filenames (`cp handler.py ses_auth.py template_parser.py derived_fields.py /asset-output/`). The glob is functionally identical for today's file set — non-recursive, so `tests/` and other subdirectories are still excluded — but structurally eliminates the failure mode that shipped a broken bundle twice (PR #105 omitted `template_parser.py`; PR #2 nearly omitted `derived_fields.py`): a new sibling module the handler imports now ships automatically instead of requiring someone to remember to add it to the list. -`tests/test_bundle_consistency.py` guards both bundling commands with a pure-AST check (no synth, no boto3, no handler import): it parses each handler.py's top-level first-party sibling imports, extracts the bundling `command=[...]` string from the corresponding CDK stack file, and asserts every required sibling module is guaranteed to ship — recognizing the PO glob and the WO recursive copy as unconditionally-safe shapes, and falling back to literal filename matching for any other (allowlist-style) shape. It also pins the PO command to the glob form specifically, so a future revert back to a filename allowlist that omits a sibling fails CI rather than merely relying on the general detection logic. Runs in the existing pytest step, before synth. +**Phase 2: widened asset root, both processors on the glob.** Both `cdk/po_stack.py` and `cdk/wo_stack.py` widen their bundled email-processor's `Code.from_asset` root from the per-pipeline dir (`../lambdas/po/email_processor`, `../lambdas/wo/email_processor`) to the shared parent, `../lambdas` — the prerequisite for the Phase 3 `lambdas/shared/` extraction, which needs a bundling root able to reach a sibling `shared/` package outside either pipeline's own dir (this move ships **zero handler code changes** — `git diff -- lambdas/` is empty for this PR). With bundling present, CDK mounts the asset root as the container's working directory, so both the `pip install -r` path and the `cp` source operand became repo-relative to `lambdas/`: `-r po/email_processor/requirements.txt` (resp. `wo/email_processor/requirements.txt`) and `cp po/email_processor/*.py /asset-output/` (resp. `cp wo/email_processor/*.py /asset-output/`). The pip `--platform manylinux2014_aarch64 --only-binary=:all:` pin — removing it once shipped x86 wheels into the ARM64 function and caused a total outage (PR #34) — is preserved byte-for-byte on both. + +Both bundled `from_asset` calls also gain `exclude=['**/__pycache__/**', '**/tests/**', '**/package/**']`. This is load-bearing, not cosmetic: `Code.from_asset` does not honor `.gitignore`, and widening the root to `../lambdas` means the untracked, 44 MB `lambdas/po/email_processor/package/` dir (a stale vendored dependency tree; deletion is a separate, deliberate call — not part of this change) would otherwise be staged into the *source fingerprint* `from_asset` hashes to decide whether to re-bundle. Because CI never has that local-only directory, an un-excluded root would diverge the local vs. CI asset hash on every synth/deploy and force spurious redeploys; the `**/tests/**` and `__pycache__` excludes keep the hash stable for the same reason. Note the exclude does **not** decouple the two pipelines' asset hashes: `from_asset` hashes with its default `AssetHashType.SOURCE`, so the fingerprint is computed over *all* of `../lambdas` minus only the excluded `__pycache__`/`tests`/`package` paths — PO's and WO's first-party source (both `email_processor` trees, plus the two `web_ui`s and the `site_extractor`) therefore both feed **both** email-processors' hash. Editing any non-excluded file under `lambdas/` changes both email-processors' source fingerprint and redeploys both functions with byte-identical bundles. That coupling is an accepted cost of the shared-root design (the bundling `cp` glob still copies only each pipeline's own `*.py` into the zip); the excludes exist solely to strip local-only/irrelevant cruft that would diverge local vs. CI, not to isolate PO's tree from WO's — which `SOURCE` hashing cannot do here. + +**WO bundling: glob replaces the whole-dir copy — deliberate prod-zip shrinkage.** WO's bundling command changes from a recursive `cp -r . /asset-output/` (the entire `wo/email_processor/` source dir, copied into the deployed zip) to the same scoped, non-recursive glob PO uses: `cp wo/email_processor/*.py /asset-output/`. This intentionally drops from the production zip: +- `requirements.txt` — needed only at bundle time (`pip install -r ...`), never at runtime; +- the entire `tests/` tree (`lambdas/wo/email_processor/tests/`) — real scrubbed `.eml` fixtures, golden JSON, and test modules; +- any first-party `__pycache__/*.pyc` a local `cp -r .` would have picked up (the currently-deployed zip carries none, but the exclude keeps future local builds equally clean). + +This is cleanup, not a regression: none of those file classes are imported at runtime by `handler.handler`, so the acceptance bar for this change on WO is "the runtime-imported module set is unchanged, plus a post-deploy smoke pass" — not a byte-identical zip diff (that stricter bar applies to PO only, whose deployed zip was already this tight before this change). All four first-party top-level `.py` files WO's handler needs — `__init__.py`, `handler.py`, `ses_auth.py`, `template_parser.py` — are still shipped; the glob retains `__init__.py` because it is itself a top-level `.py` file, not a special case requiring a separate copy rule. + +**Plain (non-bundled) `from_asset` calls gain `exclude` too.** The three non-bundled Lambda assets — `po-web-ui`, `po-ingest-site-extractor`, `workorder-web-ui` — each add `exclude=['**/__pycache__/**']`. Nothing else about these three changes: each keeps its own scoped asset path (`../lambdas/po/web_ui`, etc.) rather than widening to `../lambdas`, and none gains bundling. Without the exclude, a developer's local `__pycache__` — again invisible to `from_asset`'s `.gitignore`-blind staging — makes that function's asset hash nondeterministic across machines and forces spurious redeploys. + +`tests/test_bundle_consistency.py` guards all of the above with a pure-AST check (no synth, no boto3, no handler import): it parses each handler.py's top-level first-party sibling imports, extracts the bundling `command=[...]` string from the corresponding CDK stack file, and asserts every required sibling module is guaranteed to ship. It recognizes both the scoped glob (`cp po/email_processor/*.py` / `cp wo/email_processor/*.py`, with or without a path prefix) and a whole-dir recursive copy (`cp -r . /asset-output/`) as unconditionally-safe shapes, and falls back to literal filename matching for any other (allowlist-style) shape. It pins each stack's command to the scoped-glob form specifically — a future revert to a narrowed single-file copy, a commented-out glob, or a filename allowlist missing a sibling all fail CI loudly instead of silently shipping a broken bundle. Runs in the existing pytest step, before synth. ## Security @@ -245,7 +258,7 @@ The canonical map of Sea Haven's AWS infrastructure lives in Confluence. This pr ## CI/CD GitHub Actions with reusable workflows from `Sea-Haven-Industries/.github` (all pinned to a commit SHA of `main`): -- **CI** (`ci.yaml`, PR to `main`): linting + `cdk synth` via `ci-python-sam.yaml` +- **CI** (`ci.yaml`, PR to `main`): linting + `cdk synth` via `ci-python-sam.yaml`. `cdk synth`'s Docker-bundled asset build for `po-email-processor` and `workorder-email-processor` mounts the widened `../lambdas` asset root (Phase 2, see [Deploy-Pipeline Guards](#deploy-pipeline-guards-phase-0)) as build context, not just each function's own subdirectory — the `exclude` list on both `from_asset` calls strips local-only `__pycache__`/`package/` (and `tests/`) cruft from that wider mount's source fingerprint, so CI's asset hash matches a clean local checkout, and each function's scoped `cp` glob copies only its own pipeline's `*.py` into the zip. (The exclude does not, and under `SOURCE` hashing cannot, keep the *other* pipeline's tracked source out of the fingerprint (see the PO bundling note above on `SOURCE` hashing) — but that source is identical in CI and local, so it does not cause hash divergence.) - **CD** (`deploy.yaml`, push to `main`): CDK deploy via `cd-cdk.yaml` (OIDC auth), followed by the synchronous `post-deploy-script: scripts/post-deploy-smoke.sh` healthcheck gate (see [Deploy-Pipeline Guards](#deploy-pipeline-guards-phase-0)) — `cd-cdk.yaml`'s `stack-name` input only accepts one stack, so the smoke script itself enumerates both `po-email-processor` and `workorder-email-processor` - Plus dependency review and PR labeler workflows on every PR @@ -310,7 +323,11 @@ cdk/ app.py # Two stacks: po-ingest + WorkorderIngestStack po_stack.py # Purchase order pipeline resources wo_stack.py # Work order pipeline resources -lambdas/ +lambdas/ # Phase 2: shared Code.from_asset("../lambdas") bundling root for + # BOTH po-email-processor and workorder-email-processor -- each + # bundling command `cp`s only its own po/email_processor/*.py or + # wo/email_processor/*.py subset out; site_extractor and both + # web_ui assets keep their own narrower, non-bundled asset root po/ # PO pipeline Lambdas email_processor/ handler.py # template-first + Bedrock fallback, EMF metric, merge writes, {"healthcheck": true} early-return diff --git a/cdk/po_stack.py b/cdk/po_stack.py index 09e4e17..9946a94 100644 --- a/cdk/po_stack.py +++ b/cdk/po_stack.py @@ -239,14 +239,15 @@ class PoIngestStack(Stack): architecture=lambda_.Architecture.ARM_64, handler="handler.handler", code=lambda_.Code.from_asset( - "../lambdas/po/email_processor", + "../lambdas", + exclude=["**/__pycache__/**", "**/tests/**", "**/package/**"], bundling=cdk.BundlingOptions( image=lambda_.Runtime.PYTHON_3_12.bundling_image, command=[ "bash", "-c", "pip install --platform manylinux2014_aarch64 --only-binary=:all: " - "-r requirements.txt -t /asset-output && " + "-r po/email_processor/requirements.txt -t /asset-output && " # NOTE: non-recursive glob (not `cp -r`) so tests/ and # the stale package/ dir are never shipped -- only # top-level .py siblings of handler.py. This replaces @@ -260,7 +261,7 @@ class PoIngestStack(Stack): # handler.py's first-party imports and asserts this # command ships all of them, so a future revert back # to an allowlist that omits a sibling fails CI. - "cp ./*.py /asset-output/", + "cp po/email_processor/*.py /asset-output/", ], ), ), @@ -599,7 +600,9 @@ class PoIngestStack(Stack): runtime=lambda_.Runtime.PYTHON_3_12, architecture=lambda_.Architecture.ARM_64, handler="handler.handler", - code=lambda_.Code.from_asset("../lambdas/po/web_ui"), + code=lambda_.Code.from_asset( + "../lambdas/po/web_ui", exclude=["**/__pycache__/**"] + ), timeout=Duration.seconds(60), memory_size=256, log_retention=logs.RetentionDays.TWO_MONTHS, @@ -678,7 +681,9 @@ class PoIngestStack(Stack): runtime=lambda_.Runtime.PYTHON_3_12, architecture=lambda_.Architecture.ARM_64, handler="handler.handler", - code=lambda_.Code.from_asset("../lambdas/po/site_extractor"), + code=lambda_.Code.from_asset( + "../lambdas/po/site_extractor", exclude=["**/__pycache__/**"] + ), timeout=Duration.seconds(60), memory_size=256, log_retention=logs.RetentionDays.TWO_MONTHS, diff --git a/cdk/wo_stack.py b/cdk/wo_stack.py index f156f36..f302b00 100644 --- a/cdk/wo_stack.py +++ b/cdk/wo_stack.py @@ -238,15 +238,16 @@ class WorkorderIngestStack(Stack): architecture=lambda_.Architecture.ARM_64, handler="handler.handler", code=lambda_.Code.from_asset( - "../lambdas/wo/email_processor", + "../lambdas", + exclude=["**/__pycache__/**", "**/tests/**", "**/package/**"], bundling=cdk.BundlingOptions( image=lambda_.Runtime.PYTHON_3_12.bundling_image, command=[ "bash", "-c", "pip install --platform manylinux2014_aarch64 --only-binary=:all: " - "-r requirements.txt -t /asset-output && " - "cp -r . /asset-output/", + "-r wo/email_processor/requirements.txt -t /asset-output && " + "cp wo/email_processor/*.py /asset-output/", ], ), ), @@ -545,7 +546,9 @@ class WorkorderIngestStack(Stack): runtime=lambda_.Runtime.PYTHON_3_12, architecture=lambda_.Architecture.ARM_64, handler="handler.handler", - code=lambda_.Code.from_asset("../lambdas/wo/web_ui"), + code=lambda_.Code.from_asset( + "../lambdas/wo/web_ui", exclude=["**/__pycache__/**"] + ), timeout=Duration.seconds(15), memory_size=128, log_retention=logs.RetentionDays.TWO_MONTHS, diff --git a/tests/test_bundle_consistency.py b/tests/test_bundle_consistency.py index 0f0c3fa..b76d042 100644 --- a/tests/test_bundle_consistency.py +++ b/tests/test_bundle_consistency.py @@ -34,6 +34,19 @@ PO_EXPECTED_TOP_LEVEL_MODULES = frozenset( {"handler", "ses_auth", "template_parser", "derived_fields"} ) +# Pipeline-scoped glob shapes the per-stack ships-all pins accept. Phase 2 +# widened the bundling cwd to the shared ../lambdas asset root, so the executed +# glob carries its own pipeline's path prefix (`po/email_processor/*.py`); a +# bare `*.py`/`./*.py` prefix is still accepted for the legacy in-dir cwd. A +# WRONG-pipeline prefix (`wo/email_processor/*.py` in po_stack) or an arbitrary +# directory (`venv/lib/*.py`) is deliberately NOT accepted: it would false-pass +# the ships-all shape check while staging a bundle missing a required sibling +# (e.g. derived_fields.py, which lives only under po/) -- a runtime ImportError +# that green CI must never wave through. Do NOT relax these to an any-prefix +# pattern: that reintroduces exactly the wrong-pipeline false-pass this pins out. +PO_SCOPED_GLOB_RE = r"(?:^|\s)(?:\./|po/email_processor/)?\*\.py(?:\s|$)" +WO_SCOPED_GLOB_RE = r"(?:^|\s)(?:\./|wo/email_processor/)?\*\.py(?:\s|$)" + def _first_party_sibling_imports(handler_path: Path) -> set[str]: """Top-level module names handler.py imports that are first-party siblings. @@ -116,9 +129,16 @@ def _bundling_ships_all(command: str, sibling_names: set[str]) -> bool: Inspects only the `cp` commands bash actually executes (comments stripped via `_executed_cp_commands`). An executed copy ships every top-level .py sibling unconditionally in two shapes: - - a non-recursive glob copy, e.g. `cp ./*.py /asset-output/` (PO); + - a non-recursive glob copy whose (optional) path prefix resolves, under + the ../lambdas bundling cwd, to a directory that ACTUALLY contains + every required sibling, e.g. `cp po/email_processor/*.py /asset-output/` + (PO, Phase 2). The directory is filesystem-checked, not merely + pattern-matched: a wrong-pipeline or arbitrary-directory glob + (`cp wo/email_processor/*.py` in po_stack, `cp venv/lib/*.py`) does NOT + qualify, because that directory does not hold every required sibling -- + so it can never short-circuit to True while dropping a module; - a recursive copy of the WHOLE source dir, e.g. `cp -r . /asset-output/` - (WO) -- the source operand must be `.`/`./`, so a narrowed recursive + -- the source operand must be `.`/`./`, so a narrowed recursive copy like `cp -r ./package /asset-output/` does NOT qualify. Any other executed `cp` is treated as an explicit filename allowlist (the legacy PO form) and is safe only if every required `.py` literally @@ -130,8 +150,17 @@ def _bundling_ships_all(command: str, sibling_names: set[str]) -> bool: return False copied_files: set[str] = set() for cp in cp_cmds: - if re.search(r"(?:^|\s)\.?/?\*\.py(?:\s|$)", cp): - return True + glob_match = re.search(r"(?:^|\s)((?:[\w./-]+/)?)\*\.py(?:\s|$)", cp) + if glob_match: + # A `*.py` glob ships every top-level .py in the globbed directory + # -- but ONLY that directory. Resolve its prefix against the + # ../lambdas asset root (the Phase 2 bundling cwd) and confirm it + # actually holds every required sibling, so a wrong-pipeline glob + # cannot pass the ships-all check on the mere presence of a `*.py`. + glob_dir = (REPO_ROOT / "lambdas" / glob_match.group(1)).resolve() + if all((glob_dir / f"{name}.py").exists() for name in sibling_names): + return True + continue if re.search(r"cp\s+-r\s+\.\/?\s+/asset-output", cp): return True # Explicit-allowlist shape: collect the copied source filenames, @@ -158,10 +187,12 @@ def test_po_bundling_ships_all_first_party_siblings(): # allowlist. Checked against the executed cp (comments stripped) so the # old glob text surviving only in a comment cannot satisfy this. executed_cps = _executed_cp_commands(command) - assert any(re.search(r"(?:^|\s)\.?/?\*\.py(?:\s|$)", cp) for cp in executed_cps), ( - "cdk/po_stack.py bundling command must EXECUTE the non-recursive glob " - "'cp ./*.py /asset-output/' so every first-party sibling handler.py " - f"imports ships automatically. Executed cp commands: {executed_cps}" + assert any(re.search(PO_SCOPED_GLOB_RE, cp) for cp in executed_cps), ( + "cdk/po_stack.py bundling command must EXECUTE the PO-scoped non-recursive " + "glob 'cp po/email_processor/*.py /asset-output/' so every first-party " + "sibling handler.py imports ships automatically. A wrong-pipeline or " + "arbitrary-directory glob is rejected here on purpose. " + f"Executed cp commands: {executed_cps}" ) assert _bundling_ships_all(command, siblings), ( f"cdk/po_stack.py bundling command does not ship all of {sorted(siblings)}: " @@ -175,18 +206,20 @@ def test_wo_bundling_ships_all_first_party_siblings(): command = _extract_bundling_command(WO_STACK) - # WO is out of scope for the Phase 0 glob change -- it ships everything - # via a recursive `cp -r` of the whole source dir, which is a different - # (broader, not narrower) mechanism that also guarantees every sibling - # ships. Assert the executed cp recursively copies the WHOLE dir (source - # `.`/`./`), so a narrowed `cp -r ./package /asset-output/` does not pass. + # Phase 2: WO now ships via the same scoped non-recursive glob as PO, + # copying only wo/email_processor/*.py from the widened ../lambdas asset + # root. This deliberately drops tests/, __pycache__, and requirements.txt + # from the production zip (docs/refactor-evaluation.md Phase 2). Checked + # against the comment-stripped executed cp so an old `cp -r .` surviving + # only in a comment cannot satisfy this, and a revert to the whole-dir + # copy (which would re-ship tests/) fails loudly. executed_cps = _executed_cp_commands(command) - assert any( - re.search(r"cp\s+-r\s+\.\/?\s+/asset-output", cp) for cp in executed_cps - ), ( - "cdk/wo_stack.py bundling command is expected to recursively copy the " - f"whole source dir so every first-party sibling ships. Executed cp " - f"commands: {executed_cps}" + assert any(re.search(WO_SCOPED_GLOB_RE, cp) for cp in executed_cps), ( + "cdk/wo_stack.py bundling command must EXECUTE the WO-scoped non-recursive " + "glob 'cp wo/email_processor/*.py /asset-output/' so every first-party " + "sibling ships while tests/ and requirements.txt are excluded. A " + "wrong-pipeline or arbitrary-directory glob is rejected here on purpose. " + f"Executed cp commands: {executed_cps}" ) assert _bundling_ships_all(command, siblings), ( f"cdk/wo_stack.py bundling command does not ship all of {sorted(siblings)}: " @@ -248,3 +281,83 @@ def test_detection_logic_catches_allowlist_missing_a_sibling(): "cp handler.py ses_auth.py template_parser.py derived_fields.py /asset-output/" ) assert _bundling_ships_all(complete_allowlist_command, siblings) + + +def test_detection_logic_rejects_narrowed_scoped_glob(): + """A narrowed single-file cp (no `*`) must not satisfy the scoped-glob pin. + + Phase 2 widened the glob's source prefix to `po/email_processor/*.py` + (resp. `wo/email_processor/*.py`) against the ../lambdas asset root. + Guard against a narrowing regression -- e.g. a bad find/replace that + keeps the path prefix but drops the `*` and copies only handler.py -- + from silently passing as ships-all. `cp po/email_processor/handler.py` + has a path prefix but no glob star, so the scoped-glob regex must not + match it, and it also fails the explicit-allowlist fallback because it + omits ses_auth/template_parser/derived_fields. + """ + siblings = _first_party_sibling_imports(PO_HANDLER) + + narrowed_command = ( + "pip install --platform manylinux2014_aarch64 --only-binary=:all: " + "-r po/email_processor/requirements.txt -t /asset-output && " + "cp po/email_processor/handler.py /asset-output/" + ) + assert not _bundling_ships_all(narrowed_command, siblings) + + +def test_detection_logic_rejects_commented_out_scoped_glob(): + """A scoped glob surviving only in a `#` comment must not pass. + + Simulates a revert that narrows the executed `cp` to a single file but + leaves the old scoped-glob text trailing as a comment (e.g. a + half-finished revert). `_executed_cp_commands` strips `#` comments + before any regex sees the segment, so the glob text alone -- never + actually executed by bash -- cannot false-pass the pin. + """ + siblings = _first_party_sibling_imports(WO_HANDLER) + + commented_out_command = ( + "pip install --platform manylinux2014_aarch64 --only-binary=:all: " + "-r wo/email_processor/requirements.txt -t /asset-output && " + "cp wo/email_processor/handler.py /asset-output/ " + "# was: cp wo/email_processor/*.py /asset-output/" + ) + assert not _bundling_ships_all(commented_out_command, siblings) + + +def test_detection_logic_rejects_wrong_pipeline_glob(): + """A glob pointing at the WRONG pipeline (or any other dir) must not pass. + + Phase 2 widened the bundling cwd to the shared ../lambdas asset root, so a + copy-paste slip that leaves po_stack copying `wo/email_processor/*.py` + would stage a bundle missing derived_fields.py (which lives only under + po/email_processor) -- a runtime ImportError. `_bundling_ships_all` + resolves the globbed directory against the lambdas root and confirms it + actually holds every required sibling, so a wrong-pipeline or + arbitrary-directory glob is rejected rather than short-circuiting to True + on the mere presence of a `*.py` token. This is the missing-sibling class + (PR #105 / PR #2) the AST test exists to catch at CI time. + """ + po_siblings = _first_party_sibling_imports(PO_HANDLER) + assert "derived_fields" in po_siblings # lives only under po/email_processor + + wrong_pipeline_command = ( + "pip install --platform manylinux2014_aarch64 --only-binary=:all: " + "-r po/email_processor/requirements.txt -t /asset-output && " + "cp wo/email_processor/*.py /asset-output/" + ) + assert not _bundling_ships_all(wrong_pipeline_command, po_siblings) + + arbitrary_dir_command = ( + "pip install --platform manylinux2014_aarch64 --only-binary=:all: " + "-r po/email_processor/requirements.txt -t /asset-output && " + "cp venv/lib/*.py /asset-output/" + ) + assert not _bundling_ships_all(arbitrary_dir_command, po_siblings) + + # The per-stack shape-check pins reject the wrong prefix too: the PO pin + # matches its own scoped glob but not the WO one, and vice versa. + assert re.search(PO_SCOPED_GLOB_RE, "cp po/email_processor/*.py /asset-output/") + assert not re.search(PO_SCOPED_GLOB_RE, "cp wo/email_processor/*.py /asset-output/") + assert re.search(WO_SCOPED_GLOB_RE, "cp wo/email_processor/*.py /asset-output/") + assert not re.search(WO_SCOPED_GLOB_RE, "cp po/email_processor/*.py /asset-output/")