diff --git a/docs/po-template-parser.md b/docs/po-template-parser.md index 63f28bf..82bf517 100644 --- a/docs/po-template-parser.md +++ b/docs/po-template-parser.md @@ -155,7 +155,7 @@ Reason codes emitted by `validate()` / `try_deterministic_parse()` (closed set): - `email_type` in enum and == template's expected type (`email_type_mismatch`). - `po_number` valid shape `^[A-Z0-9]{1,6}-\d+$` **and** byte-equals the subject id (`po_id_mismatch` / `missing_required_field`). -**Implemented (new_po):** safe-status enum (`unrecognized_status`), single-line only (`multiline_unsupported`), USD only (`non_usd`). +**Implemented (new_po):** safe-status enum (`unrecognized_status`), single-line only (`multiline_unsupported`), USD only (`non_usd`) — enforced at **both** levels: the Total-block top-level currency must be exactly `USD` (rule 8) **and** every line item's captured currency plus its re-derived `for ` body token must byte-equal it (a single non-USD line item fails closed even when the Total block reads USD). **Implemented (new_po value-level rules V1–V13 — ✅ done, one reason code each):** the gate **re-derives every byte proof from `email_data["body"]`** (never trusting extractor-carried state), with the six value-level reason codes: - `amount_mismatch` — money fidelity (each Decimal re-serializes byte-identically to its re-derived source token, non-digit/comma border — the `18,624.05`→`624.05` kill switch), `sum(lines) == total` (exact Decimal; deliberately **no** `qty×price == amount` rule — corpus shows partial quantities), dual-`Total` byte-identity, and quantity/unit/price coherence vs the summary and `EA` evidence lines. @@ -238,7 +238,7 @@ Plus `missing_required_field` for the required labeled fields (per-item metadata 3. **Strict single-line/USD-only v1 acceptable?** ✅ **Resolved (PR #1): yes.** Covers ~99.8% of new_po; multi-line and non-USD fail closed with honest reason codes. 4. **Confirm the real cancellation `Status` spelling** — `handler.py` hardcodes `CANCELLED_STATUS='Cancelled'`; the cancellation body doesn't restate it. Verify against a known-cancelled PO in the table. (Doesn't affect parser output — `save_cancellation` hardcodes the string.) 5. **Comments** — should `New Comment on Purchase Order` emails create/update PO records at all, or be dropped? (Out of scope for the parser, but a real data-quality decision.) -6. **Repo fixtures** — ✅ **Resolved (PR #1): committed sanitized corpus** under `lambdas/po/email_processor/tests/fixtures/` (35 scrubbed `.eml`: 17 single-line new-PO, 8 cancellations, 4 comments, 3 non-Coupa, 3 multi-line; plus 17 synthetic adversarial mutations). Transport/auth headers scrubbed to same-shape placeholders (structure preserved, `ses_auth` still passes), per-file digit cipher on identifiers, amounts remapped with `sum==total` re-established, and a narrow `.gitignore` exception scoped to the PO fixtures path (never blanket `*.eml`). Source S3 keys deliberately unrecorded (they ARE the SES receipt tokens the scrub replaced). +6. **Repo fixtures** — ✅ **Resolved (PR #1): committed sanitized corpus** under `lambdas/po/email_processor/tests/fixtures/` (35 scrubbed `.eml`: 17 single-line new-PO, 8 cancellations, 4 comments, 3 non-Coupa, 3 multi-line; plus 18 synthetic adversarial mutations). Transport/auth headers scrubbed to same-shape placeholders (structure preserved, `ses_auth` still passes), per-file digit cipher on identifiers, amounts remapped with `sum==total` re-established, and a narrow `.gitignore` exception scoped to the PO fixtures path (never blanket `*.eml`). Source S3 keys deliberately unrecorded (they ARE the SES receipt tokens the scrub replaced). **Fixture-build pins recorded during PR #1 (locked by goldens):** - **`ship_to.street`/`ship_to.address` join convention:** multi-line segments joined with `'\n'`; `address` = the lines from `ship_to.name` through `United States` inclusive (this is what `enrich_parsed` promotes to `ship_to_raw` on the site-extractor stream). diff --git a/lambdas/po/email_processor/template_parser.py b/lambdas/po/email_processor/template_parser.py index 7a9962a..4e0ec76 100644 --- a/lambdas/po/email_processor/template_parser.py +++ b/lambdas/po/email_processor/template_parser.py @@ -722,6 +722,16 @@ def _validate_new_po_values(candidate, email_data): # noqa: PLR0911, PLR0912, P return False, "amount_mismatch" if not _money_border_ok(m.string, serialized): return False, "amount_mismatch" + # Line-level currency pin (rule 8 companion): rule 8 already proved the + # top-level currency is exactly 'USD', but that only covers the Total + # block. Every line item's captured currency AND its re-derived body + # token must byte-equal it too -- a single non-USD line item fails + # closed even when the Total block reads USD (the non-USD path is + # entirely unexercised at every level, not just the total). + if item.get("currency") != candidate.get("currency"): + return False, "non_usd" + if m.group("cur") != candidate.get("currency"): + return False, "non_usd" summary_matches = [ m for i in range(more_detail) diff --git a/lambdas/po/email_processor/tests/fixtures/adversarial/adv-line-non-usd.eml b/lambdas/po/email_processor/tests/fixtures/adversarial/adv-line-non-usd.eml new file mode 100644 index 0000000..ef747e6 --- /dev/null +++ b/lambdas/po/email_processor/tests/fixtures/adversarial/adv-line-non-usd.eml @@ -0,0 +1,592 @@ +Return-Path: <0100abcdef012345-abcdef01-2345-6789-abcd-ef0123456789-6789ab@mail.coupahost.com> +Received: from a41-238.smtp-out.amazonses.com (a41-238.smtp-out.amazonses.com [203.0.113.25]) + by inbound-smtp.us-east-1.amazonaws.com with SMTP id fixture0123456789fixture0123456789fixtur + for amazon_po@int.seahaven.com; + Tue, 12 May 2026 14:02:54 +0000 (UTC) +Received-SPF: pass (spfCheck: domain of mail.coupahost.com designates 203.0.113.25 as permitted sender) client-ip=203.0.113.25; envelope-from=0100abcdef012345-abcdef01-2345-6789-abcd-ef0123456789-6789ab@mail.coupahost.com; helo=a41-238.smtp-out.amazonses.com; +Authentication-Results: amazonses.com; + spf=pass (spfCheck: domain of mail.coupahost.com designates 203.0.113.25 as permitted sender) client-ip=203.0.113.25; envelope-from=0100abcdef012345-abcdef01-2345-6789-abcd-ef0123456789-6789ab@mail.coupahost.com; helo=a41-238.smtp-out.amazonses.com; + dkim=pass header.i=@amazonses.com; + dkim=pass header.i=@amazon.coupahost.com; + dmarc=pass header.from=amazon.coupahost.com; +X-SES-RECEIPT: ScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholderTokenVal== +X-SES-DKIM-SIGNATURE: a=rsa-sha256; q=dns/txt; b=Scrub/bedFixturePlaceholderTokenValue01234+56789QqZzXxJjKkWwYyVvScrubbedFixturePlaceholder/Tok+enValue0123456789QqZzXxJjKkWwYyVvScrubbedFixturePlaceh/olderTokenValue012345=; c=relaxed/simple; s=224i4yxa5dv7c2xz3womw6peuasteono; d=amazonses.com; t=1778594574; v=1; bh=ScrubbedFi+xture+PlaceholderTokenValue01234=; h=From:To:Cc:Bcc:Subject:Date:Message-ID:MIME-Version:Content-Type:X-SES-RECEIPT; +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; + s=vtqzvxxcuybp44vxc3gxytxk7c3q5pf4; d=coupahost.com; t=1778594574; + i=@amazon.coupahost.com; + h=Date:From:To:Message-ID:Subject:Mime-Version:Content-Type:Content-Transfer-Encoding; + bh=ScrubbedFi+xture+PlaceholderTokenValue01234=; + b=ScrubbedFixt+urePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvS + crubbedFixturePla/ceholderTokenValue0123456789QqZzXxJ/jK/kWwYyV+vSc + rubbedFixturePlaceholderTokenValue012345= +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; + s=224i4yxa5dv7c2xz3womw6peuasteono; d=amazonses.com; t=1778594574; + h=Date:From:To:Message-ID:Subject:Mime-Version:Content-Type:Content-Transfer-Encoding:Feedback-ID; + bh=ScrubbedFi+xture+PlaceholderTokenValue01234=; + b=ScrubbedFixturePlaceholderTokenValue0123456789QqZzXxJjKkWwYyVvSc + rubbedFixturePlaceholder+TokenValue0123456789QqZzXxJjK+kWwYyVvScrub + bedFixturePlaceholderTokenValue012345678= +Date: Tue, 12 May 2026 14:02:53 +0000 +From: Amazon +To: amazon_po@int.seahaven.com +Message-ID: +Subject: ***Copy for Reference*** New Purchase Order 2D-12503765 has been + issued +Mime-Version: 1.0 +Content-Type: multipart/alternative; + boundary="--==_mimepart_6a03330dccc3f_2224e2ae416689"; + charset=UTF-8 +Content-Transfer-Encoding: 7bit +auto-submitted: auto-generated +X-Auto-Response-Suppress: All +X-Coupa-Message-ID: abcdef01-2345-6789-abcd-ef0123456789@amazon.coupahost.com +Feedback-ID: ::1.us-east-1.ScrubbedFixturePlaceholderTokenValue0123456=:AmazonSES +X-SES-Outgoing: 2026.05.12-203.0.113.25 + + +----==_mimepart_6a03330dccc3f_2224e2ae416689 +Content-Type: text/plain; + charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Powered by + +http://www.coupa.com + +Amazon Purchase Order #2D-12503765 + +Hi Amazon, + +This is to inform you Purchase Order 2D-12503765 from Amazon has been issue= +d, and this same notification has been sent to your fellow team members wit= +h access to the Coupa Supplier Portal. + +Submitted By +Priya Raghavan + +On Behalf Of +Priya Raghavan + +Supplier +SEA HAVEN INDUSTRIES:CORPORATE AP:SERVICES(OK6HE) + +Total + +55,206.00 + +USD + +Items + +GENERAL LABOURER + +1.0 EACH x 55,206.00 + +55,206.00 + +USD + +https://supplier.coupahost.com/orders/12503765?instance_url=3Dhttps%3A%2F%2= +Famazon.coupahost.com%2F&supplier_id=3D648018& + +More Detail + +PO ID +2D-12503765 + +Department +None + +Status +Issued - Created + +Last Opened +None + +Order Date +05/12/26 + +Acknowledged At +None + +Revision Date +05/12/26 + +Payment Term +90 NET + +Req # +19886224 + +Shipping +None + +Supplier + +SEA HAVEN INDUSTRIES:CORPORATE AP:SERVICES(OK6HE) + +80 ORVILLE DR STE 100 +BOHEMIA, NY 11716-2505 +United States + +Shipping + +Amazon.com Services LLC (DYO1) +500 Hudson Valley +New Windsor, NY 12553-4744 +United States +Location Code: 207735629 +Attn: Priya Raghavan + +Lines + +1.0 EA +GENERAL LABOURER for 55,206.00 CAD + + Supplier SEA HAVEN INDUSTRIES:CORPORATE AP:SERVICES(OK6HE) =E2=80=A2 Need= + By 05/12/26 =E2=80=A2 Category Maintenance - Facilities =E2=80=A2 Part N= +umber "HVAC/Heating System Remova" =E2=80=A2 Account 1D-2S31-2231-73970-00= +00-0000-0000 =E2=80=A2 Period 2D Ops 2026 + +=C2=A0 + +Total + +55,206.00 + +USD + +https://www.coupa.com + +Total Spend Management - Make Margins Multiply + +To get immediate updates via SMS or change notification preferences, go htt= +ps://supplier.coupahost.com/notification_preferences + +----==_mimepart_6a03330dccc3f_2224e2ae416689 +Content-Type: text/html; + charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + + + + + + + + +
+
+ + + + + + + + + + +
+ + + + + +
+ + + + +
+ +Powered by + +
+		 + + + + +
+3D"Coupa +
+
+ +
+ + + + + + + +
+ + + + + + + +
+3D"Amazon" +
+

Amazon Purchase Order #2D-12503765

+
+
+ + + + +
+

Hi Amazon,

+

This is to inform you Purchase Order 2D-12503765 from Amazon has been= + issued, and this same notification has been sent to your fellow team membe= +rs with access to the Coupa Supplier Portal.

+
+ + + + + + + + + + + + + + + + + + + + + +
Submitted ByPriya Raghavan
On Behalf OfPriya Raghavan
SupplierSEA HAVEN INDUSTRIES:CORPORATE AP:SERVICES(OK6HE)
Total +55,206.00 + +USD + +
Items + + + + + + +
GENERAL LABOURER +1.0 EACH x 55,206.00 + + +55,206.00 + + +USD + +
+ +
+ +
+
+ +
View Order
+

+ + +
+More Detail +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PO ID= +2D-12503765Depar= +tmentNone
Statu= +sIssued - CreatedLast = +OpenedNone
Order= + Date05/12/26Ackno= +wledged AtNone
Revis= +ion Date05/12/26Payme= +nt Term90 NET
Req #= +19886224Shipp= +ingNone
+
+
+ + + + + + + + +
+
Supplier
+
+SEA HAVEN INDUSTRIES:CORPORATE AP:SERVICES(OK6HE) + +80 ORVILLE DR STE 100
BOHEMIA, NY 11716-2505
United States + + + + + +		 +
Shipping
+
+ +Amazon.com Services LLC (DYO1)
500 Hudson Valley
New Windsor, NY 1255= +3-4744
United States
Location Code: 207735629
Attn: Priya Raghavan + + + +
+ + +
Lines +
+ + + + + + + + + + + + + + + +
+ +1.0 EA +GENERAL LABOURER for 55,206.00 USD +
+ + + Supplier SEA HAVEN INDUSTRIES:CORPORATE AP:= +SERVICES(OK6HE) =E2=80=A2 Need= + By 05/12/26 =E2=80=A2 Catego= +ry Maintenance - Facilities =E2=80=A2= + Part Number "HVAC/Heating System Remova" =E2=80= +=A2 Account 1D-2S31-2231-73970-0000-0000-0000 =E2=80=A2 Period 2D Ops 2026 + + + + +
+=C2=A0 +
+
+
+ + +Total + + +55,206.00 + +USD + + + + + + +
+
+ +
+ + +
+
+
+ + + + +
+ + + + +
+
+3D"Coupa +
+ +Total Spend Management - Make Margins Multiply + +
+
+
+ + + + +
+
+To get immediate updates via SMS or change notification preferences, go here and a= +djust your settings +
+
+
+ +
+
+
+ + + +----==_mimepart_6a03330dccc3f_2224e2ae416689-- diff --git a/lambdas/po/email_processor/tests/test_po_validation_gate.py b/lambdas/po/email_processor/tests/test_po_validation_gate.py index 17ae7c3..22b7729 100644 --- a/lambdas/po/email_processor/tests/test_po_validation_gate.py +++ b/lambdas/po/email_processor/tests/test_po_validation_gate.py @@ -50,8 +50,11 @@ ADVERSARIAL_REASONS = { # safe-status enum "adv-status-pending": "unrecognized_status", "adv-missing-status": "unrecognized_status", - # synthetic CAD (no non-USD new_po exists in the corpus) + # synthetic CAD (no non-USD new_po exists in the corpus): Total-block + # variant AND line-item-only variant (Total block still reads USD -- the + # line-level pin must fail closed on its own, rule 8 alone cannot see it) "adv-non-usd": "non_usd", + "adv-line-non-usd": "non_usd", # identity proofs "adv-po-id-mismatch": "po_id_mismatch", "adv-url-id-mismatch": "po_id_mismatch", @@ -218,6 +221,16 @@ def test_v12_residual_artifact_rejected(artifact): _assert_reason(cand, "residual_artifact") +def test_line_item_currency_must_equal_usd(): + # The non_usd gate is NOT satisfied by the Total block alone: a line item + # whose captured currency diverges from the proven-USD top-level currency + # fails closed (candidate-level mutation; the body-level variant is the + # adv-line-non-usd adversarial fixture). + cand = _good_candidate() + cand["line_items"][0]["currency"] = "CAD" + _assert_reason(cand, "non_usd") + + def test_v13_quantity_must_match_summary_line(): cand = _good_candidate() cand["line_items"][0]["quantity"] = Decimal("2.0") # body says 1.0