mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-10-07 16:08:54 +00:00
Implement PO new_po extraction and value-level gate
Replace the extract_new_po scaffold stub with the full section-windowed extractor (duplicate-label anchoring, sentinel ship-to, label-keyed U+2022 bullet split, Decimal money from three anchored contexts only) and add value-level gate rules V1-V13. Both new_po_not_implemented scaffold guards are removed; rules 6-8 (unrecognized_status, multiline_unsupported, non_usd) go live. The gate re-derives every byte proof from the email body so an extractor bug cannot vouch for itself: amount re-serialization with a digit/comma border check (the thousands-separator truncation kill switch), sum(lines)==total against both Total blocks, anchor/supplier identity proofs, USPS address shape on the raw pre-enrichment zip, bullet label discipline, and sentinel/artifact hygiene. Any failure falls closed to the LLM; a validation failure is never a parsed result. Refs: #99
This commit is contained in:
parent
5c40ccfe66
commit
33b33fe256
1 changed files with 618 additions and 49 deletions
|
|
@ -48,15 +48,14 @@ import re
|
||||||
from decimal import Decimal, InvalidOperation
|
from decimal import Decimal, InvalidOperation
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# IMPLEMENTATION STATUS (scaffold)
|
# IMPLEMENTATION STATUS
|
||||||
# [x] contract + recursive skeleton/normalize
|
# [x] contract + recursive skeleton/normalize
|
||||||
# [x] classify_template (both templates)
|
# [x] classify_template (both templates)
|
||||||
# [x] coupa_cancellation extract + gate (simple, safe)
|
# [x] coupa_cancellation extract + gate
|
||||||
# [ ] coupa_new_po extract -- labeled fields, duplicate-label anchoring,
|
# [x] coupa_new_po extract -- labeled fields, duplicate-label anchoring,
|
||||||
# U+2022 line split, Decimal amounts <-- TODO, needs fixtures
|
# U+2022 line split, Decimal amounts (built against the scrubbed real
|
||||||
# [ ] coupa_new_po value-level gate rules <-- TODO, currently FAILS CLOSED
|
# fixture corpus in tests/fixtures/)
|
||||||
# Until the new_po TODOs land, a new_po email deterministically returns
|
# [x] coupa_new_po value-level gate rules V1-V13 (see validate())
|
||||||
# reason 'new_po_not_implemented' and falls back to the LLM -- safe by design.
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
# The contract keys (23), EXACTLY -- mirrors the AI EXTRACTION_PROMPT fields.
|
# The contract keys (23), EXACTLY -- mirrors the AI EXTRACTION_PROMPT fields.
|
||||||
|
|
@ -142,6 +141,69 @@ _PO_ID_RE = re.compile(r"^[A-Z0-9]{1,6}-\d+$")
|
||||||
# U+2022 bullet delimiting per-line metadata in the Lines section.
|
# U+2022 bullet delimiting per-line metadata in the Lines section.
|
||||||
_BULLET = "•"
|
_BULLET = "•"
|
||||||
|
|
||||||
|
# --- new_po layout patterns (built against the scrubbed real fixture corpus) ---
|
||||||
|
# Money tokens are read ONLY from three anchored contexts: the Lines-section
|
||||||
|
# '<desc> for <amt> <CCY>' line, the Total-block standalone amount line, and the
|
||||||
|
# Items-summary '<qty> <UNIT> x <price>' line. NEVER free money-shaped scanning:
|
||||||
|
# the Items summary carries unit-price tokens distinct from line amounts.
|
||||||
|
_MONEY_RE = re.compile(r"\d{1,3}(?:,\d{3})*\.\d{2}")
|
||||||
|
_CURRENCY_RE = re.compile(r"[A-Z]{3}")
|
||||||
|
# Items-summary quantity line, e.g. '1.0 EACH x 55,206.00'.
|
||||||
|
_SUMMARY_ITEM_RE = re.compile(
|
||||||
|
r"^(?P<qty>\d+(?:\.\d+)?) (?P<unit>[A-Z]+) x (?P<price>\d{1,3}(?:,\d{3})*\.\d{2})$"
|
||||||
|
)
|
||||||
|
# Lines-block quantity evidence line, e.g. '1.0 EA' (gate rule V13 cross-check).
|
||||||
|
_LINE_QTY_RE = re.compile(r"^(?P<qty>\d+(?:\.\d+)?) (?P<unit>[A-Z]+)$")
|
||||||
|
# Lines-block description/amount line. GREEDY desc: '.+' binds the LAST ' for ',
|
||||||
|
# so a description containing the word 'for' can never shift the amount.
|
||||||
|
_LINE_DESC_AMT_RE = re.compile(
|
||||||
|
r"^(?P<desc>.+) for (?P<amt>\d{1,3}(?:,\d{3})*\.\d{2}) (?P<cur>[A-Z]{3})$"
|
||||||
|
)
|
||||||
|
_VIEW_ORDER_URL_RE = re.compile(r"^https://supplier\.coupahost\.com/orders/\S+$")
|
||||||
|
_ORDER_URL_ID_RE = re.compile(r"^https://supplier\.coupahost\.com/orders/(\d+)\b")
|
||||||
|
_LOCATION_CODE_RE = re.compile(r"^Location Code: (?P<lc>\d+)$")
|
||||||
|
_ATTN_RE = re.compile(r"^Attn: (?P<attn>.+)$")
|
||||||
|
# Ship-to city line immediately preceding 'United States'.
|
||||||
|
_CITY_STATE_ZIP_RE = re.compile(
|
||||||
|
r"^(?P<city>.+), (?P<state>[A-Z]{2}) (?P<zip>\d{5}(?:-\d{4})?)$"
|
||||||
|
)
|
||||||
|
_US_SENTINEL = "United States"
|
||||||
|
_SUPPLIER_MARKER = (
|
||||||
|
"SEA HAVEN" # case-sensitive; drift becomes fallback, never wrong data
|
||||||
|
)
|
||||||
|
|
||||||
|
# More Detail block: label lines, each expected exactly once (gate rule V9).
|
||||||
|
_MORE_DETAIL_FIELDS = {
|
||||||
|
"Department": "department",
|
||||||
|
"Status": "po_status",
|
||||||
|
"Last Opened": "last_opened",
|
||||||
|
"Order Date": "order_date",
|
||||||
|
"Acknowledged At": "acknowledged_at",
|
||||||
|
"Revision Date": "revision_date",
|
||||||
|
"Payment Term": "payment_terms",
|
||||||
|
"Req #": "requisition_number",
|
||||||
|
}
|
||||||
|
_MORE_DETAIL_LABELS = ("PO ID", *_MORE_DETAIL_FIELDS)
|
||||||
|
|
||||||
|
# Per-line bullet metadata: closed label set, assigned purely by leading label
|
||||||
|
# (longest label first), NEVER by ordinal position -- real data has an optional
|
||||||
|
# 'Part Number' segment between 'Category' and 'Account', and every run starts
|
||||||
|
# with a 'Supplier <name>' segment.
|
||||||
|
_BULLET_LABELS = ("Part Number", "Need By", "Category", "Account", "Period", "Supplier")
|
||||||
|
_BULLET_FIELDS = {
|
||||||
|
"Need By": "need_by",
|
||||||
|
"Category": "category",
|
||||||
|
"Account": "account_code",
|
||||||
|
"Period": "period",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Frozen USPS state/territory codes (50 states + DC + territories).
|
||||||
|
_USPS_STATES = frozenset(
|
||||||
|
"""AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS
|
||||||
|
MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI
|
||||||
|
WY DC PR VI GU AS MP""".split()
|
||||||
|
)
|
||||||
|
|
||||||
# Sentinel: a label was present but its value did not parse. Must FAIL the gate
|
# Sentinel: a label was present but its value did not parse. Must FAIL the gate
|
||||||
# (present-but-unparseable), distinct from an absent value (None).
|
# (present-but-unparseable), distinct from an absent value (None).
|
||||||
_UNPARSEABLE = "__UNPARSEABLE__"
|
_UNPARSEABLE = "__UNPARSEABLE__"
|
||||||
|
|
@ -206,6 +268,74 @@ def _to_decimal(raw):
|
||||||
return _UNPARSEABLE
|
return _UNPARSEABLE
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Line navigation helpers (shared by the extractor and the gate; both operate
|
||||||
|
# on email_data["body"] only -- the gate re-derives its own byte evidence and
|
||||||
|
# never trusts extractor-carried state it can re-derive)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def _visible(line):
|
||||||
|
"""True when the raw line carries visible content. The literal placeholder
|
||||||
|
'None' IS visible (unlike _clean, which maps it to None), so label values
|
||||||
|
of 'None' are found -- not skipped over into the next label line."""
|
||||||
|
return bool(line.replace("\r", "").replace("\xa0", "").strip())
|
||||||
|
|
||||||
|
|
||||||
|
def _indices(lines, label):
|
||||||
|
"""All indices whose _clean-ed content equals the label exactly."""
|
||||||
|
return [i for i, ln in enumerate(lines) if _clean(ln) == label]
|
||||||
|
|
||||||
|
|
||||||
|
def _find_after(lines, label, start):
|
||||||
|
"""First index >= start whose _clean-ed content equals label, or None."""
|
||||||
|
for i in range(start, len(lines)):
|
||||||
|
if _clean(lines[i]) == label:
|
||||||
|
return i
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _next_visible(lines, idx, end=None):
|
||||||
|
"""Index of the first visible line strictly after idx (before end), or None."""
|
||||||
|
stop = len(lines) if end is None else min(end, len(lines))
|
||||||
|
for j in range(idx + 1, stop):
|
||||||
|
if _visible(lines[j]):
|
||||||
|
return j
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _split_bullet_segments(raw_line):
|
||||||
|
"""Split a Lines-section metadata line on the bare U+2022 bullet; strip each
|
||||||
|
segment of spaces and nbsp; drop empty segments."""
|
||||||
|
segments = []
|
||||||
|
for seg in raw_line.replace("\r", "").split(_BULLET):
|
||||||
|
seg = seg.replace("\xa0", " ").strip()
|
||||||
|
if seg:
|
||||||
|
segments.append(seg)
|
||||||
|
return segments
|
||||||
|
|
||||||
|
|
||||||
|
def _match_bullet_label(segment):
|
||||||
|
"""(label, value) by leading-label prefix match (longest label first)
|
||||||
|
against the closed _BULLET_LABELS set, or (None, None) if unrecognized."""
|
||||||
|
for label in sorted(_BULLET_LABELS, key=len, reverse=True):
|
||||||
|
if segment == label:
|
||||||
|
return label, None
|
||||||
|
if segment.startswith(label + " "):
|
||||||
|
return label, segment[len(label) :].strip()
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def _walk_leaves(obj):
|
||||||
|
"""Yield every scalar leaf of a nested dict/list candidate."""
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
for v in obj.values():
|
||||||
|
yield from _walk_leaves(v)
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
for v in obj:
|
||||||
|
yield from _walk_leaves(v)
|
||||||
|
else:
|
||||||
|
yield obj
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Subject helpers
|
# Subject helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -247,35 +377,191 @@ def extract_cancellation(email_data):
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# T1: coupa_new_po -> new_po
|
# T1: coupa_new_po -> new_po
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
def _assign_bullet_metadata(item, raw_line):
|
||||||
|
"""Assign the U+2022 metadata segments to the item BY LEADING LABEL.
|
||||||
|
|
||||||
|
'Supplier' is recognized but not stored on the item (gate rule V5 proves it
|
||||||
|
byte-equals supplier.name from the body); 'Part Number' is recognized but
|
||||||
|
discarded (LINE_ITEM_KEYS has no slot -- inventing one would break the
|
||||||
|
key_set_mismatch rule and LLM-path shape parity). Unrecognized or duplicate
|
||||||
|
segments are the gate's job to reject (rule V8)."""
|
||||||
|
for segment in _split_bullet_segments(raw_line):
|
||||||
|
label, value = _match_bullet_label(segment)
|
||||||
|
field = _BULLET_FIELDS.get(label)
|
||||||
|
if field and item[field] is None:
|
||||||
|
item[field] = _clean(value)
|
||||||
|
|
||||||
|
|
||||||
def extract_new_po(email_data):
|
def extract_new_po(email_data):
|
||||||
"""Extract the new_po contract from the text/plain body.
|
"""Extract the new_po contract from the text/plain body.
|
||||||
|
|
||||||
TODO(new_po): this is the core extraction work and needs fixtures before it
|
Permissive capture, section-windowed: each anchor line is located by exact
|
||||||
can be trusted. Implement, in order, guarding each with a gate rule below:
|
_clean-ed full-line equality, STRICTLY AFTER the previous anchor. Missing
|
||||||
* labeled single-occurrence fields (PO ID, Status, Order Date, Revision
|
anchors leave fields None -- the gate then fails closed. Representation-
|
||||||
Date, Payment Term, Req #, Submitted By, On Behalf Of, view_order_url,
|
agnostic: matches only on _clean-ed lines, never on '\\r'-suffixed literals
|
||||||
Department/Last Opened/Acknowledged At with literal 'None' -> null).
|
(body line endings are decode-path dependent).
|
||||||
* DUPLICATE labels: 'Supplier', 'Shipping', 'Total' each appear TWICE
|
|
||||||
(summary placeholder + detail block; first 'Shipping' is literally
|
Duplicate labels: 'Supplier', 'Shipping', 'Total' each appear TWICE (summary
|
||||||
'None'). Anchor supplier.name on the FIRST 'Supplier', the ship_to block
|
placeholder + detail block; the first 'Shipping' value is literally 'None').
|
||||||
on the SECOND 'Shipping', and pick a deterministic 'Total'.
|
supplier.name anchors on the FIRST 'Supplier'; ship_to on the SECOND
|
||||||
* ship_to by SENTINEL anchors ('United States' / 'Location Code:' /
|
'Shipping'; total on the SECOND 'Total'.
|
||||||
'Attn:'), never fixed line offsets (multi-line street unobserved).
|
|
||||||
* line_items: split the metadata line on the bare U+2022 (_BULLET) and
|
DERIVED_KEYS (site_code, trade, fiscal_year) stay None -- filled later by
|
||||||
assign each segment BY LEADING LABEL word ('Need By '/'Category '/
|
the shared post-stage identically on both paths.
|
||||||
'Account '/'Period '/'Part Number '), never by ordinal position.
|
|
||||||
* amounts via _to_decimal (thousands-separator safe); currency = the exact
|
|
||||||
3-letter token adjacent to the amount.
|
|
||||||
* coupa_category = verbatim 'Category' bullet value.
|
|
||||||
Leave DERIVED_KEYS (site_code, trade, fiscal_year) as None -- filled later by
|
|
||||||
the shared post-stage.
|
|
||||||
"""
|
"""
|
||||||
candidate = _empty_candidate()
|
candidate = _empty_candidate()
|
||||||
candidate["email_type"] = "new_po"
|
candidate["email_type"] = "new_po"
|
||||||
candidate["po_number"] = _subject_po_id(email_data)
|
candidate["po_number"] = _subject_po_id(email_data)
|
||||||
|
|
||||||
|
lines = _plain_lines(email_data["body"])
|
||||||
|
|
||||||
|
# --- Summary section (start .. 'More Detail') ---
|
||||||
|
more_detail = _find_after(lines, "More Detail", 0)
|
||||||
|
summary_end = more_detail if more_detail is not None else len(lines)
|
||||||
|
for label, field in (
|
||||||
|
("Submitted By", "submitted_by"),
|
||||||
|
("On Behalf Of", "on_behalf_of"),
|
||||||
|
):
|
||||||
|
idx = _find_after(lines, label, 0)
|
||||||
|
if idx is not None and idx < summary_end:
|
||||||
|
j = _next_visible(lines, idx, summary_end)
|
||||||
|
if j is not None:
|
||||||
|
candidate[field] = _clean(lines[j])
|
||||||
|
sup1 = _find_after(lines, "Supplier", 0)
|
||||||
|
if sup1 is not None and sup1 < summary_end:
|
||||||
|
j = _next_visible(lines, sup1, summary_end)
|
||||||
|
if j is not None:
|
||||||
|
candidate["supplier"]["name"] = _clean(lines[j])
|
||||||
|
# view_order_url: the unique orders link in the summary (0 or >1 -> None).
|
||||||
|
url_lines = [
|
||||||
|
_clean(lines[i])
|
||||||
|
for i in range(summary_end)
|
||||||
|
if _VIEW_ORDER_URL_RE.match(_clean(lines[i]) or "")
|
||||||
|
]
|
||||||
|
if len(url_lines) == 1:
|
||||||
|
candidate["view_order_url"] = url_lines[0]
|
||||||
|
# Items-summary quantity lines ('1.0 EACH x 55,206.00'), collected in order.
|
||||||
|
summary_items = [
|
||||||
|
m
|
||||||
|
for i in range(summary_end)
|
||||||
|
if (m := _SUMMARY_ITEM_RE.fullmatch(_clean(lines[i]) or ""))
|
||||||
|
]
|
||||||
|
|
||||||
|
# --- More Detail block ('More Detail' .. second 'Supplier') ---
|
||||||
|
sup2 = None
|
||||||
|
if more_detail is not None:
|
||||||
|
sup2 = _find_after(lines, "Supplier", more_detail + 1)
|
||||||
|
md_end = sup2 if sup2 is not None else len(lines)
|
||||||
|
for label, field in _MORE_DETAIL_FIELDS.items():
|
||||||
|
idx = _find_after(lines, label, more_detail + 1)
|
||||||
|
if idx is not None and idx < md_end:
|
||||||
|
j = _next_visible(lines, idx, md_end)
|
||||||
|
if j is not None:
|
||||||
|
candidate[field] = _clean(lines[j])
|
||||||
|
# The FIRST 'Shipping' lives in this block; its value must be the literal
|
||||||
|
# 'None' placeholder (gate rule V4 tripwire) and is never used for ship_to.
|
||||||
|
|
||||||
|
# --- ship_to (second 'Shipping' .. 'Lines'), sentinel-anchored ---
|
||||||
|
ship2 = _find_after(lines, "Shipping", sup2 + 1) if sup2 is not None else None
|
||||||
|
lines_anchor = _find_after(lines, "Lines", ship2 + 1) if ship2 is not None else None
|
||||||
|
st_end = lines_anchor if lines_anchor is not None else len(lines)
|
||||||
|
ship_to = candidate["ship_to"]
|
||||||
|
if ship2 is not None:
|
||||||
|
name_idx = _next_visible(lines, ship2, st_end)
|
||||||
|
if name_idx is not None:
|
||||||
|
ship_to["name"] = _clean(lines[name_idx])
|
||||||
|
us_idx = _find_after(lines, _US_SENTINEL, name_idx + 1)
|
||||||
|
if us_idx is not None and us_idx < st_end:
|
||||||
|
city_idx = us_idx - 1
|
||||||
|
m = None
|
||||||
|
if city_idx > name_idx:
|
||||||
|
m = _CITY_STATE_ZIP_RE.fullmatch(_clean(lines[city_idx]) or "")
|
||||||
|
if m:
|
||||||
|
ship_to["city"] = m.group("city")
|
||||||
|
ship_to["state"] = m.group("state")
|
||||||
|
ship_to["zip"] = m.group("zip")
|
||||||
|
street = [
|
||||||
|
_clean(lines[j])
|
||||||
|
for j in range(name_idx + 1, city_idx)
|
||||||
|
if _visible(lines[j])
|
||||||
|
]
|
||||||
|
if street:
|
||||||
|
ship_to["street"] = "\n".join(street)
|
||||||
|
ship_to["address"] = "\n".join(
|
||||||
|
_clean(lines[j])
|
||||||
|
for j in range(name_idx, us_idx + 1)
|
||||||
|
if _visible(lines[j])
|
||||||
|
)
|
||||||
|
for j in range(us_idx + 1, st_end):
|
||||||
|
cl = _clean(lines[j]) or ""
|
||||||
|
lc = _LOCATION_CODE_RE.fullmatch(cl)
|
||||||
|
if lc and ship_to["location_code"] is None:
|
||||||
|
ship_to["location_code"] = lc.group("lc")
|
||||||
|
attn = _ATTN_RE.fullmatch(cl)
|
||||||
|
if attn and ship_to["attn"] is None:
|
||||||
|
ship_to["attn"] = _clean(attn.group("attn"))
|
||||||
|
|
||||||
|
# --- Lines section ('Lines' .. second 'Total') ---
|
||||||
|
# Item blocks are delimited by the lone U+00A0 line(s). EVERY block is
|
||||||
|
# extracted, even when >1, so gate rule 7 fires with honest
|
||||||
|
# multiline_unsupported data (never silently keep item 0).
|
||||||
|
total2 = (
|
||||||
|
_find_after(lines, "Total", lines_anchor + 1)
|
||||||
|
if lines_anchor is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
items = []
|
||||||
|
if lines_anchor is not None:
|
||||||
|
end = total2 if total2 is not None else len(lines)
|
||||||
|
blocks, block = [], []
|
||||||
|
for j in range(lines_anchor + 1, end):
|
||||||
|
if lines[j].replace("\r", "") == "\xa0":
|
||||||
|
blocks.append(block)
|
||||||
|
block = []
|
||||||
|
else:
|
||||||
|
block.append(lines[j])
|
||||||
|
blocks.append(block)
|
||||||
|
for block in blocks:
|
||||||
|
visible = [ln for ln in block if _visible(ln)]
|
||||||
|
if not visible:
|
||||||
|
continue
|
||||||
|
item = _empty_line_item()
|
||||||
|
for raw in visible:
|
||||||
|
dm = _LINE_DESC_AMT_RE.fullmatch(_clean(raw) or "")
|
||||||
|
if dm and item["description"] is None:
|
||||||
|
item["description"] = _clean(dm.group("desc"))
|
||||||
|
item["amount"] = _to_decimal(dm.group("amt"))
|
||||||
|
item["currency"] = dm.group("cur")
|
||||||
|
elif _BULLET in raw:
|
||||||
|
_assign_bullet_metadata(item, raw)
|
||||||
|
# The optional '<qty> EA' evidence line is not stored: quantity/
|
||||||
|
# unit/price come from the Items summary; gate rule V13
|
||||||
|
# cross-checks the EA line against it from the body.
|
||||||
|
items.append(item)
|
||||||
|
if items:
|
||||||
|
candidate["line_items"] = items
|
||||||
|
# coupa_category is a VERBATIM copy of item 0's Category bullet value.
|
||||||
|
candidate["coupa_category"] = items[0]["category"]
|
||||||
|
if len(summary_items) == 1 and len(items) == 1:
|
||||||
|
m = summary_items[0]
|
||||||
|
items[0]["quantity"] = _to_decimal(m.group("qty"))
|
||||||
|
items[0]["unit"] = m.group("unit")
|
||||||
|
items[0]["price"] = _to_decimal(m.group("price"))
|
||||||
|
|
||||||
|
# --- Total block (second 'Total' .. end) ---
|
||||||
|
if total2 is not None:
|
||||||
|
amt_idx = _next_visible(lines, total2)
|
||||||
|
if amt_idx is not None:
|
||||||
|
token = _clean(lines[amt_idx]) or ""
|
||||||
|
if _MONEY_RE.fullmatch(token):
|
||||||
|
candidate["total_amount"] = _to_decimal(token)
|
||||||
|
cur_idx = _next_visible(lines, amt_idx)
|
||||||
|
if cur_idx is not None:
|
||||||
|
cur_token = _clean(lines[cur_idx]) or ""
|
||||||
|
if _CURRENCY_RE.fullmatch(cur_token):
|
||||||
|
candidate["currency"] = cur_token
|
||||||
|
|
||||||
# DERIVED_KEYS intentionally left None (shared post-stage fills them).
|
# DERIVED_KEYS intentionally left None (shared post-stage fills them).
|
||||||
# TODO(new_po): populate the remaining fields per the docstring, then remove
|
|
||||||
# the fail-closed guard in validate().
|
|
||||||
return _normalize(candidate)
|
return _normalize(candidate)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -333,11 +619,6 @@ def validate(candidate, template_id, email_data):
|
||||||
return True, "ok"
|
return True, "ok"
|
||||||
|
|
||||||
# ---- coupa_new_po ----
|
# ---- coupa_new_po ----
|
||||||
# SCAFFOLD GUARD: extract_new_po is not implemented yet, so a new_po parse is
|
|
||||||
# never provable -> fail closed to the LLM with an honest reason. Remove this
|
|
||||||
# line once extract_new_po + the value-level rules below are implemented.
|
|
||||||
return False, "new_po_not_implemented"
|
|
||||||
|
|
||||||
# (6) status must be one of the confirmed-safe strings.
|
# (6) status must be one of the confirmed-safe strings.
|
||||||
status = candidate.get("po_status")
|
status = candidate.get("po_status")
|
||||||
if status not in NEW_PO_SAFE_STATUSES:
|
if status not in NEW_PO_SAFE_STATUSES:
|
||||||
|
|
@ -351,22 +632,310 @@ def validate(candidate, template_id, email_data):
|
||||||
if candidate.get("currency") != "USD":
|
if candidate.get("currency") != "USD":
|
||||||
return False, "non_usd"
|
return False, "non_usd"
|
||||||
|
|
||||||
# TODO(new_po) value-level gate rules -- implement alongside extract_new_po:
|
# (V1-V13) value-level rules: every byte proof is RE-DERIVED from
|
||||||
# * amount fidelity: re-serialize each Decimal to the source grouped format
|
# email_data["body"] -- the gate never trusts extractor-carried state it
|
||||||
# and require byte-equality with the matched substring; reject if the
|
# can re-derive, so an extractor bug cannot vouch for itself.
|
||||||
# token is bordered by a digit/comma (guards '18,624.05' -> '624.05').
|
return _validate_new_po_values(candidate, email_data)
|
||||||
# Then require sum(line_items[].amount) == total_amount.
|
|
||||||
# * duplicate-label anchoring: reject Shipping=='None'; require ship_to has
|
|
||||||
# a 'Location Code:' line; supplier.name contains 'SEA HAVEN';
|
def _money_border_ok(line_text, serialized):
|
||||||
# ship_to.name != supplier.name.
|
"""True when `serialized` occurs in the source line with a character before
|
||||||
# * bullet metadata: every U+2022 segment carries a recognized leading
|
it that is not a digit or comma (the '18,624.05' -> '624.05' kill switch:
|
||||||
# label; fail on any unlabeled segment.
|
a truncated capture re-serializes as '624.05', but every occurrence of that
|
||||||
# * address shape: ship_to.state null-or-valid-USPS; ship_to.zip null-or
|
string in its source line is preceded by a comma or digit)."""
|
||||||
# ^\d{5}(-\d{4})?$ after pad_zip; tokenize city/state/zip by pattern.
|
idx = line_text.find(serialized)
|
||||||
# * _UNPARSEABLE discipline: reject any field still == _UNPARSEABLE.
|
while idx != -1:
|
||||||
# * whitespace hygiene: reject any value still containing \r or \xa0.
|
prev = line_text[idx - 1] if idx > 0 else ""
|
||||||
# Until the above land, new_po is not provable -> fail closed to the LLM.
|
if prev not in "0123456789,":
|
||||||
return False, "new_po_not_implemented"
|
return True
|
||||||
|
idx = line_text.find(serialized, idx + 1)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_new_po_values(candidate, email_data): # noqa: PLR0911, PLR0912, PLR0915
|
||||||
|
"""Value-level gate rules V1-V13 for coupa_new_po. FAIL CLOSED.
|
||||||
|
|
||||||
|
Anchor integrity (V4) runs first because every later byte proof needs the
|
||||||
|
anchor frame; the remaining rules run in spec order. Each rule re-derives
|
||||||
|
its evidence from email_data["body"] -- duplicated regex work, negligible
|
||||||
|
at ~57 emails/day, in exchange for extractor-bug independence."""
|
||||||
|
lines = _plain_lines(email_data["body"])
|
||||||
|
items = candidate.get("line_items") or []
|
||||||
|
ship_to = candidate.get("ship_to") or {}
|
||||||
|
supplier_name = (candidate.get("supplier") or {}).get("name")
|
||||||
|
|
||||||
|
# --- V4 anchor integrity -> anchor_violation ---
|
||||||
|
more_detail_idxs = _indices(lines, "More Detail")
|
||||||
|
supplier_idxs = _indices(lines, "Supplier")
|
||||||
|
shipping_idxs = _indices(lines, "Shipping")
|
||||||
|
total_idxs = _indices(lines, "Total")
|
||||||
|
if len(more_detail_idxs) != 1:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
if len(supplier_idxs) != 2 or len(shipping_idxs) != 2 or len(total_idxs) != 2:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
more_detail = more_detail_idxs[0]
|
||||||
|
# The FIRST Shipping value must be the literal 'None' placeholder -- the
|
||||||
|
# dup-label-swap tripwire (a real address under the first label means the
|
||||||
|
# layout drifted; ship_to would have been read from the wrong block).
|
||||||
|
first_ship_val = ""
|
||||||
|
if shipping_idxs[0] + 1 < len(lines):
|
||||||
|
first_ship_val = (
|
||||||
|
lines[shipping_idxs[0] + 1].replace("\r", "").replace("\xa0", "").strip()
|
||||||
|
)
|
||||||
|
if first_ship_val != "None":
|
||||||
|
return False, "anchor_violation"
|
||||||
|
lines_anchor = _find_after(lines, "Lines", shipping_idxs[1] + 1)
|
||||||
|
if lines_anchor is None:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
# Section ordering: summary Supplier < More Detail < detail Supplier <
|
||||||
|
# second Shipping < Lines < second Total; summary Total before More Detail.
|
||||||
|
if not (
|
||||||
|
supplier_idxs[0]
|
||||||
|
< more_detail
|
||||||
|
< supplier_idxs[1]
|
||||||
|
< shipping_idxs[1]
|
||||||
|
< lines_anchor
|
||||||
|
< total_idxs[1]
|
||||||
|
):
|
||||||
|
return False, "anchor_violation"
|
||||||
|
if not (total_idxs[0] < more_detail < shipping_idxs[0] < supplier_idxs[1]):
|
||||||
|
return False, "anchor_violation"
|
||||||
|
lines_end = total_idxs[1]
|
||||||
|
|
||||||
|
# --- V1 money fidelity -> amount_mismatch ---
|
||||||
|
# Every captured money value must re-locate its raw source token in the
|
||||||
|
# body: token fullmatches the grouped money shape, format(value, ',.2f')
|
||||||
|
# byte-equals it, and the character before it is not a digit/comma.
|
||||||
|
for_matches = []
|
||||||
|
for j in range(lines_anchor + 1, lines_end):
|
||||||
|
m = _LINE_DESC_AMT_RE.fullmatch(_clean(lines[j]) or "")
|
||||||
|
if m:
|
||||||
|
for_matches.append(m)
|
||||||
|
if len(for_matches) != len(items):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
for item, m in zip(items, for_matches):
|
||||||
|
amount = item.get("amount")
|
||||||
|
if not isinstance(amount, Decimal):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
serialized = format(amount, ",.2f")
|
||||||
|
if serialized != m.group("amt"):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if not _money_border_ok(m.string, serialized):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
summary_matches = [
|
||||||
|
m
|
||||||
|
for i in range(more_detail)
|
||||||
|
if (m := _SUMMARY_ITEM_RE.fullmatch(_clean(lines[i]) or ""))
|
||||||
|
]
|
||||||
|
price = items[0].get("price") if items else None
|
||||||
|
if price is not None:
|
||||||
|
if not isinstance(price, Decimal) or len(summary_matches) != 1:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
serialized = format(price, ",.2f")
|
||||||
|
if serialized != summary_matches[0].group("price"):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if not _money_border_ok(summary_matches[0].string, serialized):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
total = candidate.get("total_amount")
|
||||||
|
if not isinstance(total, Decimal):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
|
||||||
|
# --- V3 dual-Total proof -> amount_mismatch (V2 needs its token, so it
|
||||||
|
# derives here; sum proof follows immediately) ---
|
||||||
|
total_tokens = []
|
||||||
|
for t_idx in total_idxs:
|
||||||
|
a_idx = _next_visible(lines, t_idx)
|
||||||
|
if a_idx is None:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
token = _clean(lines[a_idx]) or ""
|
||||||
|
if not _MONEY_RE.fullmatch(token):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
c_idx = _next_visible(lines, a_idx)
|
||||||
|
cur_token = (_clean(lines[c_idx]) or "") if c_idx is not None else ""
|
||||||
|
if not _CURRENCY_RE.fullmatch(cur_token):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
total_tokens.append((token, cur_token))
|
||||||
|
if total_tokens[0] != total_tokens[1]:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
serialized = format(total, ",.2f")
|
||||||
|
if serialized != total_tokens[1][0]:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if candidate.get("currency") != total_tokens[1][1]:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
|
||||||
|
# --- V2 sum proof -> amount_mismatch (exact Decimal equality; deliberately
|
||||||
|
# NO qty*price==amount rule -- corpus shows partial quantities) ---
|
||||||
|
if sum(item["amount"] for item in items) != total:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
|
||||||
|
# --- V5 supplier proof -> anchor_violation ---
|
||||||
|
if not supplier_name or _SUPPLIER_MARKER not in supplier_name:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
det_idx = _next_visible(lines, supplier_idxs[1], shipping_idxs[1])
|
||||||
|
if det_idx is None or _clean(lines[det_idx]) != supplier_name:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
sum_idx = _next_visible(lines, supplier_idxs[0], more_detail)
|
||||||
|
if sum_idx is None or _clean(lines[sum_idx]) != supplier_name:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
if ship_to.get("name") == supplier_name:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
|
||||||
|
# --- V8 bullet discipline -> bullet_label_unrecognized (V5's per-item
|
||||||
|
# Supplier-segment proof rides the same walk) ---
|
||||||
|
bullet_lines = [
|
||||||
|
lines[j] for j in range(lines_anchor + 1, lines_end) if _BULLET in lines[j]
|
||||||
|
]
|
||||||
|
if len(bullet_lines) != len(items):
|
||||||
|
return False, "bullet_label_unrecognized"
|
||||||
|
for raw in bullet_lines:
|
||||||
|
seen = {}
|
||||||
|
for segment in _split_bullet_segments(raw):
|
||||||
|
label, value = _match_bullet_label(segment)
|
||||||
|
if label is None or label in seen:
|
||||||
|
return False, "bullet_label_unrecognized"
|
||||||
|
seen[label] = value
|
||||||
|
if set(seen) - {
|
||||||
|
"Supplier",
|
||||||
|
"Need By",
|
||||||
|
"Category",
|
||||||
|
"Account",
|
||||||
|
"Period",
|
||||||
|
"Part Number",
|
||||||
|
}:
|
||||||
|
return False, "bullet_label_unrecognized"
|
||||||
|
if not {"Supplier", "Need By", "Category", "Account", "Period"} <= set(seen):
|
||||||
|
return False, "bullet_label_unrecognized"
|
||||||
|
if _clean(seen["Supplier"]) != supplier_name:
|
||||||
|
return False, "anchor_violation"
|
||||||
|
|
||||||
|
# --- V6 ship_to required -> missing_required_field ---
|
||||||
|
for field in ("name", "street", "city", "state", "zip", "location_code"):
|
||||||
|
if not ship_to.get(field):
|
||||||
|
return False, "missing_required_field"
|
||||||
|
if not re.fullmatch(r"\d+", ship_to["location_code"]):
|
||||||
|
return False, "missing_required_field"
|
||||||
|
lc_values = [
|
||||||
|
m.group("lc")
|
||||||
|
for j in range(shipping_idxs[1] + 1, lines_anchor)
|
||||||
|
if (m := _LOCATION_CODE_RE.fullmatch(_clean(lines[j]) or ""))
|
||||||
|
]
|
||||||
|
if ship_to["location_code"] not in lc_values:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
attn_values = [
|
||||||
|
_clean(m.group("attn"))
|
||||||
|
for j in range(shipping_idxs[1] + 1, lines_anchor)
|
||||||
|
if (m := _ATTN_RE.fullmatch(_clean(lines[j]) or ""))
|
||||||
|
]
|
||||||
|
if attn_values:
|
||||||
|
if ship_to.get("attn") not in attn_values:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
elif ship_to.get("attn") is not None:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
|
||||||
|
# --- V7 address shape -> address_shape_invalid (gated on the RAW
|
||||||
|
# pre-enrichment zip: validate() runs BEFORE enrich_parsed, so a short zip
|
||||||
|
# like '2149' fails closed to the LLM path where pad_zip repairs it --
|
||||||
|
# both paths then get identical pad_zip treatment downstream) ---
|
||||||
|
us_idx = _find_after(lines, _US_SENTINEL, shipping_idxs[1] + 1)
|
||||||
|
if us_idx is None or us_idx >= lines_anchor:
|
||||||
|
return False, "address_shape_invalid"
|
||||||
|
m = _CITY_STATE_ZIP_RE.fullmatch(_clean(lines[us_idx - 1]) or "")
|
||||||
|
if not m:
|
||||||
|
return False, "address_shape_invalid"
|
||||||
|
if (
|
||||||
|
m.group("city") != ship_to["city"]
|
||||||
|
or m.group("state") != ship_to["state"]
|
||||||
|
or m.group("zip") != ship_to["zip"]
|
||||||
|
):
|
||||||
|
return False, "address_shape_invalid"
|
||||||
|
if ship_to["state"] not in _USPS_STATES:
|
||||||
|
return False, "address_shape_invalid"
|
||||||
|
if not re.fullmatch(r"\d{5}(-\d{4})?", ship_to["zip"]):
|
||||||
|
return False, "address_shape_invalid"
|
||||||
|
|
||||||
|
# --- V9 required labeled fields -> missing_required_field ---
|
||||||
|
for field in (
|
||||||
|
"po_status",
|
||||||
|
"order_date",
|
||||||
|
"payment_terms",
|
||||||
|
"requisition_number",
|
||||||
|
"submitted_by",
|
||||||
|
"view_order_url",
|
||||||
|
):
|
||||||
|
if candidate.get(field) is None:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
for item in items:
|
||||||
|
for field in (
|
||||||
|
"description",
|
||||||
|
"amount",
|
||||||
|
"currency",
|
||||||
|
"need_by",
|
||||||
|
"category",
|
||||||
|
"account_code",
|
||||||
|
"period",
|
||||||
|
):
|
||||||
|
if item.get(field) is None:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
for label in _MORE_DETAIL_LABELS:
|
||||||
|
if len(_indices(lines, label)) != 1:
|
||||||
|
return False, "missing_required_field"
|
||||||
|
if candidate.get("coupa_category") != items[0].get("category"):
|
||||||
|
return False, "missing_required_field"
|
||||||
|
# Nullable by design: on_behalf_of, department, last_opened, acknowledged_at,
|
||||||
|
# revision_date, attn, quantity, unit, price.
|
||||||
|
|
||||||
|
# --- V10 PO identity proofs -> po_id_mismatch ---
|
||||||
|
po = candidate["po_number"]
|
||||||
|
po_id_idx = _indices(lines, "PO ID")[0]
|
||||||
|
v_idx = _next_visible(lines, po_id_idx)
|
||||||
|
if v_idx is None or _clean(lines[v_idx]) != po:
|
||||||
|
return False, "po_id_mismatch"
|
||||||
|
if not _indices(lines, f"Amazon Purchase Order #{po}"):
|
||||||
|
return False, "po_id_mismatch"
|
||||||
|
um = _ORDER_URL_ID_RE.match(candidate.get("view_order_url") or "")
|
||||||
|
if not um or um.group(1) != po.split("-", 1)[1]:
|
||||||
|
return False, "po_id_mismatch"
|
||||||
|
|
||||||
|
# --- V11 sentinel discipline -> unparseable_value ---
|
||||||
|
for leaf in _walk_leaves(candidate):
|
||||||
|
if isinstance(leaf, str) and leaf == _UNPARSEABLE:
|
||||||
|
return False, "unparseable_value"
|
||||||
|
|
||||||
|
# --- V12 hygiene -> residual_artifact ---
|
||||||
|
for leaf in _walk_leaves(candidate):
|
||||||
|
if isinstance(leaf, str) and ("\r" in leaf or "\xa0" in leaf):
|
||||||
|
return False, "residual_artifact"
|
||||||
|
|
||||||
|
# --- V13 quantity/unit/price coherence -> amount_mismatch ---
|
||||||
|
quantity = items[0].get("quantity")
|
||||||
|
unit = items[0].get("unit")
|
||||||
|
if quantity is not None or unit is not None or price is not None:
|
||||||
|
if not isinstance(quantity, Decimal) or quantity <= 0:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if not unit or not re.fullmatch(r"[A-Z]+", unit):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if not isinstance(price, Decimal):
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if len(summary_matches) != 1:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if _to_decimal(summary_matches[0].group("qty")) != quantity:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if summary_matches[0].group("unit") != unit:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
# The Lines-block '<qty> EA' evidence line must numeric-equal the
|
||||||
|
# summary quantity when present.
|
||||||
|
ea_matches = [
|
||||||
|
m
|
||||||
|
for j in range(lines_anchor + 1, lines_end)
|
||||||
|
if (m := _LINE_QTY_RE.fullmatch(_clean(lines[j]) or ""))
|
||||||
|
]
|
||||||
|
if ea_matches:
|
||||||
|
if len(ea_matches) != 1:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
if _to_decimal(ea_matches[0].group("qty")) != quantity:
|
||||||
|
return False, "amount_mismatch"
|
||||||
|
|
||||||
|
return True, "ok"
|
||||||
|
|
||||||
|
|
||||||
def try_deterministic_parse(email_data):
|
def try_deterministic_parse(email_data):
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue