procurement-ingest/lambdas/po/email_processor/enrichment.py

"""PO post-parse enrichment (PO-only; WO has no enrichment stage).

Adds metadata, promotes nested ship-to fields, canonicalizes numeric types
across both parse paths, and runs the deterministic derived-field classifiers
(site_code, trade, fiscal_year) that FILL GAPS but never overwrite an
LLM-supplied value during the bake. Pure module: no boto3. The derived-agreement
shadow EMF is emitted via telemetry (ai_fallback path only).
"""

import logging
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation

from derived_fields import derive_all
from telemetry import _emit_derived_agreement_metric

logger = logging.getLogger()
logger.setLevel(logging.INFO)

# The three classifier outputs derive_all() computes. Python fills these when
# the extraction path left them null; on ai_fallback the LLM value (if any)
# stays authoritative during the bake and Python only shadows it.
DERIVED_FIELDS = ("site_code", "trade", "fiscal_year")


def pad_zip(zip_code: str | None) -> str | None:
    if not zip_code:
        return zip_code
    clean = zip_code.strip().split("-")[0]
    if clean.isdigit() and len(clean) < 5:
        return clean.zfill(5) + zip_code.strip()[len(clean) :]
    return zip_code


def enrich_parsed(parsed: dict, s3_key: str, email_subject: str, *, parse_method: str):
    """Add metadata and promote nested fields to top level.

    ``parse_method`` ("template" | "ai_fallback") selects the derived-field
    shadow behavior below: agreement telemetry is emitted only on ai_fallback,
    where an LLM value exists to compare the Python classifier against.
    """
    now = datetime.now(timezone.utc).isoformat()
    parsed["raw_s3_key"] = s3_key
    parsed["processed_at"] = now
    parsed["data_source"] = "email"
    parsed["email_subject"] = email_subject

    ship_to = parsed.get("ship_to") or {}
    if ship_to.get("address"):
        parsed["ship_to_raw"] = ship_to["address"]
    if ship_to.get("state"):
        parsed["state"] = ship_to["state"]

    if ship_to.get("zip"):
        ship_to["zip"] = pad_zip(ship_to["zip"])

    # Canonical numeric type for quantity/price across BOTH parse paths: the
    # template parser emits Decimal (DynamoDB Number) while EXTRACTION_PROMPT
    # asks the LLM for these two fields as JSON strings (which the Bedrock
    # json.loads leaves as str -> DynamoDB String). Coercing here -- in the
    # SHARED post-stage -- converges the attribute type to Number for
    # equivalent parsed dicts, preserving the two-path parity contract on the
    # purchase-orders table stream. Prompt rewording itself is PR #2 scope.
    # A non-numeric string is left verbatim (still stored, as a String) --
    # dropping it would lose LLM-extracted evidence.
    for item in parsed.get("line_items") or []:
        if not isinstance(item, dict):
            continue
        for field in ("quantity", "price"):
            value = item.get(field)
            if isinstance(value, str):
                try:
                    item[field] = Decimal(value.replace(",", "").strip())
                except InvalidOperation:
                    pass
            elif isinstance(value, (int, float)) and not isinstance(value, bool):
                # parse_float=Decimal means floats can't occur on the LLM path,
                # but a bare JSON int would slip through as Python int; coerce
                # so both paths emit one canonical Decimal type (cross-review FIX).
                item[field] = Decimal(str(value))

    # Derived-field classification (site_code, trade, fiscal_year). Python
    # derivation FILLS GAPS on BOTH paths but NEVER OVERWRITES: an LLM-supplied
    # value (only possible on the ai_fallback path) stays authoritative during
    # the bake period. On ai_fallback we additionally emit one shadow EMF record
    # per field comparing the Python value to the LLM value, so agreement can be
    # measured before Python becomes authoritative and the rules are dropped
    # from EXTRACTION_PROMPT (a post-bake follow-up).
    #
    # The whole block is wrapped defensively: derive_all() is total and pure,
    # but this is an S3-async Lambda where any uncaught exception means a retry
    # storm -> DLQ, so no classification/telemetry error may ever fail the
    # invocation.
    try:
        python_vals = derive_all(parsed)
        for field in DERIVED_FIELDS:
            llm_value = parsed.get(field)
            python_value = python_vals.get(field)
            if llm_value is None and python_value is not None:
                # Python fills the gap on both paths.
                parsed[field] = python_value
            # else: a non-None LLM value (ai_fallback only) is kept as-is.
            if parse_method == "ai_fallback":
                _emit_derived_agreement_metric(
                    field, llm_value, python_value, parsed.get("po_number")
                )
    except Exception:  # noqa: BLE001 - telemetry must never fail the invocation
        logger.exception(
            "derived-field classification/telemetry failed; continuing without it"
        )

    return parsed