procurement-ingest/tests/support/loader.py

"""The ONE Lambda-module loader for the whole procurement-ingest test suite.

Every test root (the repo-root ``tests/`` suite AND the two per-pipeline
``lambdas/*/email_processor/tests`` suites) loads Lambda modules through this
single ``load_lambda_module`` -- there is exactly one copy of the sys.modules
save/restore dance in the repo, and the repo-root ``conftest.py`` re-exports it.

``from conftest import ...`` is deliberately NOT used: three ``conftest.py``
files exist across the roots and pytest's per-directory sys.path prepending
makes the bare name ``conftest`` resolve nondeterministically -- exactly the
bare-name-collision class this phase eliminates. The loader lives here instead,
imported the same way from every invocation directory as ``tests.support``.
"""

import importlib.util
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[2]
_SHARED_DIR = REPO_ROOT / "lambdas" / "shared"


# Sibling modules imported by bare name from the handlers (the Lambda runtime
# puts each function's own directory on sys.path; the CDK bundling then cp's the
# shared modules in flat beside handler.py so those bare imports resolve too).
# template_parser/derived_fields are duplicated PER PIPELINE, so their bare
# names MUST be bound to the right pipeline's file around each handler exec --
# relying on sys.path ordering (or on whatever a previously collected suite left
# in sys.modules) silently binds a handler to the OTHER pipeline's sibling.
# ses_auth/email_parsing/emf/web_ui_auth are now single-sourced under
# lambdas/shared/ (Phase 3); the loop below resolves them from there via a
# shared-dir fallback.
#
# Phase 5 decomposed each God-handler into flat siblings (prompts/telemetry/
# extraction/enrichment/persistence). The order below is DEPENDENCY-TOPOLOGICAL,
# not alphabetical: the loader binds each bare name in sys.modules right after
# exec'ing it, so a sibling whose module body does `from <x> import ...` must
# appear AFTER <x> here or its exec ImportErrors. The load-bearing edges are
# emf < telemetry, prompts < extraction, and derived_fields + telemetry <
# enrichment. WO has no enrichment/derived_fields sibling -- the loader's
# `if not sibling_path.exists(): continue` silently skips them there, so one
# unified tuple serves both pipelines.
#
# web_ui_auth was added (no dependencies; after emf) so
# load_lambda_module("po"|"wo", "web_ui/handler") can bind the web_ui handlers'
# bare `from web_ui_auth import is_authenticated` (lambdas/po/web_ui/handler.py:15
# and the wo equivalent) via the same shared-dir fallback.
_SIBLING_MODULES = (
    "ses_auth",
    "email_parsing",
    "emf",
    "web_ui_auth",
    "prompts",
    "template_parser",
    "derived_fields",
    "telemetry",
    "extraction",
    "enrichment",
    "persistence",
)


def _load_module(path, module_name):
    if module_name in sys.modules:
        return sys.modules[module_name]
    spec = importlib.util.spec_from_file_location(module_name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[module_name] = module
    spec.loader.exec_module(module)
    return module


def load_lambda_module(pipeline, name):
    """Load a Lambda module by file path under a unique, deterministic name.

    ``pipeline`` is one of ``{"po", "wo", "shared"}`` and ``name`` is the path
    under ``lambdas/<pipeline>/`` without the ``.py`` suffix (e.g.
    ``"email_processor/handler"``, ``"web_ui/handler"``, or ``"ses_auth"`` for
    shared). The module name is ``f"{pipeline}_{name.replace('/', '_')}"`` --
    this reproduces the existing unique names byte-for-byte
    (``po_email_processor_handler``, ``wo_email_processor_handler``,
    ``shared_ses_auth``), so every existing ``sys.modules`` sibling key
    (``po_email_processor_handler__persistence`` etc.) is unchanged.

    The handler files all share the basename ``handler.py`` and are not
    importable as packages, so a plain ``import handler`` would collide across
    Lambdas. The same loader serves leaf modules (``ses_auth.py``,
    ``web_ui_auth.py``), which are likewise loaded by file path.

    Handler modules import their siblings by bare name (e.g. ``from
    template_parser import try_deterministic_parse``). Each sibling is loaded
    from the handler's own directory (falling back to ``lambdas/shared/``) under
    a unique module name and registered under its bare name only for the
    duration of the handler exec, then the previous binding is restored -- so
    this loader is deterministic regardless of collection order and of what the
    per-Lambda test suites (which put their own module dir on sys.path) have
    already cached in sys.modules.

    The save/restore dance does NOT shrink to nothing: template_parser (and
    derived_fields/prompts/telemetry/extraction/enrichment/persistence) remain
    duplicated bare names ACROSS pipelines. One pytest session execs BOTH
    handlers; without per-exec bare-name binding + restore, whichever pipeline
    loads second silently binds to the first pipeline's sibling. Only
    ses_auth/email_parsing/emf/web_ui_auth are single-sourced.
    """
    path = REPO_ROOT / "lambdas" / pipeline / f"{name}.py"
    module_name = f"{pipeline}_{name.replace('/', '_')}"
    if module_name in sys.modules:
        return sys.modules[module_name]
    # Keep the handler dir on sys.path for parity with the Lambda runtime.
    handler_dir = str(path.parent)
    if handler_dir not in sys.path:
        sys.path.insert(0, handler_dir)
    if path.name != "handler.py":
        # Leaf modules (e.g. ses_auth.py / web_ui_auth.py) have no sibling
        # imports of the per-pipeline kind the dance guards.
        return _load_module(path, module_name)
    saved = {}
    for sibling in _SIBLING_MODULES:
        # Per-pipeline siblings (template_parser/derived_fields/...) resolve next
        # to the handler; the shared, single-sourced siblings (ses_auth/
        # email_parsing/emf/web_ui_auth) fall back to lambdas/shared/. No
        # ambiguity: post Phase 3 the shared names exist ONLY under shared/, the
        # per-pipeline names ONLY next to the handler.
        sibling_path = path.parent / f"{sibling}.py"
        if not sibling_path.exists():
            sibling_path = _SHARED_DIR / f"{sibling}.py"
        if not sibling_path.exists():
            continue
        saved[sibling] = sys.modules.get(sibling)
        sys.modules[sibling] = _load_module(sibling_path, f"{module_name}__{sibling}")
    try:
        module = _load_module(path, module_name)
    finally:
        for sibling, previous in saved.items():
            if previous is not None:
                sys.modules[sibling] = previous
            else:
                sys.modules.pop(sibling, None)
    return module