mirror of
https://github.com/Sea-Haven-Industries/procurement-ingest.git
synced 2026-09-30 15:23:13 +00:00
f5c09757f3
* feat(lambda): report unhandled Lambda errors to Sentry * fix(lambda): strip Sentry stack-frame locals * style(tests): wrap long lines in sentry_init tests
158 lines
7.5 KiB
Python
158 lines
7.5 KiB
Python
"""The ONE Lambda-module loader for the whole procurement-ingest test suite.
|
|
|
|
Every test root (the repo-root ``tests/`` suite AND the two per-pipeline
|
|
``lambdas/*/email_processor/tests`` suites) loads Lambda modules through this
|
|
single ``load_lambda_module`` -- there is exactly one copy of the sys.modules
|
|
save/restore dance in the repo, and the repo-root ``conftest.py`` re-exports it.
|
|
|
|
``from conftest import ...`` is deliberately NOT used: three ``conftest.py``
|
|
files exist across the roots and pytest's per-directory sys.path prepending
|
|
makes the bare name ``conftest`` resolve nondeterministically -- exactly the
|
|
bare-name-collision class this phase eliminates. The loader lives here instead,
|
|
imported the same way from every invocation directory as ``tests.support``.
|
|
"""
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
_SHARED_DIR = REPO_ROOT / "lambdas" / "shared"
|
|
|
|
|
|
# Sibling modules imported by bare name from the handlers (the Lambda runtime
|
|
# puts each function's own directory on sys.path; the CDK bundling then cp's the
|
|
# shared modules in flat beside handler.py so those bare imports resolve too).
|
|
# template_parser/derived_fields are duplicated PER PIPELINE, so their bare
|
|
# names MUST be bound to the right pipeline's file around each handler exec --
|
|
# relying on sys.path ordering (or on whatever a previously collected suite left
|
|
# in sys.modules) silently binds a handler to the OTHER pipeline's sibling.
|
|
# ses_auth/email_parsing/emf/web_ui_auth are now single-sourced under
|
|
# lambdas/shared/ (Phase 3); the loop below resolves them from there via a
|
|
# shared-dir fallback.
|
|
#
|
|
# Phase 5 decomposed each God-handler into flat siblings (prompts/telemetry/
|
|
# extraction/enrichment/persistence). The order below is DEPENDENCY-TOPOLOGICAL,
|
|
# not alphabetical: the loader binds each bare name in sys.modules right after
|
|
# exec'ing it, so a sibling whose module body does `from <x> import ...` must
|
|
# appear AFTER <x> here or its exec ImportErrors. The load-bearing edges are
|
|
# emf < telemetry, prompts < extraction, and derived_fields + telemetry <
|
|
# enrichment. WO has no enrichment/derived_fields sibling -- the loader's
|
|
# `if not sibling_path.exists(): continue` silently skips them there, so one
|
|
# unified tuple serves both pipelines.
|
|
#
|
|
# web_ui_auth was added (no dependencies; after emf) so
|
|
# load_lambda_module("po"|"wo", "web_ui/handler") can bind the web_ui handlers'
|
|
# bare `from web_ui_auth import is_authenticated` (lambdas/po/web_ui/handler.py:15
|
|
# and the wo equivalent) via the same shared-dir fallback. sentry_init is the
|
|
# same shape: every handler does `import sentry_init` (side-effect SDK init).
|
|
_SIBLING_MODULES = (
|
|
"ses_auth",
|
|
"email_parsing",
|
|
"emf",
|
|
"web_ui_auth",
|
|
"sentry_init",
|
|
# procurement-api siblings (lambdas/api/): pagination/serialization/router
|
|
# have no sibling deps; wo_repo/po_repo import pagination, so they follow
|
|
# it. These names exist only under lambdas/api/, so the po/wo handler
|
|
# loads skip them via the exists() check.
|
|
"pagination",
|
|
"serialization",
|
|
"router",
|
|
"wo_repo",
|
|
"po_repo",
|
|
"prompts",
|
|
"template_parser",
|
|
"derived_fields",
|
|
"telemetry",
|
|
"extraction",
|
|
"enrichment",
|
|
"persistence",
|
|
# SHOC emitter siblings (lambdas/wo/shoc_emitter/): envelope and delivery
|
|
# are both standalone (no sibling deps, stdlib + boto3 only), so appending
|
|
# them at the end keeps the tuple dependency-topological; the order between
|
|
# the two is irrelevant. They exist only under wo/shoc_emitter/, so every
|
|
# other handler load skips them via the exists() check.
|
|
"envelope",
|
|
"delivery",
|
|
)
|
|
|
|
|
|
def _load_module(path, module_name):
|
|
if module_name in sys.modules:
|
|
return sys.modules[module_name]
|
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[module_name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def load_lambda_module(pipeline, name):
|
|
"""Load a Lambda module by file path under a unique, deterministic name.
|
|
|
|
``pipeline`` is one of ``{"po", "wo", "shared"}`` and ``name`` is the path
|
|
under ``lambdas/<pipeline>/`` without the ``.py`` suffix (e.g.
|
|
``"email_processor/handler"``, ``"web_ui/handler"``, or ``"ses_auth"`` for
|
|
shared). The module name is ``f"{pipeline}_{name.replace('/', '_')}"`` --
|
|
this reproduces the existing unique names byte-for-byte
|
|
(``po_email_processor_handler``, ``wo_email_processor_handler``,
|
|
``shared_ses_auth``), so every existing ``sys.modules`` sibling key
|
|
(``po_email_processor_handler__persistence`` etc.) is unchanged.
|
|
|
|
The handler files all share the basename ``handler.py`` and are not
|
|
importable as packages, so a plain ``import handler`` would collide across
|
|
Lambdas. The same loader serves leaf modules (``ses_auth.py``,
|
|
``web_ui_auth.py``), which are likewise loaded by file path.
|
|
|
|
Handler modules import their siblings by bare name (e.g. ``from
|
|
template_parser import try_deterministic_parse``). Each sibling is loaded
|
|
from the handler's own directory (falling back to ``lambdas/shared/``) under
|
|
a unique module name and registered under its bare name only for the
|
|
duration of the handler exec, then the previous binding is restored -- so
|
|
this loader is deterministic regardless of collection order and of what the
|
|
per-Lambda test suites (which put their own module dir on sys.path) have
|
|
already cached in sys.modules.
|
|
|
|
The save/restore dance does NOT shrink to nothing: template_parser (and
|
|
derived_fields/prompts/telemetry/extraction/enrichment/persistence) remain
|
|
duplicated bare names ACROSS pipelines. One pytest session execs BOTH
|
|
handlers; without per-exec bare-name binding + restore, whichever pipeline
|
|
loads second silently binds to the first pipeline's sibling. Only
|
|
ses_auth/email_parsing/emf/web_ui_auth are single-sourced.
|
|
"""
|
|
path = REPO_ROOT / "lambdas" / pipeline / f"{name}.py"
|
|
module_name = f"{pipeline}_{name.replace('/', '_')}"
|
|
if module_name in sys.modules:
|
|
return sys.modules[module_name]
|
|
# Keep the handler dir on sys.path for parity with the Lambda runtime.
|
|
handler_dir = str(path.parent)
|
|
if handler_dir not in sys.path:
|
|
sys.path.insert(0, handler_dir)
|
|
if path.name != "handler.py":
|
|
# Leaf modules (e.g. ses_auth.py / web_ui_auth.py) have no sibling
|
|
# imports of the per-pipeline kind the dance guards.
|
|
return _load_module(path, module_name)
|
|
saved = {}
|
|
for sibling in _SIBLING_MODULES:
|
|
# Per-pipeline siblings (template_parser/derived_fields/...) resolve next
|
|
# to the handler; the shared, single-sourced siblings (ses_auth/
|
|
# email_parsing/emf/web_ui_auth) fall back to lambdas/shared/. No
|
|
# ambiguity: post Phase 3 the shared names exist ONLY under shared/, the
|
|
# per-pipeline names ONLY next to the handler.
|
|
sibling_path = path.parent / f"{sibling}.py"
|
|
if not sibling_path.exists():
|
|
sibling_path = _SHARED_DIR / f"{sibling}.py"
|
|
if not sibling_path.exists():
|
|
continue
|
|
saved[sibling] = sys.modules.get(sibling)
|
|
sys.modules[sibling] = _load_module(sibling_path, f"{module_name}__{sibling}")
|
|
try:
|
|
module = _load_module(path, module_name)
|
|
finally:
|
|
for sibling, previous in saved.items():
|
|
if previous is not None:
|
|
sys.modules[sibling] = previous
|
|
else:
|
|
sys.modules.pop(sibling, None)
|
|
return module
|