feat(lambda): report unhandled Lambda errors to Sentry

conftest.py

@@ -29,6 +29,9 @@ os.environ.setdefault("AWS_DEFAULT_REGION", "us-east-1")
 os.environ.setdefault("AWS_ACCESS_KEY_ID", "testing")
 os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "testing")
 os.environ.setdefault("AWS_SESSION_TOKEN", "testing")
+# Handlers import sentry_init at module load. Drop a developer-shell DSN so
+# pytest never talks to Sentry (init_sentry no-ops when unset).
+os.environ.pop("SENTRY_DSN", None)
 
 # Imported for its side effect and DELIBERATELY BEFORE the handler modules are
 # loaded below: moto registers its botocore stubber hook into botocore's

lambdas/api/handler.py

@@ -14,6 +14,7 @@ import logging
 from pathlib import Path
 
 import po_repo
+import sentry_init  # noqa: F401
 import wo_repo
 from botocore.exceptions import ClientError
 from pagination import BadCursor, clamp_limit

lambdas/api/requirements.txt

@@ -1 +1,2 @@
-# Dependabot anchor only. The procurement-api Lambda is stdlib+boto3 (provided by the runtime); nothing is pip-installed into the bundle.
+# boto3 is provided by the Lambda Python runtime (lambda-template.md) — do not vendor it.
+sentry-sdk==2.68.1

lambdas/po/email_processor/handler.py

@@ -20,6 +20,7 @@ routing.
 import logging
 
 import boto3
+import sentry_init  # noqa: F401
 from email_parsing import parse_raw_email
 from enrichment import enrich_parsed
 from extraction import extract_with_claude

lambdas/po/email_processor/requirements.txt

@@ -1,2 +1,3 @@
 # Per-function dependencies. Leave empty if the function uses only boto3 and stdlib.
 # boto3 is provided by the Lambda Python runtime (lambda-template.md) — do not vendor it.
+sentry-sdk==2.68.1

lambdas/po/site_extractor/handler.py

@@ -9,6 +9,7 @@ import re
 from datetime import datetime, timezone
 
 import boto3
+import sentry_init  # noqa: F401
 from boto3.dynamodb.types import TypeDeserializer
 
 logger = logging.getLogger()

lambdas/po/site_extractor/requirements.txt

@@ -1 +1,2 @@
 boto3==1.43.78
+sentry-sdk==2.68.1

lambdas/po/web_ui/handler.py

@@ -12,6 +12,7 @@ from decimal import Decimal
 from html import escape as esc
 
 import boto3
+import sentry_init  # noqa: F401
 from web_ui_auth import is_authenticated
 
 logger = logging.getLogger()

lambdas/po/web_ui/requirements.txt

@@ -1 +1,2 @@
 boto3==1.43.78
+sentry-sdk==2.68.1

lambdas/shared/sentry_init.py

@@ -0,0 +1,98 @@
+"""Shared Sentry SDK init for every procurement-ingest Lambda.
+
+Imported for side effect from each handler. ``init_sentry()`` is a no-op when
+``SENTRY_DSN`` is unset so pytest, local invokes, and a missing HCP var never
+talk to Sentry. ``before_send`` strips auth headers and drops request/extra
+keys that can hold MIME bodies, Bedrock prompts, or HMAC secret material.
+"""
+
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+_HEADER_DROP_NAMES = frozenset(
+    {
+        "authorization",
+        "x-auth-token",
+        "cookie",
+        "x-amz-security-token",
+    }
+)
+_DROP_REQUEST_KEYS = frozenset(
+    {
+        "body",
+        "Body",
+        "data",
+        "cookies",
+        "raw_email",
+        "prompt",
+        "secret",
+        "SecretString",
+        "hmac",
+        "keys",
+    }
+)
+_DROP_EXTRA_NEEDLES = (
+    "body",
+    "email",
+    "prompt",
+    "secret",
+    "hmac",
+    "token",
+    "mime",
+    "raw_email",
+)
+
+
+def _drop_header(name):
+    lower = str(name).lower()
+    return lower in _HEADER_DROP_NAMES or lower.startswith("x-amz-")
+
+
+def _scrub_headers(headers):
+    if isinstance(headers, dict):
+        return {k: v for k, v in headers.items() if not _drop_header(k)}
+    if isinstance(headers, list):
+        kept = []
+        for pair in headers:
+            if isinstance(pair, (list, tuple)) and pair and _drop_header(pair[0]):
+                continue
+            kept.append(pair)
+        return kept
+    return headers
+
+
+def _before_send(event, _hint):
+    request = event.get("request")
+    if isinstance(request, dict):
+        headers = request.get("headers")
+        if headers is not None:
+            request["headers"] = _scrub_headers(headers)
+        for key in list(request):
+            if key in _DROP_REQUEST_KEYS or str(key).lower() in {"body", "data"}:
+                request.pop(key, None)
+    extra = event.get("extra")
+    if isinstance(extra, dict):
+        for key in list(extra):
+            lower = str(key).lower()
+            if any(needle in lower for needle in _DROP_EXTRA_NEEDLES):
+                extra.pop(key, None)
+    return event
+
+
+def init_sentry():
+    dsn = os.environ.get("SENTRY_DSN")
+    if not dsn:
+        return
+    sentry_sdk.init(
+        dsn=dsn,
+        integrations=[AwsLambdaIntegration(timeout_warning=True)],
+        send_default_pii=False,
+        enable_logs=False,
+        traces_sample_rate=0.0,
+        before_send=_before_send,
+    )
+
+
+init_sentry()

lambdas/wo/email_processor/handler.py

@@ -18,6 +18,7 @@ import logging
 import re
 
 import boto3
+import sentry_init  # noqa: F401
 from email_parsing import parse_raw_email
 from extraction import extract_with_bedrock
 from persistence import save_event, save_work_order

lambdas/wo/email_processor/requirements.txt

@@ -1,2 +1,3 @@
 # Per-function dependencies. Leave empty if the function uses only boto3 and stdlib.
 # boto3 is provided by the Lambda Python runtime (lambda-template.md) — do not vendor it.
+sentry-sdk==2.68.1

lambdas/wo/shoc_emitter/handler.py

@@ -36,6 +36,7 @@ import time
 import boto3
 import delivery
 import envelope
+import sentry_init  # noqa: F401
 from botocore.config import Config
 
 logger = logging.getLogger()

lambdas/wo/shoc_emitter/requirements.txt

@@ -0,0 +1,2 @@
+# boto3 is provided by the Lambda Python runtime (lambda-template.md) — do not vendor it.
+sentry-sdk==2.68.1

lambdas/wo/shoc_hmac_rotator/handler.py

@@ -25,6 +25,7 @@ import secrets
 from datetime import datetime, timezone
 
 import boto3
+import sentry_init  # noqa: F401
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)

lambdas/wo/shoc_hmac_rotator/requirements.txt

@@ -0,0 +1,2 @@
+# boto3 is provided by the Lambda Python runtime (lambda-template.md) — do not vendor it.
+sentry-sdk==2.68.1

lambdas/wo/web_ui/handler.py

@@ -11,6 +11,7 @@ import os
 from html import escape as esc
 
 import boto3
+import sentry_init  # noqa: F401
 from web_ui_auth import is_authenticated
 
 logger = logging.getLogger()

lambdas/wo/web_ui/requirements.txt

@@ -1 +1,2 @@
 boto3>=1.43.78
+sentry-sdk==2.68.1

terraform/api.tf

@@ -127,6 +127,7 @@ resource "aws_lambda_function" "procurement_api" {
       COMMENTS_TABLE               = aws_dynamodb_table.work_order_comments_kebab.name
       PO_TABLE                     = aws_dynamodb_table.purchase_orders.name
       WEB_UI_AUTH_TOKEN_SECRET_ARN = var.web_ui_auth_token_secret_arn
+      SENTRY_DSN                   = var.sentry_dsn
     }
   }
 

terraform/build_packages.sh

@@ -112,12 +112,14 @@ maybe_pip_install "${BUILD}/po_email_processor"
 
 copy_py_dir "po/web_ui" "${BUILD}/po_web_ui"
 copy_src_file "shared/web_ui_auth.py" "${BUILD}/po_web_ui/web_ui_auth.py"
+copy_src_file "shared/sentry_init.py" "${BUILD}/po_web_ui/sentry_init.py"
 if [[ -f "${SRC}/po/web_ui/requirements.txt" ]]; then
   copy_src_file "po/web_ui/requirements.txt" "${BUILD}/po_web_ui/requirements.txt"
 fi
 maybe_pip_install "${BUILD}/po_web_ui"
 
 copy_py_dir "po/site_extractor" "${BUILD}/po_site_extractor"
+copy_src_file "shared/sentry_init.py" "${BUILD}/po_site_extractor/sentry_init.py"
 if [[ -f "${SRC}/po/site_extractor/requirements.txt" ]]; then
   copy_src_file "po/site_extractor/requirements.txt" "${BUILD}/po_site_extractor/requirements.txt"
 fi
@@ -132,20 +134,32 @@ maybe_pip_install "${BUILD}/wo_email_processor"
 
 copy_py_dir "wo/web_ui" "${BUILD}/wo_web_ui"
 copy_src_file "shared/web_ui_auth.py" "${BUILD}/wo_web_ui/web_ui_auth.py"
+copy_src_file "shared/sentry_init.py" "${BUILD}/wo_web_ui/sentry_init.py"
 if [[ -f "${SRC}/wo/web_ui/requirements.txt" ]]; then
   copy_src_file "wo/web_ui/requirements.txt" "${BUILD}/wo_web_ui/requirements.txt"
 fi
 maybe_pip_install "${BUILD}/wo_web_ui"
 
 copy_py_dir "wo/shoc_emitter" "${BUILD}/wo_shoc_emitter"
+copy_src_file "shared/sentry_init.py" "${BUILD}/wo_shoc_emitter/sentry_init.py"
+if [[ -f "${SRC}/wo/shoc_emitter/requirements.txt" ]]; then
+  copy_src_file "wo/shoc_emitter/requirements.txt" "${BUILD}/wo_shoc_emitter/requirements.txt"
+fi
+maybe_pip_install "${BUILD}/wo_shoc_emitter"
 
 copy_py_dir "wo/shoc_hmac_rotator" "${BUILD}/wo_shoc_hmac_rotator"
+copy_src_file "shared/sentry_init.py" "${BUILD}/wo_shoc_hmac_rotator/sentry_init.py"
+if [[ -f "${SRC}/wo/shoc_hmac_rotator/requirements.txt" ]]; then
+  copy_src_file "wo/shoc_hmac_rotator/requirements.txt" "${BUILD}/wo_shoc_hmac_rotator/requirements.txt"
+fi
+maybe_pip_install "${BUILD}/wo_shoc_hmac_rotator"
 
 copy_py_dir "api" "${BUILD}/procurement_api"
 for f in openapi.json docs.html redoc.standalone.js fonts.css; do
   copy_src_file "api/${f}" "${BUILD}/procurement_api/${f}"
 done
 copy_src_file "shared/web_ui_auth.py" "${BUILD}/procurement_api/web_ui_auth.py"
+copy_src_file "shared/sentry_init.py" "${BUILD}/procurement_api/sentry_init.py"
 if [[ -f "${SRC}/api/requirements.txt" ]]; then
   copy_src_file "api/requirements.txt" "${BUILD}/procurement_api/requirements.txt"
 fi

terraform/po_lambda.tf

@@ -47,6 +47,7 @@ resource "aws_lambda_function" "po_email_processor" {
       PO_TABLE             = aws_dynamodb_table.purchase_orders.name
       ALLOWED_DKIM_DOMAINS = "amazon.coupahost.com"
       BEDROCK_MODEL_ID     = local.bedrock_model_id
+      SENTRY_DSN           = var.sentry_dsn
     }
   }
 
@@ -79,6 +80,7 @@ resource "aws_lambda_function" "po_web_ui" {
     variables = {
       PO_TABLE                     = aws_dynamodb_table.purchase_orders.name
       WEB_UI_AUTH_TOKEN_SECRET_ARN = var.web_ui_auth_token_secret_arn
+      SENTRY_DSN                   = var.sentry_dsn
     }
   }
 
@@ -109,6 +111,7 @@ resource "aws_lambda_function" "po_site_extractor" {
     variables = {
       VERIFIED_SITES_TABLE = aws_dynamodb_table.verified_sites.name
       PENDING_REVIEW_TABLE = aws_dynamodb_table.pending_site_review.name
+      SENTRY_DSN           = var.sentry_dsn
     }
   }
 

terraform/variables.tf

@@ -4,6 +4,13 @@ variable "aws_region" {
   default     = "us-east-1"
 }
 
+variable "sentry_dsn" {
+  type        = string
+  sensitive   = true
+  default     = ""
+  description = "Sentry DSN. Empty disables the SDK. Set in HCP, never in git."
+}
+
 variable "web_ui_auth_token_secret_arn" {
   type        = string
   description = "Secrets Manager ARN for the shared web UI / docs auth token (exact ARN, including suffix)"

terraform/wo_lambda.tf

@@ -39,6 +39,7 @@ resource "aws_lambda_function" "wo_email_processor" {
       COMMENTS_TABLE       = aws_dynamodb_table.work_order_comments_kebab.name
       ALLOWED_DKIM_DOMAINS = "seahaven.com"
       BEDROCK_MODEL_ID     = local.bedrock_model_id
+      SENTRY_DSN           = var.sentry_dsn
     }
   }
 
@@ -71,6 +72,7 @@ resource "aws_lambda_function" "wo_web_ui" {
       WORK_ORDERS_TABLE            = aws_dynamodb_table.work_orders_kebab.name
       COMMENTS_TABLE               = aws_dynamodb_table.work_order_comments_kebab.name
       WEB_UI_AUTH_TOKEN_SECRET_ARN = var.web_ui_auth_token_secret_arn
+      SENTRY_DSN                   = var.sentry_dsn
     }
   }
 

terraform/wo_shoc.tf

@@ -188,6 +188,12 @@ resource "aws_lambda_function" "wo_shoc_hmac_rotator" {
   s3_key           = aws_s3_object.lambda["wo_shoc_hmac_rotator"].key
   source_code_hash = data.archive_file.lambda["wo_shoc_hmac_rotator"].output_base64sha256
 
+  environment {
+    variables = {
+      SENTRY_DSN = var.sentry_dsn
+    }
+  }
+
   depends_on = [
     aws_s3_object.lambda,
     aws_cloudwatch_log_group.wo_shoc_hmac_rotator,
@@ -295,6 +301,7 @@ resource "aws_lambda_function" "wo_shoc_emitter" {
       # Stream ARN classification (PLAT-11); must match ESM source table names.
       WORK_ORDERS_TABLE = aws_dynamodb_table.work_orders_kebab.name
       COMMENTS_TABLE    = aws_dynamodb_table.work_order_comments_kebab.name
+      SENTRY_DSN        = var.sentry_dsn
     }
   }
 

tests/requirements.txt

@@ -1,2 +1,3 @@
 moto==5.2.2
 pytest-cov==7.1.0
+sentry-sdk==2.68.1

tests/support/loader.py

@@ -44,12 +44,14 @@ _SHARED_DIR = REPO_ROOT / "lambdas" / "shared"
 # web_ui_auth was added (no dependencies; after emf) so
 # load_lambda_module("po"|"wo", "web_ui/handler") can bind the web_ui handlers'
 # bare `from web_ui_auth import is_authenticated` (lambdas/po/web_ui/handler.py:15
-# and the wo equivalent) via the same shared-dir fallback.
+# and the wo equivalent) via the same shared-dir fallback. sentry_init is the
+# same shape: every handler does `import sentry_init` (side-effect SDK init).
 _SIBLING_MODULES = (
     "ses_auth",
     "email_parsing",
     "emf",
     "web_ui_auth",
+    "sentry_init",
     # procurement-api siblings (lambdas/api/): pagination/serialization/router
     # have no sibling deps; wo_repo/po_repo import pagination, so they follow
     # it. These names exist only under lambdas/api/, so the po/wo handler

tests/test_bundle_consistency.py

@@ -43,7 +43,9 @@ SHARED_DIR = REPO_ROOT / "lambdas" / "shared"
 # the same name already expected in shared/ mask a pipeline-dir stray) --
 # see test_no_shared_module_shadow_in_pipeline_dirs and the per-dir exact-set
 # pins below.
-SHARED_MODULES = frozenset({"ses_auth", "email_parsing", "emf", "web_ui_auth"})
+SHARED_MODULES = frozenset(
+    {"ses_auth", "email_parsing", "emf", "web_ui_auth", "sentry_init"}
+)
 
 # Exact top-level .py stems each dir must hold. Pinned as exact sets (both
 # bounds): copy_py_dir ships every top-level .py in these dirs, so a stray
@@ -333,7 +335,7 @@ def test_shared_dir_ships_no_unexpected_top_level_modules():
 
     copy_shared_all ships every top-level .py under lambdas/shared/ into BOTH
     email-processor bundles, so a stray scratch/secrets .py here would leak into
-    both production zips. Pinned to exactly the four single-sourced modules.
+    both production zips. Pinned to exactly the single-sourced shared modules.
     """
     top_level = {p.stem for p in SHARED_DIR.glob("*.py")}
     assert top_level == set(SHARED_MODULES), (
@@ -398,6 +400,7 @@ def test_detection_logic_catches_allowlist_missing_a_sibling():
             "po/email_processor/extraction.py",
             "po/email_processor/enrichment.py",
             "po/email_processor/persistence.py",
+            "shared/sentry_init.py",
         ]
     )
     assert _packaging_ships_all(complete, siblings)
@@ -532,9 +535,10 @@ def test_api_bundle_stages_spec_and_docs_page():
 def test_shoc_emitter_bundling_ships_all_first_party_siblings():
     """The SHOC emitter package must ship every sibling handler.py imports."""
     required = _first_party_sibling_imports(SHOC_EMITTER_HANDLER)
-    assert required == {"envelope", "delivery"}, (
+    assert required == {"envelope", "delivery", "sentry_init"}, (
         f"expected the emitter handler's first-party siblings to be envelope + "
-        f"delivery, found {sorted(required)} — update this pin deliberately"
+        f"delivery + sentry_init, found {sorted(required)} — update this pin "
+        "deliberately"
     )
     recipes = _parse_build_packages()
     recipe = recipes["wo_shoc_emitter"]
@@ -548,19 +552,20 @@ def test_shoc_emitter_bundling_ships_all_first_party_siblings():
 
 
 def test_shoc_rotator_bundling_ships_handler():
-    """The rotator package must ship handler.py (it has no first-party siblings)."""
+    """The rotator package must ship handler.py and shared sentry_init."""
     required = _first_party_sibling_imports(SHOC_ROTATOR_HANDLER)
-    assert required == set(), (
-        f"the rotator handler grew first-party sibling imports "
-        f"{sorted(required)} — extend this ships-all test to require them"
+    assert required == {"sentry_init"}, (
+        f"the rotator handler first-party siblings {sorted(required)} — "
+        "expected only sentry_init; extend this ships-all test if more appear"
     )
     recipes = _parse_build_packages()
     recipe = recipes["wo_shoc_hmac_rotator"]
     assert "wo/shoc_hmac_rotator" in recipe.py_dirs, (
         f"wo_shoc_hmac_rotator must copy_py_dir wo/shoc_hmac_rotator. Recipe: {recipe}"
     )
-    assert _packaging_ships_all(recipe, {"handler"}), (
-        f"wo_shoc_hmac_rotator packaging does not ship handler.py. Recipe: {recipe}"
+    assert _packaging_ships_all(recipe, {"handler", "sentry_init"}), (
+        f"wo_shoc_hmac_rotator packaging does not ship handler.py + sentry_init. "
+        f"Recipe: {recipe}"
     )
 
 
@@ -597,3 +602,22 @@ def test_email_processor_packages_do_not_collide_with_web_ui_dirs():
         assert not any("email_processor" in d for d in recipe.py_dirs), (
             f"{name} packaging unexpectedly copies an email_processor dir: {recipe}"
         )
+
+
+def test_non_email_processor_packages_copy_sentry_init():
+    """Functions that do not copy_shared_all must copy sentry_init by name."""
+    recipes = _parse_build_packages()
+    for name in (
+        "po_web_ui",
+        "wo_web_ui",
+        "procurement_api",
+        "po_site_extractor",
+        "wo_shoc_emitter",
+        "wo_shoc_hmac_rotator",
+    ):
+        recipe = recipes[name]
+        assert "shared/sentry_init.py" in recipe.src_files, (
+            f"terraform/build_packages.sh must copy_src_file shared/sentry_init.py "
+            f"into {name} so `import sentry_init` resolves at cold start. "
+            f"Recipe: {recipe}"
+        )

tests/test_sentry_init.py

@@ -0,0 +1,102 @@
+"""sentry_init: DSN no-op, init options, and before_send scrub."""
+
+from unittest.mock import patch
+
+import pytest
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+from tests.support import load_lambda_module
+
+
+@pytest.fixture
+def sentry_mod():
+    return load_lambda_module("shared", "sentry_init")
+
+
+def _reexec(mod, monkeypatch, dsn=None):
+    if dsn is None:
+        monkeypatch.delenv("SENTRY_DSN", raising=False)
+    else:
+        monkeypatch.setenv("SENTRY_DSN", dsn)
+    with patch("sentry_sdk.init") as mocked:
+        mod.__spec__.loader.exec_module(mod)
+        return mocked
+
+
+def test_unset_dsn_does_not_init(sentry_mod, monkeypatch):
+    mocked = _reexec(sentry_mod, monkeypatch, dsn=None)
+    mocked.assert_not_called()
+
+
+def test_empty_dsn_does_not_init(sentry_mod, monkeypatch):
+    mocked = _reexec(sentry_mod, monkeypatch, dsn="")
+    mocked.assert_not_called()
+
+
+def test_set_dsn_inits_lambda_integration(sentry_mod, monkeypatch):
+    mocked = _reexec(sentry_mod, monkeypatch, dsn="https://key@o1.ingest.sentry.io/1")
+    mocked.assert_called_once()
+    kwargs = mocked.call_args.kwargs
+    assert kwargs["dsn"] == "https://key@o1.ingest.sentry.io/1"
+    assert kwargs["send_default_pii"] is False
+    assert kwargs["enable_logs"] is False
+    assert kwargs["traces_sample_rate"] == 0.0
+    assert kwargs["before_send"] is sentry_mod._before_send
+    integrations = kwargs["integrations"]
+    assert len(integrations) == 1
+    assert isinstance(integrations[0], AwsLambdaIntegration)
+    assert integrations[0].timeout_warning is True
+
+
+def test_before_send_strips_auth_and_sigv4_headers(sentry_mod):
+    event = {
+        "request": {
+            "headers": {
+                "Authorization": "Bearer secret",
+                "X-Auth-Token": "tok",
+                "X-Amz-Date": "20260101T000000Z",
+                "Content-Type": "application/json",
+            },
+            "url": "https://procurement-api.seahaven.com/work-orders",
+        }
+    }
+    out = sentry_mod._before_send(event, {})
+    assert out["request"]["headers"] == {"Content-Type": "application/json"}
+    assert out["request"]["url"] == "https://procurement-api.seahaven.com/work-orders"
+
+
+def test_before_send_strips_list_headers(sentry_mod):
+    event = {
+        "request": {
+            "headers": [
+                ("Authorization", "Bearer secret"),
+                ("Content-Type", "application/json"),
+            ]
+        }
+    }
+    out = sentry_mod._before_send(event, {})
+    assert out["request"]["headers"] == [("Content-Type", "application/json")]
+
+
+def test_before_send_drops_body_prompt_and_secret_keys(sentry_mod):
+    event = {
+        "request": {
+            "body": "<email>raw mime</email>",
+            "data": {"prompt": "EXTRACT"},
+            "method": "POST",
+        },
+        "extra": {
+            "raw_email": "From: attacker",
+            "bedrock_prompt": "ignore previous",
+            "hmac_secret": "aabbcc",
+            "po_number": "123",
+        },
+    }
+    out = sentry_mod._before_send(event, {})
+    assert "body" not in out["request"]
+    assert "data" not in out["request"]
+    assert out["request"]["method"] == "POST"
+    assert "raw_email" not in out["extra"]
+    assert "bedrock_prompt" not in out["extra"]
+    assert "hmac_secret" not in out["extra"]
+    assert out["extra"]["po_number"] == "123"