feat(observability): add Sentry error reporting to Lambdas (#241)

Unhandled errors and timeout warnings go to Sentry when SENTRY_DSN is set; Slack and 3CX secrets are stripped before send.

src/holiday-router/app.py

@@ -27,6 +27,7 @@ import json
 import logging
 import os
 
+import shared.sentry_init  # noqa: F401
 from shared.schedule import FALLBACK_EXTENSION, ShiftSchedule
 from shared.secrets import get_secret
 from shared.three_cx_client import ThreeCXClient

src/release-notifier/app.py

@@ -15,6 +15,7 @@ import os
 
 from slack_sdk import WebClient
 
+import shared.sentry_init  # noqa: F401
 from shared.blocks import build_release_announcement_blocks
 from shared.secrets import get_secret
 

src/ring-scheduler/app.py

@@ -10,6 +10,7 @@ import os
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
+import shared.sentry_init  # noqa: F401
 from shared.ring_scheduler import update_queue_routing
 from shared.schedule import (
     FALLBACK_EXTENSION,

src/roster-sync/app.py

@@ -10,9 +10,10 @@ import os
 from datetime import datetime
 from zoneinfo import ZoneInfo
 
-from shared.three_cx_client import ThreeCXClient
+import shared.sentry_init  # noqa: F401
 from shared.schedule import ShiftSchedule
 from shared.secrets import get_secret
+from shared.three_cx_client import ThreeCXClient
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)

src/shared/requirements.txt

@@ -1,2 +1,4 @@
 boto3>=1.43.78
 requests>=2.34.2
+sentry-sdk==2.68.1
+

src/shared/shared/sentry_init.py

@@ -0,0 +1,138 @@
+"""Shared Sentry SDK init for every afterhours-shift-manager Lambda.
+
+Imported for side effect from each handler. ``init_sentry()`` is a no-op when
+``SENTRY_DSN`` is unset so pytest, local invokes, and a missing SAM parameter
+never talk to Sentry. ``before_send`` strips auth and Slack signing headers,
+drops request/extra keys that can hold Slack payloads or 3CX credential
+material, and removes exception stack-frame locals.
+``include_local_variables=False`` keeps those locals out of the event in the
+first place.
+"""
+
+import os
+
+import sentry_sdk
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+_HEADER_DROP_NAMES = frozenset(
+    {
+        "authorization",
+        "x-auth-token",
+        "cookie",
+        "x-amz-security-token",
+        "x-slack-signature",
+    }
+)
+_DROP_REQUEST_KEYS = frozenset(
+    {
+        "body",
+        "Body",
+        "data",
+        "cookies",
+        "raw_email",
+        "prompt",
+        "secret",
+        "SecretString",
+        "hmac",
+        "keys",
+    }
+)
+_DROP_EXTRA_NEEDLES = (
+    "body",
+    "email",
+    "prompt",
+    "secret",
+    "hmac",
+    "token",
+    "mime",
+    "raw_email",
+    "password",
+    "signing",
+)
+
+
+def _drop_header(name):
+    lower = str(name).lower()
+    return lower in _HEADER_DROP_NAMES or lower.startswith("x-amz-")
+
+
+def _scrub_headers(headers):
+    if isinstance(headers, dict):
+        return {k: v for k, v in headers.items() if not _drop_header(k)}
+    if isinstance(headers, list):
+        kept = []
+        for pair in headers:
+            if isinstance(pair, (list, tuple)) and pair and _drop_header(pair[0]):
+                continue
+            kept.append(pair)
+        return kept
+    return headers
+
+
+def _stacktraces(event):
+    traces = []
+    stacktrace = event.get("stacktrace")
+    if isinstance(stacktrace, dict):
+        traces.append(stacktrace)
+    for section in ("exception", "threads"):
+        container = event.get(section)
+        if not isinstance(container, dict):
+            continue
+        values = container.get("values")
+        if not isinstance(values, list):
+            continue
+        for item in values:
+            if not isinstance(item, dict):
+                continue
+            inner = item.get("stacktrace")
+            if isinstance(inner, dict):
+                traces.append(inner)
+    return traces
+
+
+def _strip_stack_locals(event):
+    """Drop frame locals. Names like ``raw``/``item`` still hold secrets."""
+    for stacktrace in _stacktraces(event):
+        frames = stacktrace.get("frames")
+        if not isinstance(frames, list):
+            continue
+        for frame in frames:
+            if isinstance(frame, dict):
+                frame.pop("vars", None)
+
+
+def _before_send(event, _hint):
+    request = event.get("request")
+    if isinstance(request, dict):
+        headers = request.get("headers")
+        if headers is not None:
+            request["headers"] = _scrub_headers(headers)
+        for key in list(request):
+            if key in _DROP_REQUEST_KEYS or str(key).lower() in {"body", "data"}:
+                request.pop(key, None)
+    extra = event.get("extra")
+    if isinstance(extra, dict):
+        for key in list(extra):
+            lower = str(key).lower()
+            if any(needle in lower for needle in _DROP_EXTRA_NEEDLES):
+                extra.pop(key, None)
+    _strip_stack_locals(event)
+    return event
+
+
+def init_sentry():
+    dsn = os.environ.get("SENTRY_DSN")
+    if not dsn:
+        return
+    sentry_sdk.init(
+        dsn=dsn,
+        integrations=[AwsLambdaIntegration(timeout_warning=True)],
+        send_default_pii=False,
+        include_local_variables=False,
+        enable_logs=False,
+        traces_sample_rate=0.0,
+        before_send=_before_send,
+    )
+
+
+init_sentry()

src/slack-bot/handler.py

@@ -5,6 +5,7 @@ import os
 
 from slack_bolt.adapter.aws_lambda import SlackRequestHandler
 
+import shared.sentry_init  # noqa: F401
 from app import create_app
 from shared.secrets import get_secret
 

src/weekly-post/app.py

@@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo
 import boto3
 from slack_sdk import WebClient
 
+import shared.sentry_init  # noqa: F401
 from shared.blocks import build_pay_summary_blocks, build_week_schedule
 from shared.schedule import FALLBACK_EXTENSION, WEEKEND_DAYS, ShiftSchedule
 from shared.secrets import get_secret

template.yaml

@@ -13,6 +13,11 @@ Parameters:
     Type: String
     Default: "801"
     Description: 3CX queue extension number to update
+  SentryDsn:
+    Type: String
+    Default: ""
+    NoEcho: true
+    Description: Sentry DSN; empty disables error reporting
 
 Globals:
   Function:
@@ -22,6 +27,10 @@ Globals:
     Architectures:
       - arm64
     PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary
+    Environment:
+      Variables:
+        SENTRY_DSN: !Ref SentryDsn
+
   # Access logging + default throttling on the implicit HTTP API (audit M-18).
   HttpApi:
     AccessLogSettings:

tests/conftest.py

@@ -25,6 +25,7 @@ def aws_env(monkeypatch):
     monkeypatch.setenv("SHIFT_CHANNEL", "C_TEST")
     monkeypatch.delenv("QUEUE_NUMBER", raising=False)
     monkeypatch.delenv("TCX_SECRET_PREFIX", raising=False)
+    monkeypatch.delenv("SENTRY_DSN", raising=False)
 
 
 def _create_table(dynamodb):

tests/requirements.txt

@@ -5,3 +5,5 @@ pytest>=9.1.1
 moto[dynamodb,ses,secretsmanager]>=5.2.2
 responses>=0.26.2
 freezegun>=1.5.5
+sentry-sdk==2.68.1
+

tests/shared/test_sentry_init.py

@@ -0,0 +1,165 @@
+"""sentry_init: DSN no-op, init options, and before_send scrub."""
+
+import importlib
+from unittest.mock import patch
+
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+import shared.sentry_init as sentry_mod
+
+
+def _reexec(monkeypatch, dsn=None):
+    if dsn is None:
+        monkeypatch.delenv("SENTRY_DSN", raising=False)
+    else:
+        monkeypatch.setenv("SENTRY_DSN", dsn)
+    with patch("sentry_sdk.init") as mocked:
+        importlib.reload(sentry_mod)
+        return mocked
+
+
+def test_unset_dsn_does_not_init(monkeypatch):
+    mocked = _reexec(monkeypatch, dsn=None)
+    mocked.assert_not_called()
+
+
+def test_empty_dsn_does_not_init(monkeypatch):
+    mocked = _reexec(monkeypatch, dsn="")
+    mocked.assert_not_called()
+
+
+def test_set_dsn_inits_lambda_integration(monkeypatch):
+    mocked = _reexec(monkeypatch, dsn="https://key@o1.ingest.sentry.io/1")
+    mocked.assert_called_once()
+    kwargs = mocked.call_args.kwargs
+    assert kwargs["dsn"] == "https://key@o1.ingest.sentry.io/1"
+    assert kwargs["send_default_pii"] is False
+    assert kwargs["include_local_variables"] is False
+    assert kwargs["enable_logs"] is False
+    assert kwargs["traces_sample_rate"] == 0.0
+    assert kwargs["before_send"] is sentry_mod._before_send
+    integrations = kwargs["integrations"]
+    assert len(integrations) == 1
+    assert isinstance(integrations[0], AwsLambdaIntegration)
+    assert integrations[0].timeout_warning is True
+
+
+def test_before_send_strips_auth_and_sigv4_headers():
+    event = {
+        "request": {
+            "headers": {
+                "Authorization": "Bearer secret",
+                "X-Auth-Token": "tok",
+                "X-Amz-Date": "20260101T000000Z",
+                "Content-Type": "application/json",
+            },
+            "url": "https://example.invalid/oncall",
+        }
+    }
+    out = sentry_mod._before_send(event, {})
+    assert out["request"]["headers"] == {"Content-Type": "application/json"}
+    assert out["request"]["url"] == "https://example.invalid/oncall"
+
+
+def test_before_send_strips_slack_signature_header():
+    event = {
+        "request": {
+            "headers": {
+                "X-Slack-Signature": "v0=abc",
+                "X-Slack-Request-Timestamp": "123",
+                "Content-Type": "application/json",
+            }
+        }
+    }
+    out = sentry_mod._before_send(event, {})
+    assert out["request"]["headers"] == {
+        "X-Slack-Request-Timestamp": "123",
+        "Content-Type": "application/json",
+    }
+
+
+def test_before_send_strips_list_headers():
+    event = {
+        "request": {
+            "headers": [
+                ("Authorization", "Bearer secret"),
+                ("X-Slack-Signature", "v0=abc"),
+                ("Content-Type", "application/json"),
+            ]
+        }
+    }
+    out = sentry_mod._before_send(event, {})
+    assert out["request"]["headers"] == [("Content-Type", "application/json")]
+
+
+def test_before_send_drops_body_and_secret_keys():
+    event = {
+        "request": {
+            "body": "token=xoxb-secret&command=/oncall",
+            "data": {"signing_secret": "abc"},
+            "method": "POST",
+        },
+        "extra": {
+            "slack_signing_secret": "abc",
+            "tcx_password": "hunter2",
+            "bot_token": "xoxb-secret",
+            "hmac_secret": "aabbcc",
+            "shift_date": "2026-08-29",
+        },
+    }
+    out = sentry_mod._before_send(event, {})
+    assert "body" not in out["request"]
+    assert "data" not in out["request"]
+    assert out["request"]["method"] == "POST"
+    assert "slack_signing_secret" not in out["extra"]
+    assert "tcx_password" not in out["extra"]
+    assert "bot_token" not in out["extra"]
+    assert "hmac_secret" not in out["extra"]
+    assert out["extra"]["shift_date"] == "2026-08-29"
+
+
+def test_before_send_drops_exception_and_thread_frame_locals():
+    event = {
+        "exception": {
+            "values": [
+                {
+                    "stacktrace": {
+                        "frames": [
+                            {
+                                "function": "handler",
+                                "vars": {
+                                    "signing_secret": "abc",
+                                    "SecretString": "aabbcc",
+                                },
+                            }
+                        ]
+                    }
+                }
+            ]
+        },
+        "threads": {
+            "values": [
+                {
+                    "stacktrace": {
+                        "frames": [
+                            {
+                                "function": "_authenticate_user",
+                                "vars": {"password": "hunter2"},
+                            }
+                        ]
+                    }
+                }
+            ]
+        },
+        "stacktrace": {
+            "frames": [{"function": "get_secret", "vars": {"item": {"token": "x"}}}]
+        },
+    }
+    out = sentry_mod._before_send(event, {})
+    assert "vars" not in out["exception"]["values"][0]["stacktrace"]["frames"][0]
+    assert "vars" not in out["threads"]["values"][0]["stacktrace"]["frames"][0]
+    assert "vars" not in out["stacktrace"]["frames"][0]
+    assert (
+        out["exception"]["values"][0]["stacktrace"]["frames"][0]["function"]
+        == "handler"
+    )