From 6eb2e52a74b44990dfab421f8c7de93934f44530 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:52:28 +0000 Subject: [PATCH] feat(observability): add Sentry error reporting to Lambdas (#241) Unhandled errors and timeout warnings go to Sentry when SENTRY_DSN is set; Slack and 3CX secrets are stripped before send. --- src/holiday-router/app.py | 1 + src/release-notifier/app.py | 1 + src/ring-scheduler/app.py | 1 + src/roster-sync/app.py | 3 +- src/shared/requirements.txt | 2 + src/shared/shared/sentry_init.py | 138 ++++++++++++++++++++++++++ src/slack-bot/handler.py | 1 + src/weekly-post/app.py | 1 + template.yaml | 9 ++ tests/conftest.py | 1 + tests/requirements.txt | 2 + tests/shared/test_sentry_init.py | 165 +++++++++++++++++++++++++++++++ 12 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 src/shared/shared/sentry_init.py create mode 100644 tests/shared/test_sentry_init.py diff --git a/src/holiday-router/app.py b/src/holiday-router/app.py index edfb71f..13f6077 100644 --- a/src/holiday-router/app.py +++ b/src/holiday-router/app.py @@ -27,6 +27,7 @@ import json import logging import os +import shared.sentry_init # noqa: F401 from shared.schedule import FALLBACK_EXTENSION, ShiftSchedule from shared.secrets import get_secret from shared.three_cx_client import ThreeCXClient diff --git a/src/release-notifier/app.py b/src/release-notifier/app.py index d54d4d7..41a7ece 100644 --- a/src/release-notifier/app.py +++ b/src/release-notifier/app.py @@ -15,6 +15,7 @@ import os from slack_sdk import WebClient +import shared.sentry_init # noqa: F401 from shared.blocks import build_release_announcement_blocks from shared.secrets import get_secret diff --git a/src/ring-scheduler/app.py b/src/ring-scheduler/app.py index 48cdfb5..77424e7 100644 --- a/src/ring-scheduler/app.py +++ b/src/ring-scheduler/app.py @@ -10,6 +10,7 @@ import os from datetime import datetime from zoneinfo import ZoneInfo +import shared.sentry_init # noqa: F401 from shared.ring_scheduler import update_queue_routing from shared.schedule import ( FALLBACK_EXTENSION, diff --git a/src/roster-sync/app.py b/src/roster-sync/app.py index 116f097..86f0e8f 100644 --- a/src/roster-sync/app.py +++ b/src/roster-sync/app.py @@ -10,9 +10,10 @@ import os from datetime import datetime from zoneinfo import ZoneInfo -from shared.three_cx_client import ThreeCXClient +import shared.sentry_init # noqa: F401 from shared.schedule import ShiftSchedule from shared.secrets import get_secret +from shared.three_cx_client import ThreeCXClient logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/src/shared/requirements.txt b/src/shared/requirements.txt index 58bd4d8..376ef7f 100644 --- a/src/shared/requirements.txt +++ b/src/shared/requirements.txt @@ -1,2 +1,4 @@ boto3>=1.43.78 requests>=2.34.2 +sentry-sdk==2.68.1 + diff --git a/src/shared/shared/sentry_init.py b/src/shared/shared/sentry_init.py new file mode 100644 index 0000000..e7b6bd5 --- /dev/null +++ b/src/shared/shared/sentry_init.py @@ -0,0 +1,138 @@ +"""Shared Sentry SDK init for every afterhours-shift-manager Lambda. + +Imported for side effect from each handler. ``init_sentry()`` is a no-op when +``SENTRY_DSN`` is unset so pytest, local invokes, and a missing SAM parameter +never talk to Sentry. ``before_send`` strips auth and Slack signing headers, +drops request/extra keys that can hold Slack payloads or 3CX credential +material, and removes exception stack-frame locals. +``include_local_variables=False`` keeps those locals out of the event in the +first place. +""" + +import os + +import sentry_sdk +from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration + +_HEADER_DROP_NAMES = frozenset( + { + "authorization", + "x-auth-token", + "cookie", + "x-amz-security-token", + "x-slack-signature", + } +) +_DROP_REQUEST_KEYS = frozenset( + { + "body", + "Body", + "data", + "cookies", + "raw_email", + "prompt", + "secret", + "SecretString", + "hmac", + "keys", + } +) +_DROP_EXTRA_NEEDLES = ( + "body", + "email", + "prompt", + "secret", + "hmac", + "token", + "mime", + "raw_email", + "password", + "signing", +) + + +def _drop_header(name): + lower = str(name).lower() + return lower in _HEADER_DROP_NAMES or lower.startswith("x-amz-") + + +def _scrub_headers(headers): + if isinstance(headers, dict): + return {k: v for k, v in headers.items() if not _drop_header(k)} + if isinstance(headers, list): + kept = [] + for pair in headers: + if isinstance(pair, (list, tuple)) and pair and _drop_header(pair[0]): + continue + kept.append(pair) + return kept + return headers + + +def _stacktraces(event): + traces = [] + stacktrace = event.get("stacktrace") + if isinstance(stacktrace, dict): + traces.append(stacktrace) + for section in ("exception", "threads"): + container = event.get(section) + if not isinstance(container, dict): + continue + values = container.get("values") + if not isinstance(values, list): + continue + for item in values: + if not isinstance(item, dict): + continue + inner = item.get("stacktrace") + if isinstance(inner, dict): + traces.append(inner) + return traces + + +def _strip_stack_locals(event): + """Drop frame locals. Names like ``raw``/``item`` still hold secrets.""" + for stacktrace in _stacktraces(event): + frames = stacktrace.get("frames") + if not isinstance(frames, list): + continue + for frame in frames: + if isinstance(frame, dict): + frame.pop("vars", None) + + +def _before_send(event, _hint): + request = event.get("request") + if isinstance(request, dict): + headers = request.get("headers") + if headers is not None: + request["headers"] = _scrub_headers(headers) + for key in list(request): + if key in _DROP_REQUEST_KEYS or str(key).lower() in {"body", "data"}: + request.pop(key, None) + extra = event.get("extra") + if isinstance(extra, dict): + for key in list(extra): + lower = str(key).lower() + if any(needle in lower for needle in _DROP_EXTRA_NEEDLES): + extra.pop(key, None) + _strip_stack_locals(event) + return event + + +def init_sentry(): + dsn = os.environ.get("SENTRY_DSN") + if not dsn: + return + sentry_sdk.init( + dsn=dsn, + integrations=[AwsLambdaIntegration(timeout_warning=True)], + send_default_pii=False, + include_local_variables=False, + enable_logs=False, + traces_sample_rate=0.0, + before_send=_before_send, + ) + + +init_sentry() diff --git a/src/slack-bot/handler.py b/src/slack-bot/handler.py index 3a3e2fc..11c899f 100644 --- a/src/slack-bot/handler.py +++ b/src/slack-bot/handler.py @@ -5,6 +5,7 @@ import os from slack_bolt.adapter.aws_lambda import SlackRequestHandler +import shared.sentry_init # noqa: F401 from app import create_app from shared.secrets import get_secret diff --git a/src/weekly-post/app.py b/src/weekly-post/app.py index 45d5744..b17c9ab 100644 --- a/src/weekly-post/app.py +++ b/src/weekly-post/app.py @@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo import boto3 from slack_sdk import WebClient +import shared.sentry_init # noqa: F401 from shared.blocks import build_pay_summary_blocks, build_week_schedule from shared.schedule import FALLBACK_EXTENSION, WEEKEND_DAYS, ShiftSchedule from shared.secrets import get_secret diff --git a/template.yaml b/template.yaml index 9e3145d..3c43a92 100644 --- a/template.yaml +++ b/template.yaml @@ -13,6 +13,11 @@ Parameters: Type: String Default: "801" Description: 3CX queue extension number to update + SentryDsn: + Type: String + Default: "" + NoEcho: true + Description: Sentry DSN; empty disables error reporting Globals: Function: @@ -22,6 +27,10 @@ Globals: Architectures: - arm64 PermissionsBoundary: arn:aws:iam::328440206208:policy/seahaven-lambda-execution-boundary + Environment: + Variables: + SENTRY_DSN: !Ref SentryDsn + # Access logging + default throttling on the implicit HTTP API (audit M-18). HttpApi: AccessLogSettings: diff --git a/tests/conftest.py b/tests/conftest.py index f307f29..0cdb295 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ def aws_env(monkeypatch): monkeypatch.setenv("SHIFT_CHANNEL", "C_TEST") monkeypatch.delenv("QUEUE_NUMBER", raising=False) monkeypatch.delenv("TCX_SECRET_PREFIX", raising=False) + monkeypatch.delenv("SENTRY_DSN", raising=False) def _create_table(dynamodb): diff --git a/tests/requirements.txt b/tests/requirements.txt index 39d4cdf..91deca3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,3 +5,5 @@ pytest>=9.1.1 moto[dynamodb,ses,secretsmanager]>=5.2.2 responses>=0.26.2 freezegun>=1.5.5 +sentry-sdk==2.68.1 + diff --git a/tests/shared/test_sentry_init.py b/tests/shared/test_sentry_init.py new file mode 100644 index 0000000..53051d8 --- /dev/null +++ b/tests/shared/test_sentry_init.py @@ -0,0 +1,165 @@ +"""sentry_init: DSN no-op, init options, and before_send scrub.""" + +import importlib +from unittest.mock import patch + +from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration + +import shared.sentry_init as sentry_mod + + +def _reexec(monkeypatch, dsn=None): + if dsn is None: + monkeypatch.delenv("SENTRY_DSN", raising=False) + else: + monkeypatch.setenv("SENTRY_DSN", dsn) + with patch("sentry_sdk.init") as mocked: + importlib.reload(sentry_mod) + return mocked + + +def test_unset_dsn_does_not_init(monkeypatch): + mocked = _reexec(monkeypatch, dsn=None) + mocked.assert_not_called() + + +def test_empty_dsn_does_not_init(monkeypatch): + mocked = _reexec(monkeypatch, dsn="") + mocked.assert_not_called() + + +def test_set_dsn_inits_lambda_integration(monkeypatch): + mocked = _reexec(monkeypatch, dsn="https://key@o1.ingest.sentry.io/1") + mocked.assert_called_once() + kwargs = mocked.call_args.kwargs + assert kwargs["dsn"] == "https://key@o1.ingest.sentry.io/1" + assert kwargs["send_default_pii"] is False + assert kwargs["include_local_variables"] is False + assert kwargs["enable_logs"] is False + assert kwargs["traces_sample_rate"] == 0.0 + assert kwargs["before_send"] is sentry_mod._before_send + integrations = kwargs["integrations"] + assert len(integrations) == 1 + assert isinstance(integrations[0], AwsLambdaIntegration) + assert integrations[0].timeout_warning is True + + +def test_before_send_strips_auth_and_sigv4_headers(): + event = { + "request": { + "headers": { + "Authorization": "Bearer secret", + "X-Auth-Token": "tok", + "X-Amz-Date": "20260101T000000Z", + "Content-Type": "application/json", + }, + "url": "https://example.invalid/oncall", + } + } + out = sentry_mod._before_send(event, {}) + assert out["request"]["headers"] == {"Content-Type": "application/json"} + assert out["request"]["url"] == "https://example.invalid/oncall" + + +def test_before_send_strips_slack_signature_header(): + event = { + "request": { + "headers": { + "X-Slack-Signature": "v0=abc", + "X-Slack-Request-Timestamp": "123", + "Content-Type": "application/json", + } + } + } + out = sentry_mod._before_send(event, {}) + assert out["request"]["headers"] == { + "X-Slack-Request-Timestamp": "123", + "Content-Type": "application/json", + } + + +def test_before_send_strips_list_headers(): + event = { + "request": { + "headers": [ + ("Authorization", "Bearer secret"), + ("X-Slack-Signature", "v0=abc"), + ("Content-Type", "application/json"), + ] + } + } + out = sentry_mod._before_send(event, {}) + assert out["request"]["headers"] == [("Content-Type", "application/json")] + + +def test_before_send_drops_body_and_secret_keys(): + event = { + "request": { + "body": "token=xoxb-secret&command=/oncall", + "data": {"signing_secret": "abc"}, + "method": "POST", + }, + "extra": { + "slack_signing_secret": "abc", + "tcx_password": "hunter2", + "bot_token": "xoxb-secret", + "hmac_secret": "aabbcc", + "shift_date": "2026-08-29", + }, + } + out = sentry_mod._before_send(event, {}) + assert "body" not in out["request"] + assert "data" not in out["request"] + assert out["request"]["method"] == "POST" + assert "slack_signing_secret" not in out["extra"] + assert "tcx_password" not in out["extra"] + assert "bot_token" not in out["extra"] + assert "hmac_secret" not in out["extra"] + assert out["extra"]["shift_date"] == "2026-08-29" + + +def test_before_send_drops_exception_and_thread_frame_locals(): + event = { + "exception": { + "values": [ + { + "stacktrace": { + "frames": [ + { + "function": "handler", + "vars": { + "signing_secret": "abc", + "SecretString": "aabbcc", + }, + } + ] + } + } + ] + }, + "threads": { + "values": [ + { + "stacktrace": { + "frames": [ + { + "function": "_authenticate_user", + "vars": {"password": "hunter2"}, + } + ] + } + } + ] + }, + "stacktrace": { + "frames": [{"function": "get_secret", "vars": {"item": {"token": "x"}}}] + }, + } + out = sentry_mod._before_send(event, {}) + assert "vars" not in out["exception"]["values"][0]["stacktrace"]["frames"][0] + assert "vars" not in out["threads"]["values"][0]["stacktrace"]["frames"][0] + assert "vars" not in out["stacktrace"]["frames"][0] + assert ( + out["exception"]["values"][0]["stacktrace"]["frames"][0]["function"] + == "handler" + )