From dcbb0606e6109fc9083d2259f6bd5e6f3794dd18 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:30:44 -0400 Subject: [PATCH 01/12] Add pytest config and conftest for test discovery --- pyproject.toml | 3 +++ tests/conftest.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 pyproject.toml create mode 100644 tests/conftest.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d86f343 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["lambdas/classifier", "lambdas/slack_post", "cdk"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..90357e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +"""Central test configuration. + +Sets dummy environment variables at import time so modules that read +``os.environ`` at module load (slackio, interactions, handler) can be imported +without real AWS credentials. Real values are monkeypatched per-test where +needed. +""" + +import os + +os.environ.setdefault("SLACK_SECRET_NAME", "test-slack-secret") +os.environ.setdefault("DASHBOARD_URL_PARAM", "/test/dashboard-url") +os.environ.setdefault("ANALYTICS_BUCKET", "test-bucket") From 1a126c817936d75c24990d993a7b11d24a9f2872 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:30:44 -0400 Subject: [PATCH 02/12] Enable test execution in CI --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f8c2130..b9da53a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,5 +12,5 @@ jobs: run-sam-validate: false run-cdk-synth: true cdk-dir: cdk - run-tests: false + run-tests: true enable-qemu: true From 817e1f6a74b16bb8f6364b5a1972336ac9207646 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:30:44 -0400 Subject: [PATCH 03/12] Add synthetic export fixture and run classification quality gate in CI --- tests/fixtures/sample_export.csv | 31 ++++++ tests/test_classify.py | 185 +++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 tests/fixtures/sample_export.csv diff --git a/tests/fixtures/sample_export.csv b/tests/fixtures/sample_export.csv new file mode 100644 index 0000000..25b7552 --- /dev/null +++ b/tests/fixtures/sample_export.csv @@ -0,0 +1,31 @@ +WO Number,WO Description,Equipment Code,Organization,Due Date,Department,WO Status,Hold Reason,Last Comment,Last Comment By,Last Comment Date,Contractor,Contractor Description +WO-1001,Repair HVAC unit at dock 3,HVAC-001,ABQ5,2026-05-15,SSP,H,REPORT,
3rd attempt process for schedule confirmation. Vendor please confirm schedule start date and proceed.
,J.Smith,2026-05-10,Acme HVAC,Acme HVAC Services +WO-1002,Replace lighting ballast,LIGHT-042,ACY9,2026-05-16,RME,H,SCHEDULING,
2nd attempt process for schedule confirmation. Please provide a confirmed date for this work order.
,T.Jones,2026-05-11,Bright Electric,Bright Electric LLC +WO-1003,Fix water leak in restroom,PLUMB-007,BOS1,2026-05-17,SSP,IP,,
1st escalation sent to vendor. Awaiting response from contractor regarding scheduling.
,M.Davis,2026-05-12,Metro Plumbing,Metro Plumbing Inc +WO-1004,Inspect fire suppression system,FIRE-015,ABQ5,2026-05-18,SSP,R,,
SIM ticket TT-12345678 opened for this work order. Tracking in Amazon SIM system.
,K.Wilson,2026-05-13,FireSafe Co,FireSafe Company +WO-1005,Service generator unit,GEN-003,ACY9,2026-05-19,RME,R,,
Site tech reported vendor was a no show for Tuesday. Rescheduling required.
,P.Brown,2026-05-14,Power Gen LLC,Power Generation LLC +WO-1006,Calibrate dock door sensors,DOCK-022,BOS1,2026-05-20,SSP,IP,,
WO schedule confirmed with vendor for 5/22. All parties notified.
,L.Garcia,2026-05-15,Dock Systems,Dock Systems Inc +WO-1007,Weekly PM on conveyor belt,CONV-011,ABQ5,2026-05-21,RME,IP,,
Weekly WO scheduled. Service reports required EOD Friday per standard cadence.
,R.Martinez,2026-05-16,Belt Tech,Belt Tech Services +WO-1008,Replace worn floor mats,FLOOR-009,ACY9,2026-05-22,BBM,H,REPORT,,A.Thompson,2026-05-17,Clean Facility,Clean Facility Services +WO-1009,Inspect sprinkler heads,FIRE-031,BOS1,2026-05-23,SSP,H,SCHEDULING,,B.Anderson,2026-05-18,FireSafe Co,FireSafe Company +WO-1010,Order replacement motor,MOTOR-005,ABQ5,2026-05-24,RME,H,VENDOR,,C.Jackson,2026-05-19,Motor Supply,Motor Supply Co +WO-1011,Cancel duplicate work order,WO-DUP-001,ACY9,2026-05-25,SSP,RCAN,,
WO cancelled, created in error. Duplicate of WO-1010.
,D.White,2026-05-20,N/A,N/A +WO-1012,Perform task and close,TASK-088,BOS1,2026-05-26,RME,H,REPORT,
Vendor arrived and performed task. All work completed satisfactorily.
,E.Harris,2026-05-21,General Contractors,General Contractors LLC +WO-1013,PM on roof HVAC,HVAC-099,ABQ5,2026-05-27,SSP,R,,
WO schedule confirmed with vendor for 5/29 start time 0800.
,F.Clark,2026-05-22,Acme HVAC,Acme HVAC Services +WO-1014,Service dock bumpers weekly,DOCK-044,ACY9,2026-05-28,SSP,IP,,
Weekly WO scheduled. Service reports are required weekly per maintenance plan.
,G.Lewis,2026-05-23,Dock Systems,Dock Systems Inc +WO-1015,Replace battery backup,UPS-002,BOS1,2026-05-29,RME,H,SCHEDULING,
2nd attempt process for schedule confirmation. Vendor has not responded to prior outreach.
,H.Robinson,2026-05-24,Power Gen LLC,Power Generation LLC +WO-1016,Inspect emergency exits,EXIT-007,ABQ5,2026-05-30,SSP,R,,
3rd attempt process for schedule confirmation. Escalating to vendor management for response.
,I.Walker,2026-05-25,Safety First,Safety First Inc +WO-1017,Fix broken dock seal,DOCK-055,ACY9,2026-05-31,SSP,IP,,
Vendor confirmed schedule for 6/2 arrival.
,J.Hall,2026-05-26,Dock Systems,Dock Systems Inc +WO-1018,Service fire extinguishers,FIRE-022,BOS1,2026-06-01,SSP,RCAN,,
WO cancelled. Work was already completed under WO-1004.
,K.Young,2026-05-27,FireSafe Co,FireSafe Company +WO-1019,PM on cooling tower,COOL-001,ABQ5,2026-06-02,RME,H,REPORT,,L.Allen,2026-05-28,Cooling Tech,Cooling Tech LLC +WO-1020,Inspect electrical panel,ELEC-018,ACY9,2026-06-03,RME,H,VENDOR,,M.King,2026-05-29,Volt Electric,Volt Electric Services +WO-1021,Replace worn casters on cart,CART-003,BOS1,2026-06-04,BBM,IP,,
Procurement to provide PO for parts order before work can begin.
,N.Wright,2026-05-30,General Contractors,General Contractors LLC +WO-1022,Patch roof leak,ROOF-006,ABQ5,2026-06-05,SSP,IP,,
Copy.
,O.Scott,2026-05-31,Roof Masters,Roof Masters Inc +WO-1023,Service dock bumpers,DOCK-066,ACY9,2026-06-06,SSP,R,,
Schedule confirmed with vendor for next Tuesday 0900.
,P.Green,2026-06-01,Dock Systems,Dock Systems Inc +WO-1024,Repair bathroom faucet,PLUMB-012,BOS1,2026-06-07,SSP,IP,,
WO schedule confirmed with vendor. Technician on site 6/9.
,Q.Baker,2026-06-02,Metro Plumbing,Metro Plumbing Inc +WO-1025,Recalibrate HVAC thermostat,HVAC-033,ABQ5,2026-06-08,RME,H,SCHEDULING,,R.Adams,2026-06-03,Acme HVAC,Acme HVAC Services +WO-1026,Fix overhead door,DOOR-019,ACY9,2026-06-09,SSP,IP,,
Uplift request submitted pending management approval.
,S.Nelson,2026-06-04,Door Pro,Door Pro LLC +WO-1027,Replace signage,SIGN-004,BOS1,2026-06-10,BBM,IP,,,,2026-06-05,Sign Works,Sign Works Co +WO-1028,Inspect conveyor belt,CONV-022,ABQ5,2026-06-11,RME,IP,,,T.Carter,2026-06-06,Belt Tech,Belt Tech Services +WO-1029,Lubricate dock equipment,DOCK-077,ACY9,2026-06-12,SSP,R,,,U.Mitchell,2026-06-07,Dock Systems,Dock Systems Inc +WO-1030,Weekly PM on generator,GEN-008,BOS1,2026-06-13,RME,IP,,
Weekly WO scheduled. Service reports required EOD Friday.
,V.Perez,2026-06-08,Power Gen LLC,Power Generation LLC diff --git a/tests/test_classify.py b/tests/test_classify.py index 4d8cff8..e3fdb03 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -9,9 +9,13 @@ Canonical fixture: ``~/Downloads/_documents/Sheet1-1.xlsx`` (347 rows, 13 cols). If the file is absent the export-driven tests skip. Run with the repo venv: ./.venv/bin/python -m pytest tests/test_classify.py -s -q + +CSV fixture (always present, runs in CI): ``tests/fixtures/sample_export.csv``. """ import collections +import csv +import re import sys from pathlib import Path @@ -30,10 +34,31 @@ COL_LAST_COMMENT = 8 FIXTURE = Path.home() / "Downloads" / "_documents" / "Sheet1-1.xlsx" +# Committed CSV fixture — always present, no skipif. +CSV_FIXTURE = Path(__file__).resolve().parent / "fixtures" / "sample_export.csv" + # Max acceptable deterministic "Other" share before any AI. CLAUDE.md: the # two-axis model lands ~9% before Haiku; we hold the line at single digits. MAX_OTHER_PCT = 10.0 +# Threshold for the committed CSV fixture. Measured deterministic Other% on +# the synthetic fixture: 7.41% (2 of 27 classified rows). Threshold is set +# with headroom but still comfortably single-digit. +CSV_FIXTURE_MAX_OTHER_PCT = 9.0 + + +# --------------------------------------------------------------------------- +# CSV loader helper (yields column-indexed tuples like openpyxl row values) +# --------------------------------------------------------------------------- + + +def _load_csv_rows(path: Path) -> list[tuple]: + """Read a 13-column CSV export; return data rows as tuples (header skipped).""" + with path.open(newline="") as fh: + reader = csv.reader(fh) + rows = list(reader) + return [tuple(r) for r in rows[1:]] # drop header row + def test_classification_constants_well_formed(): assert "3rd Escalation" in classify.ESCALATION_CATEGORIES @@ -244,3 +269,163 @@ def test_known_fixture_rows_in_export(): report_completion_mismatch[COL_LAST_COMMENT], ) assert mm is not None + + +# --------------------------------------------------------------------------- +# CSV fixture tests — always run (no skipif), so the quality gate fires in CI. +# --------------------------------------------------------------------------- + + +def test_other_share_against_csv_fixture(capsys): + """Classification quality gate against the committed synthetic CSV fixture. + + Measured deterministic Other%: 7.41% (2/27). Threshold: 9.0%. + This test runs unconditionally in CI. + """ + rows = _load_csv_rows(CSV_FIXTURE) + + dist: collections.Counter = collections.Counter() + mismatches = 0 + classified = 0 + blank = 0 + other_samples: list[str] = [] + + for row in rows: + comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else "" + # Blank-comment rows are excluded from the classified total by design. + if comment is None or str(comment).strip() == "": + blank += 1 + continue + classified += 1 + category, mismatch = classify.classify( + row[COL_WO_STATUS], row[COL_HOLD_REASON], comment + ) + dist[category] += 1 + if mismatch: + mismatches += 1 + if category == "Other" and len(other_samples) < 20: + other_samples.append(classify.strip_html(comment)[:90]) + + other = dist["Other"] + other_pct = other * 100.0 / classified if classified else 0.0 + + with capsys.disabled(): + print(f"\n=== APM classifier smoke test (CSV fixture): {CSV_FIXTURE.name} ===") + print( + f"rows={len(rows)} classified={classified} " + f"blank-excluded={blank} (blank-comment rows excluded from the total)" + ) + print("--- category distribution ---") + for cat, count in dist.most_common(): + print(f" {count:4d} {count * 100.0 / classified:5.1f}% {cat}") + print(f"--- Other: {other} ({other_pct:.2f}%) ---") + for sample in other_samples: + print(f" [Other] {sample}") + print(f"--- mismatches flagged: {mismatches} ---") + + assert classified > 0, "CSV fixture produced no classified rows" + assert other_pct <= CSV_FIXTURE_MAX_OTHER_PCT, ( + f"deterministic Other {other_pct:.2f}% exceeds {CSV_FIXTURE_MAX_OTHER_PCT}% — " + "the two-axis ladder regressed against the committed fixture" + ) + # Fixture must exercise mismatch detection (WO-1012: REPORT hold + performed task). + assert mismatches >= 1, "CSV fixture should contain at least one mismatch row" + + +def test_known_rows_in_csv_fixture(): + """Anchor checks on the synthetic CSV fixture — runs unconditionally in CI.""" + rows = _load_csv_rows(CSV_FIXTURE) + + by_status: collections.defaultdict = collections.defaultdict(list) + schedule_confirmed_row = None + report_completion_mismatch = None + third_esc_rows: list[tuple] = [] + cancelled_rows: list[tuple] = [] + structured_report_rows: list[tuple] = [] + structured_scheduling_rows: list[tuple] = [] + + for row in rows: + comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else "" + if comment is None or str(comment).strip() == "": + continue + text = classify.strip_html(comment) + by_status[row[COL_WO_STATUS]].append(row) + + if ( + schedule_confirmed_row is None + and "schedule confirmed with vendor" in text.lower() + and not (row[COL_HOLD_REASON] or "").strip() + ): + schedule_confirmed_row = row + if ( + report_completion_mismatch is None + and (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT" + and "performed task" in text.lower() + ): + report_completion_mismatch = row + if re.search(r"\b3rd\b.*\battempt\b", text.lower()): + third_esc_rows.append(row) + if row[COL_WO_STATUS] == "RCAN": + cancelled_rows.append(row) + # Structured-only: HTML-wrapped empty comment (strips to "") with REPORT hold. + if (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT" and text == "": + structured_report_rows.append(row) + if (row[COL_HOLD_REASON] or "").strip().upper() == "SCHEDULING" and text == "": + structured_scheduling_rows.append(row) + + # "WO schedule confirmed with vendor" → Schedule Confirmed. + assert schedule_confirmed_row is not None, "fixture lacks a schedule-confirmed row" + cat, _ = classify.classify( + schedule_confirmed_row[COL_WO_STATUS], + schedule_confirmed_row[COL_HOLD_REASON], + schedule_confirmed_row[COL_LAST_COMMENT], + ) + assert cat == "Schedule Confirmed", f"expected Schedule Confirmed, got {cat!r}" + + # RCAN rows → Cancelled. + assert cancelled_rows, "fixture lacks an RCAN row" + cat, _ = classify.classify( + cancelled_rows[0][COL_WO_STATUS], + cancelled_rows[0][COL_HOLD_REASON], + cancelled_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Cancelled", f"expected Cancelled, got {cat!r}" + + # 3rd Escalation rows are present and classify correctly. + assert third_esc_rows, "fixture lacks a 3rd-escalation row" + cat, _ = classify.classify( + third_esc_rows[0][COL_WO_STATUS], + third_esc_rows[0][COL_HOLD_REASON], + third_esc_rows[0][COL_LAST_COMMENT], + ) + assert cat == "3rd Escalation", f"expected 3rd Escalation, got {cat!r}" + + # Structured-only REPORT rows → Report / Docs Needed. + assert structured_report_rows, "fixture lacks a structured-only REPORT hold row" + cat, _ = classify.classify( + structured_report_rows[0][COL_WO_STATUS], + structured_report_rows[0][COL_HOLD_REASON], + structured_report_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Report / Docs Needed", f"expected Report / Docs Needed, got {cat!r}" + + # Structured-only SCHEDULING rows → Awaiting Scheduling. + assert structured_scheduling_rows, "fixture lacks a structured-only SCHEDULING row" + cat, _ = classify.classify( + structured_scheduling_rows[0][COL_WO_STATUS], + structured_scheduling_rows[0][COL_HOLD_REASON], + structured_scheduling_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Awaiting Scheduling", f"expected Awaiting Scheduling, got {cat!r}" + + # REPORT-hold row with completion comment → mismatch surfaced. + assert report_completion_mismatch is not None, ( + "fixture lacks a REPORT-hold row with a completion comment (mismatch case)" + ) + _, mm = classify.classify( + report_completion_mismatch[COL_WO_STATUS], + report_completion_mismatch[COL_HOLD_REASON], + report_completion_mismatch[COL_LAST_COMMENT], + ) + assert mm is not None, "expected a mismatch reason for WO-1012 but got None" + assert "REPORT" in mm, f"mismatch reason should mention REPORT hold: {mm!r}" From 63b6f11bf66ae522e53131886d5354555729abab Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:33:26 -0400 Subject: [PATCH 04/12] Add tests for the Slack interactions endpoint --- tests/test_interactions.py | 331 +++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 tests/test_interactions.py diff --git a/tests/test_interactions.py b/tests/test_interactions.py new file mode 100644 index 0000000..1b94097 --- /dev/null +++ b/tests/test_interactions.py @@ -0,0 +1,331 @@ +"""Tests for the Slack interactions Lambda endpoint. + +Covers: signature verification, drill_category modal, drill_site modal, +base64-encoded bodies, non-block_actions payloads, and missing-field no-ops. +""" + +from __future__ import annotations + +import base64 +import json +from urllib.parse import urlencode + + +def _make_event( + payload_dict: dict, + *, + timestamp: str = "1234567890", + signature: str = "v0=fakesig", + base64_encoded: bool = False, +) -> dict: + """Build a realistic API Gateway HTTP API event for Slack interactions.""" + raw_payload = urlencode({"payload": json.dumps(payload_dict)}) + body: str + if base64_encoded: + body = base64.b64encode(raw_payload.encode("utf-8")).decode("utf-8") + else: + body = raw_payload + return { + "headers": { + "x-slack-request-timestamp": timestamp, + "x-slack-signature": signature, + "content-type": "application/x-www-form-urlencoded", + }, + "body": body, + "isBase64Encoded": base64_encoded, + } + + +def _block_actions_payload( + action_id: str, + value: str, + trigger_id: str = "trigger123", +) -> dict: + return { + "type": "block_actions", + "trigger_id": trigger_id, + "actions": [ + { + "action_id": action_id, + "value": value, + } + ], + } + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_invalid_signature_returns_401_and_no_views_open(monkeypatch): + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: False) + + views_open_calls = [] + fake_client = type( + "C", (), {"views_open": lambda self, **kw: views_open_calls.append(kw)} + ) + monkeypatch.setattr(slackio, "web_client", lambda: fake_client()) + + event = _make_event( + _block_actions_payload( + "drill_category:Report / Docs Needed", "Report / Docs Needed" + ) + ) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 401 + assert "invalid" in resp["body"].lower() + assert views_open_calls == [] + + +def test_drill_category_opens_modal_with_matching_wos(monkeypatch): + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + details = [ + {"wo_number": "WO-001", "category": "Report / Docs Needed", "site": "ABQ5"}, + {"wo_number": "WO-002", "category": "3rd Escalation", "site": "ACY9"}, + {"wo_number": "WO-003", "category": "Report / Docs Needed", "site": "ABQ5"}, + ] + monkeypatch.setattr(slackio, "read_meta_json", lambda dt, name: details) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + event = _make_event( + _block_actions_payload( + "drill_category:Report / Docs Needed", "Report / Docs Needed" + ) + ) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert len(views_open_calls) == 1 + call = views_open_calls[0] + assert call["trigger_id"] == "trigger123" + view = call["view"] + # Title must reflect the count (2 matching WOs) + assert "2" in view["title"]["text"] + assert "Report / Docs Needed" in view["title"]["text"] + + +def test_drill_site_filters_on_site_field(monkeypatch): + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + details = [ + {"wo_number": "WO-001", "category": "Report / Docs Needed", "site": "ABQ5"}, + {"wo_number": "WO-002", "category": "3rd Escalation", "site": "ACY9"}, + {"wo_number": "WO-003", "category": "Awaiting Scheduling", "site": "ABQ5"}, + ] + monkeypatch.setattr(slackio, "read_meta_json", lambda dt, name: details) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + event = _make_event(_block_actions_payload("drill_site:ABQ5", "ABQ5")) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert len(views_open_calls) == 1 + view = views_open_calls[0]["view"] + # 2 WOs match site ABQ5 + assert "2" in view["title"]["text"] + assert "ABQ5" in view["title"]["text"] + + +def test_base64_encoded_body_is_decoded(monkeypatch): + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + monkeypatch.setattr( + slackio, + "read_meta_json", + lambda dt, name: [ + {"wo_number": "WO-010", "category": "3rd Escalation", "site": "ACY9"} + ], + ) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + event = _make_event( + _block_actions_payload("drill_category:3rd Escalation", "3rd Escalation"), + base64_encoded=True, + ) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert len(views_open_calls) == 1 + + +def test_non_block_actions_type_returns_200_no_views_open(monkeypatch): + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + # "view_submission" is a valid Slack payload type but not block_actions + event = _make_event({"type": "view_submission"}) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert views_open_calls == [] + + +def test_unknown_action_kind_returns_200_no_views_open(monkeypatch): + """action_id prefix not in _FILTER_FIELD → no-op.""" + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + payload = { + "type": "block_actions", + "trigger_id": "tid", + "actions": [{"action_id": "open_dashboard", "value": "something"}], + } + event = _make_event(payload) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert views_open_calls == [] + + +def test_missing_value_returns_200_no_views_open(monkeypatch): + """action has a valid kind but missing value → no-op.""" + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + payload = { + "type": "block_actions", + "trigger_id": "tid", + "actions": [ + {"action_id": "drill_category:Report / Docs Needed"} + ], # no value key + } + event = _make_event(payload) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert views_open_calls == [] + + +def test_missing_trigger_id_returns_200_no_views_open(monkeypatch): + """Valid action but no trigger_id → no-op.""" + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + payload = { + "type": "block_actions", + # trigger_id absent + "actions": [ + { + "action_id": "drill_category:Report / Docs Needed", + "value": "Report / Docs Needed", + } + ], + } + event = _make_event(payload) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert views_open_calls == [] + + +def test_empty_details_json_shows_zero_wos_in_modal(monkeypatch): + """Details list is empty → views_open still called, modal title shows 0.""" + import interactions + import slackio + + monkeypatch.setattr(slackio, "verify_signature", lambda body, ts, sig: True) + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + monkeypatch.setattr(slackio, "read_meta_json", lambda dt, name: []) + + views_open_calls = [] + fake_client = type( + "C", + (), + {"views_open": lambda self, **kw: views_open_calls.append(kw)}, + )() + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + + event = _make_event( + _block_actions_payload("drill_category:3rd Escalation", "3rd Escalation") + ) + resp = interactions.handler(event, None) + + assert resp["statusCode"] == 200 + assert len(views_open_calls) == 1 + assert "(0)" in views_open_calls[0]["view"]["title"]["text"] From 48c030ed13ed30440a2dbc5901daca5a6de36672 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:52:53 -0400 Subject: [PATCH 05/12] Add tests for the classifier handler transforms --- tests/test_classifier_handler.py | 523 +++++++++++++++++++++++++++++++ 1 file changed, 523 insertions(+) create mode 100644 tests/test_classifier_handler.py diff --git a/tests/test_classifier_handler.py b/tests/test_classifier_handler.py new file mode 100644 index 0000000..37ef4d1 --- /dev/null +++ b/tests/test_classifier_handler.py @@ -0,0 +1,523 @@ +"""Tests for the classifier handler: pure transforms and the handler() entrypoint. + +Uses the importlib trick to avoid an ambiguous bare ``import handler`` (both +lambdas/classifier/handler.py and lambdas/slack_post/handler.py are on +pythonpath). ``awswrangler`` is stubbed at the sys.modules level before the +module is loaded, so no real AWS/network calls are made. +""" + +from __future__ import annotations + +import csv +import io +import json +import sys +import tempfile +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# --------------------------------------------------------------------------- +# Load the classifier handler under a unique module name. +# awswrangler must be stubbed before exec_module() runs the top-level imports. +# --------------------------------------------------------------------------- + +# Build a minimal awswrangler stub so handler.py's top-level `import awswrangler as wr` +# succeeds without the real package installed. +_wr_stub = types.ModuleType("awswrangler") +_wr_stub.s3 = types.ModuleType("awswrangler.s3") +_wr_stub.s3.to_parquet = MagicMock() +sys.modules.setdefault("awswrangler", _wr_stub) +sys.modules.setdefault("awswrangler.s3", _wr_stub.s3) + +import importlib.util # noqa: E402 + +_HANDLER_PATH = ( + Path(__file__).resolve().parents[1] / "lambdas" / "classifier" / "handler.py" +) +_spec = importlib.util.spec_from_file_location("classifier_handler", _HANDLER_PATH) +handler = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(handler) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_HEADER = [ + "WO Number", + "WO Description", + "Equipment Code", + "Organization", + "Due Date", + "Department", + "WO Status", + "Hold Reason", + "Last Comment", + "Last Comment By", + "Last Comment Date", + "Contractor", + "Contractor Description", +] + + +def _make_row( + wo_number="WO-001", + wo_description="Fix HVAC", + equipment_code="HVAC-01", + site="ABQ5", + due_date="2026-05-30", + department="SSP", + wo_status="IP", + hold_reason="", + last_comment="WO schedule confirmed with vendor.", + last_comment_by="tech@example.com", + last_comment_date="2026-05-28", + contractor="ABC HVAC", + contractor_description="HVAC Services", +): + return [ + wo_number, + wo_description, + equipment_code, + site, + due_date, + department, + wo_status, + hold_reason, + last_comment, + last_comment_by, + last_comment_date, + contractor, + contractor_description, + ] + + +def _write_csv(rows: list[list], header: list[str] = _HEADER) -> str: + """Write header + rows to a temp CSV file and return the path.""" + with tempfile.NamedTemporaryFile( + mode="w", suffix=".csv", delete=False, newline="" + ) as fh: + writer = csv.writer(fh) + writer.writerow(header) + writer.writerows(rows) + return fh.name + + +def _write_xlsx(rows: list[list], header: list[str] = _HEADER) -> str: + """Write header + rows to a temp xlsx file and return the path.""" + import openpyxl + + wb = openpyxl.Workbook() + ws = wb.active + ws.append(header) + for row in rows: + ws.append(row) + with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as fh: + path = fh.name + wb.save(path) + return path + + +# --------------------------------------------------------------------------- +# _resolve_columns +# --------------------------------------------------------------------------- + + +class TestResolveColumns: + def test_normal_13_col_header(self): + result = handler._resolve_columns(_HEADER) + assert result["wo_number"] == 0 + assert result["site"] == 3 # "Organization" column + assert result["wo_status"] == 6 + assert result["hold_reason"] == 7 + assert result["last_comment"] == 8 + assert result["last_comment_by"] == 9 + assert result["last_comment_date"] == 10 + + def test_header_drift_extra_spaces_and_case(self): + drifted = [ + " WO NUMBER ", + "WO DESCRIPTION", + "Equipment Code", + "Organization", + "Due Date", + "Department", + "WO Status ", + "Hold Reason", + "Last Comment", + "Last Comment By", + "Last Comment Date", + "Contractor", + "Contractor Description", + ] + result = handler._resolve_columns(drifted) + assert result["wo_number"] == 0 + assert result["wo_status"] == 6 + assert result["last_comment"] == 8 + + def test_prefix_collision_last_comment_resolves_to_exact_column(self): + """'last comment' must resolve to col 8 (Last Comment), not col 9 or 10.""" + result = handler._resolve_columns(_HEADER) + col_idx = result["last_comment"] + assert col_idx == 8 + assert _HEADER[col_idx] == "Last Comment" + + # last_comment_by and last_comment_date must not steal last_comment's slot + assert result["last_comment_by"] == 9 + assert result["last_comment_date"] == 10 + + +# --------------------------------------------------------------------------- +# _read_rows +# --------------------------------------------------------------------------- + + +class TestReadRows: + def test_csv_header_and_rows(self): + data = [_make_row(wo_number="WO-001"), _make_row(wo_number="WO-002")] + path = _write_csv(data) + header, rows = handler._read_rows(path, "raw/export.csv") + assert header[0] == "WO Number" + assert len(rows) == 2 + assert rows[0][0] == "WO-001" + assert rows[1][0] == "WO-002" + + def test_xlsx_header_and_rows(self): + data = [_make_row(wo_number="WO-003"), _make_row(wo_number="WO-004")] + path = _write_xlsx(data) + header, rows = handler._read_rows(path, "raw/export.xlsx") + assert header[0] == "WO Number" + assert len(rows) == 2 + assert rows[0][0] == "WO-003" + assert rows[1][0] == "WO-004" + + +# --------------------------------------------------------------------------- +# _build_snapshot +# --------------------------------------------------------------------------- + + +class TestBuildSnapshot: + def test_blank_comment_rows_excluded(self): + rows = [ + _make_row( + wo_number="WO-010", last_comment="Schedule confirmed." + ), + _make_row(wo_number="WO-011", last_comment=""), # blank — excluded + _make_row( + wo_number="WO-012", last_comment="" + ), # strips to "" — excluded + ] + df, blank = handler._build_snapshot(_HEADER, rows) + assert blank == 2 + assert len(df) == 1 + assert df.iloc[0]["wo_number"] == "WO-010" + + def test_missing_last_comment_column_raises(self): + # Remove all "last comment" variants so last_comment cannot resolve via + # substring match either — only columns with no "last comment" remain. + bad_header = [ + "WO Number", + "WO Description", + "Equipment Code", + "Organization", + "Due Date", + "Department", + "WO Status", + "Hold Reason", + "Contractor", + "Contractor Description", + ] + rows = [ + [ + "WO-001", + "Fix HVAC", + "HVAC", + "ABQ5", + "2026-05-30", + "SSP", + "IP", + "", + "ABC", + "HVAC", + ] + ] + with pytest.raises(ValueError, match="required columns"): + handler._build_snapshot(bad_header, rows) + + def test_missing_wo_status_column_raises(self): + bad_header = [ + "WO Number", + "WO Description", + "Equipment Code", + "Organization", + "Due Date", + "Department", + # "WO Status" missing + "Hold Reason", + "Last Comment", + "Last Comment By", + "Last Comment Date", + "Contractor", + "Contractor Description", + ] + rows = [ + [ + "WO-001", + "Fix HVAC", + "HVAC", + "ABQ5", + "2026-05-30", + "SSP", + "", + "Schedule confirmed.", + "tech@example.com", + "2026-05-28", + "ABC", + "HVAC", + ] + ] + with pytest.raises(ValueError, match="required columns"): + handler._build_snapshot(bad_header, rows) + + def test_is_escalation_and_is_action_derived(self): + import classify as clf + + rows = [ + _make_row( + wo_number="WO-020", + wo_status="H", + hold_reason="REPORT", + last_comment="3rd attempt process for schedule confirmation.", + ), + _make_row( + wo_number="WO-021", + wo_status="IP", + hold_reason="", + last_comment="WO schedule confirmed with vendor.", + ), + ] + df, _ = handler._build_snapshot(_HEADER, rows) + + esc_row = df[df["wo_number"] == "WO-020"].iloc[0] + assert bool(esc_row["is_escalation"]) is True + assert esc_row["category"] in clf.ESCALATION_CATEGORIES + + routine_row = df[df["wo_number"] == "WO-021"].iloc[0] + assert bool(routine_row["is_escalation"]) is False + assert routine_row["category"] == "Schedule Confirmed" + + +# --------------------------------------------------------------------------- +# _build_summary +# --------------------------------------------------------------------------- + + +class TestBuildSummary: + def _make_df(self): + """Build a small DataFrame via _build_snapshot.""" + rows = [ + _make_row( + wo_number="WO-030", + wo_status="H", + hold_reason="REPORT", + last_comment="3rd attempt process for schedule confirmation.", + site="ABQ5", + ), + _make_row( + wo_number="WO-031", + wo_status="H", + hold_reason="REPORT", + last_comment="3rd attempt process for schedule confirmation.", + site="ACY9", + ), + _make_row( + wo_number="WO-032", + wo_status="IP", + hold_reason="", + last_comment="WO schedule confirmed with vendor.", + site="ABQ5", + ), + # This row produces a mismatch: completion comment + IP status + _make_row( + wo_number="WO-033", + wo_status="IP", + hold_reason="REPORT", + last_comment="Vendor arrived and performed task.", + site="ABQ5", + ), + ] + df, blank = handler._build_snapshot(_HEADER, rows) + return df, blank + + def test_third_escalation_count(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert summary["third_escalation_count"] == 2 + + def test_category_counts_present(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert "3rd Escalation" in summary["category_counts"] + assert summary["category_counts"]["3rd Escalation"] == 2 + + def test_escalation_total(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert summary["escalation_total"] == 2 + + def test_action_needed_and_routine_sum_to_classified_total(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert ( + summary["action_needed"] + summary["routine"] == summary["classified_total"] + ) + + def test_top_sites_shape(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert isinstance(summary["top_sites"], list) + for entry in summary["top_sites"]: + assert "site" in entry + assert "count" in entry + # ABQ5 appears 3 times, should be first + assert summary["top_sites"][0]["site"] == "ABQ5" + + def test_mismatches_list(self): + df, blank = self._make_df() + summary = handler._build_summary(df, "2026-05-28", "raw/export.csv", blank) + assert isinstance(summary["mismatches"], list) + # WO-033 has a mismatch (completion comment + REPORT hold) + assert len(summary["mismatches"]) >= 1 + mismatch_wos = [m["wo_number"] for m in summary["mismatches"]] + assert "WO-033" in mismatch_wos + + +# --------------------------------------------------------------------------- +# _event_dt +# --------------------------------------------------------------------------- + + +class TestEventDt: + def test_event_time_extracted(self): + record = {"eventTime": "2026-05-28T22:23:40.123Z"} + assert handler._event_dt(record) == "2026-05-28" + + def test_no_event_time_falls_back_to_today(self): + from datetime import datetime, timezone + + record = {} + result = handler._event_dt(record) + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + # Result must look like a YYYY-MM-DD date string + assert len(result) == 10 + assert result[4] == "-" and result[7] == "-" + assert result == today + + +# --------------------------------------------------------------------------- +# handler() entrypoint — two S3 records, monkeypatched AWS boundaries +# --------------------------------------------------------------------------- + + +class TestHandlerEntrypoint: + def _build_csv_bytes( + self, + wo_status="IP", + last_comment="Schedule confirmed with vendor.", + ): + """Return CSV bytes for a single-row export.""" + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(_HEADER) + writer.writerow(_make_row(wo_status=wo_status, last_comment=last_comment)) + return buf.getvalue().encode("utf-8") + + def test_two_records_processed_slack_invoked_once_per_dt( + self, tmp_path, monkeypatch + ): + csv_bytes_a = self._build_csv_bytes() + csv_bytes_b = self._build_csv_bytes( + last_comment="3rd attempt process for schedule confirmation." + ) + + # Track put_object and lambda.invoke calls + put_calls: list[dict] = [] + invoke_calls: list[dict] = [] + + def fake_download_fileobj(bucket, key, fh): + if "file_a" in key: + fh.write(csv_bytes_a) + else: + fh.write(csv_bytes_b) + + fake_s3 = MagicMock() + fake_s3.download_fileobj.side_effect = fake_download_fileobj + fake_s3.put_object.side_effect = lambda **kw: put_calls.append(kw) + + fake_lambda = MagicMock() + fake_lambda.invoke.side_effect = lambda **kw: invoke_calls.append(kw) + + monkeypatch.setattr(handler, "_s3", fake_s3) + monkeypatch.setattr(handler, "_lambda", fake_lambda) + monkeypatch.setattr(handler.wr.s3, "to_parquet", MagicMock()) + + # Set the SLACK_POST_FUNCTION_NAME env var so the lambda invoke fires + monkeypatch.setenv("SLACK_POST_FUNCTION_NAME", "apm-slack-post") + + event = { + "Records": [ + { + "eventTime": "2026-05-28T10:00:00.000Z", + "s3": { + "bucket": {"name": "test-bucket"}, + "object": {"key": "raw/file_a.csv"}, + }, + }, + { + "eventTime": "2026-05-28T11:00:00.000Z", + "s3": { + "bucket": {"name": "test-bucket"}, + "object": {"key": "raw/file_b.csv"}, + }, + }, + ] + } + + result = handler.handler(event, None) + + # Both records processed + assert len(result["processed"]) == 2 + + # summary.json and details.json written for each record (2 put_object calls each = 4) + assert len(put_calls) == 4 + + # Slack post invoked exactly once (both records share the same dt "2026-05-28") + assert len(invoke_calls) == 1 + assert invoke_calls[0]["FunctionName"] == "apm-slack-post" + payload = json.loads(invoke_calls[0]["Payload"]) + assert payload["dt"] == "2026-05-28" + + def test_non_export_key_skipped(self, monkeypatch): + fake_s3 = MagicMock() + fake_lambda = MagicMock() + monkeypatch.setattr(handler, "_s3", fake_s3) + monkeypatch.setattr(handler, "_lambda", fake_lambda) + + event = { + "Records": [ + { + "eventTime": "2026-05-28T10:00:00.000Z", + "s3": { + "bucket": {"name": "test-bucket"}, + "object": {"key": "raw/not-an-export.txt"}, + }, + } + ] + } + result = handler.handler(event, None) + assert result["processed"] == [] + fake_s3.download_fileobj.assert_not_called() From e555a75d97cc6e4f2ae70bb1655c4f431376b057 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:54:47 -0400 Subject: [PATCH 06/12] Add tests for the Haiku fallback --- tests/test_haiku.py | 271 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 tests/test_haiku.py diff --git a/tests/test_haiku.py b/tests/test_haiku.py new file mode 100644 index 0000000..25d4549 --- /dev/null +++ b/tests/test_haiku.py @@ -0,0 +1,271 @@ +"""Tests for the Haiku fallback in classify_with_haiku(). + +No real network or AWS calls are made — _call_haiku and _fetch_api_key are +monkeypatched at the function level. boto3 shape tests stub the secretsmanager +client directly via monkeypatch on boto3.client. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import classify + +# --------------------------------------------------------------------------- +# Comment that will always land as "Other" deterministically (free-text with +# no structured hold signal and no regex match). +# --------------------------------------------------------------------------- +_OTHER_COMMENT = "Uplift request submitted pending management approval." +_TRIVIAL_COMMENT = "Ok" # strips to "Ok" — len == 2 < 4 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _haiku_never_called(): + """Return a _call_haiku stub that raises if invoked.""" + + def _stub(text, api_key): + raise AssertionError("_call_haiku should not have been called") + + return _stub + + +# --------------------------------------------------------------------------- +# Gating: Haiku is NOT called when the deterministic result is non-Other +# --------------------------------------------------------------------------- + + +def test_haiku_not_called_when_deterministic_result_is_non_other(monkeypatch): + """A comment that resolves deterministically must skip the Haiku path.""" + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + # "WO schedule confirmed with vendor" → Schedule Confirmed deterministically + cat, mm = classify.classify_with_haiku( + "IP", "", "WO schedule confirmed with vendor." + ) + assert cat == "Schedule Confirmed" + assert mm is None + + +def test_haiku_not_called_when_hold_reason_present(monkeypatch): + """Hold reason present → structured state handles it; Haiku must not fire.""" + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + # REPORT hold with unrecognised comment → hold decides, not Haiku + cat, _ = classify.classify_with_haiku( + "H", "REPORT", "xyz nothing here" + ) + assert cat == "Report / Docs Needed" + + +def test_haiku_not_called_when_comment_is_trivial(monkeypatch): + """Comment len < 4 after stripping → trivial, skip Haiku.""" + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + cat, _ = classify.classify_with_haiku("IP", "", _TRIVIAL_COMMENT) + assert cat == "Other" + + +def test_haiku_not_called_when_disabled_via_env(monkeypatch): + """APM_HAIKU_FALLBACK=off must suppress the Haiku call.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "off") + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +def test_haiku_not_called_when_disabled_via_zero(monkeypatch): + monkeypatch.setenv("APM_HAIKU_FALLBACK", "0") + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +def test_haiku_not_called_when_disabled_via_false(monkeypatch): + monkeypatch.setenv("APM_HAIKU_FALLBACK", "false") + monkeypatch.setattr(classify, "_call_haiku", _haiku_never_called()) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "fake-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +# --------------------------------------------------------------------------- +# Haiku IS invoked (enabled, Other, no hold, non-trivial) +# --------------------------------------------------------------------------- + + +def test_haiku_called_for_true_residual_other(monkeypatch): + """When all gates pass, _call_haiku must be invoked.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "on") + haiku_calls: list[tuple] = [] + + def _stub_haiku(text, api_key): + haiku_calls.append((text, api_key)) + return "Rescheduled" + + monkeypatch.setattr(classify, "_call_haiku", _stub_haiku) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Rescheduled" + assert len(haiku_calls) == 1 + + +def test_haiku_valid_bucket_replaces_other(monkeypatch): + """When Haiku returns a valid bucket, that bucket is used.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled") + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Rescheduled" + + +def test_haiku_none_response_stays_other(monkeypatch): + """When Haiku returns None, the result stays Other.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + monkeypatch.setattr(classify, "_call_haiku", lambda text, key: None) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +def test_haiku_other_response_stays_other(monkeypatch): + """When Haiku returns 'Other', the result stays Other (guard against self-loop).""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Other") + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +def test_haiku_exception_in_call_haiku_stays_other(monkeypatch): + """Exception raised by _call_haiku (e.g. network failure) is caught and + the result safely stays at Other.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + + def _raise(text, api_key): + raise OSError("network failure") + + monkeypatch.setattr(classify, "_call_haiku", _raise) + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + cat, _ = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Other" + + +def test_mismatch_preserved_through_haiku_path(monkeypatch): + """The deterministic mismatch reason must survive through the Haiku path.""" + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled") + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + # A comment that fires "Completed / Pending Close" intent while on a REPORT hold + # produces a mismatch; but the deterministic result is NOT Other here. To test + # mismatch preservation we need the deterministic result to be Other while a + # mismatch exists. Mismatch is computed from intent vs structured state; if the + # intent is None (→ Other from deterministic), no mismatch is produced by + # _mismatch(). So the relevant test is: Haiku fires on an Other row; the + # mismatch (None in this case) is preserved correctly. + cat, mm = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + # Comment has no intent → no mismatch → mm is None; Haiku returns Rescheduled + assert cat == "Rescheduled" + assert mm is None + + +def test_mismatch_preserved_when_haiku_overrides(monkeypatch): + """Verify mismatch from classify() passes through unchanged when Haiku overrides. + + We use a real deterministic Other-producing row and manually inject a mismatch + by monkeypatching classify.classify to return (Other, "fake mismatch reason"). + """ + monkeypatch.setenv("APM_HAIKU_FALLBACK", "1") + monkeypatch.setattr(classify, "_call_haiku", lambda text, key: "Rescheduled") + monkeypatch.setattr(classify, "_fetch_api_key", lambda: "test-key") + + original_classify = classify.classify + + def _patched_classify(wo_status, hold_reason, last_comment): + cat, _ = original_classify(wo_status, hold_reason, last_comment) + if cat == "Other": + return "Other", "injected mismatch reason" + return cat, _ + + monkeypatch.setattr(classify, "classify", _patched_classify) + + cat, mm = classify.classify_with_haiku("IP", "", _OTHER_COMMENT) + assert cat == "Rescheduled" + assert mm == "injected mismatch reason" + + +# --------------------------------------------------------------------------- +# _fetch_api_key JSON shapes +# --------------------------------------------------------------------------- + + +def _make_secretsmanager_client(secret_string: str) -> MagicMock: + """Build a minimal secretsmanager client stub returning the given secret.""" + client = MagicMock() + client.get_secret_value.return_value = {"SecretString": secret_string} + return client + + +def test_fetch_api_key_bare_string(monkeypatch): + """Bare string secret → returned as-is.""" + fake_client = _make_secretsmanager_client("sk-ant-barekey123") + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client) + key = classify._fetch_api_key() + assert key == "sk-ant-barekey123" + + +def test_fetch_api_key_json_anthropic_api_key(monkeypatch): + """JSON with 'anthropic-api-key' field → that value extracted.""" + secret = json.dumps({"anthropic-api-key": "sk-ant-jsonkey456"}) + fake_client = _make_secretsmanager_client(secret) + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client) + key = classify._fetch_api_key() + assert key == "sk-ant-jsonkey456" + + +def test_fetch_api_key_single_value_json(monkeypatch): + """Single-value JSON object with an arbitrary key → the only value extracted.""" + secret = json.dumps({"my_custom_key": "sk-ant-singleval789"}) + fake_client = _make_secretsmanager_client(secret) + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client) + key = classify._fetch_api_key() + assert key == "sk-ant-singleval789" + + +def test_fetch_api_key_json_api_key_field(monkeypatch): + """JSON with 'api_key' field → that value extracted.""" + secret = json.dumps({"api_key": "sk-ant-apikey111"}) + fake_client = _make_secretsmanager_client(secret) + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda service, **kw: fake_client) + key = classify._fetch_api_key() + assert key == "sk-ant-apikey111" From 76f60abadec466a2d351a7737293ef65d757e7b5 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:59:40 -0400 Subject: [PATCH 07/12] Add per-bucket and precedence tests for comment intent --- tests/test_comment_intent.py | 284 +++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 tests/test_comment_intent.py diff --git a/tests/test_comment_intent.py b/tests/test_comment_intent.py new file mode 100644 index 0000000..45be12f --- /dev/null +++ b/tests/test_comment_intent.py @@ -0,0 +1,284 @@ +"""Per-bucket and precedence tests for classify.comment_intent(). + +Tests operate on *already-stripped* plain text (as comment_intent() expects). +End-to-end precedence is also verified via classify.classify(), which handles +HTML stripping internally. + +Bucket coverage: + - One positive case per ladder bucket that can fire deterministically: + 3rd Escalation, 2nd Escalation, 1st Escalation, SIM Ticket, Vendor No-Show, + Weekly WO Scheduled, Avetta Project Created, Schedule Confirmed, + Completed / Pending Close, Rescheduled, Report / Docs Needed, + Awaiting Report / Invoice, Awaiting Scheduling, Status Inquiry, On Hold, + Acknowledgement / No-op. + - "Other Escalation" has no dedicated regex rule in the ladder and is + intentionally omitted (it is a Haiku-only bucket). + +Precedence / negative traps: + - 3rd attempt + report language -> 3rd Escalation (most-specific wins). + - 1st attempt process for schedule confirmation -> Awaiting Scheduling + (not an escalation; the ladder requires an explicit escalation signal). + - Bare "1st attempt to contact vendor" (no escalation keyword) -> None. + - Empty / whitespace -> None. + - "Copy" / "Copy 5/26." -> Acknowledgement / No-op (fullmatch). +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert( + 0, str(Path(__file__).resolve().parent.parent / "lambdas" / "classifier") +) + +import classify # noqa: E402 + + +# --------------------------------------------------------------------------- +# Axis-1 per-bucket positive cases +# --------------------------------------------------------------------------- + + +class TestCommentIntentBuckets: + def test_3rd_escalation_attempt(self): + assert ( + classify.comment_intent("3rd attempt process for schedule confirmation") + == "3rd Escalation" + ) + + def test_3rd_escalation_explicit(self): + assert ( + classify.comment_intent("3rd escalation sent to vendor") == "3rd Escalation" + ) + + def test_2nd_escalation_attempt(self): + assert ( + classify.comment_intent("2nd attempt process for schedule confirmation") + == "2nd Escalation" + ) + + def test_2nd_escalation_explicit(self): + assert ( + classify.comment_intent("2nd escalation sent to vendor") == "2nd Escalation" + ) + + def test_1st_escalation_explicit(self): + assert ( + classify.comment_intent("1st escalation sent to management") + == "1st Escalation" + ) + + def test_sim_ticket_with_version(self): + assert ( + classify.comment_intent("SIM ticket v1234567 created for this WO") + == "SIM Ticket" + ) + + def test_sim_ticket_tcorp_url(self): + assert ( + classify.comment_intent("SIM TT opened t.corp.amazon.com/issues/123") + == "SIM Ticket" + ) + + def test_vendor_no_show(self): + assert classify.comment_intent("Vendor was a no show today") == "Vendor No-Show" + + def test_weekly_wo_scheduled(self): + assert ( + classify.comment_intent( + "Weekly WO scheduled. Service reports required EOD Friday." + ) + == "Weekly WO Scheduled" + ) + + def test_avetta_project_created(self): + assert ( + classify.comment_intent("Avetta project created for this work") + == "Avetta Project Created" + ) + + def test_avetta_work_request(self): + assert ( + classify.comment_intent("Work request created in Avetta for this job") + == "Avetta Project Created" + ) + + def test_schedule_confirmed_with_vendor(self): + assert ( + classify.comment_intent("Schedule confirmed with vendor for next week") + == "Schedule Confirmed" + ) + + def test_schedule_confirmed_vendor_confirmed(self): + assert classify.comment_intent("Vendor confirmed") == "Schedule Confirmed" + + def test_completed_pending_close_performed_task(self): + assert ( + classify.comment_intent("Vendor arrived and performed task") + == "Completed / Pending Close" + ) + + def test_completed_pending_close_completed_by(self): + assert ( + classify.comment_intent("Completed by contractor on Monday") + == "Completed / Pending Close" + ) + + def test_completed_pending_close_cant_close(self): + assert ( + classify.comment_intent("Can't close the WO") == "Completed / Pending Close" + ) + + def test_completed_pending_close_pending_close(self): + assert classify.comment_intent("Pending close") == "Completed / Pending Close" + + def test_rescheduled(self): + assert ( + classify.comment_intent("WO rescheduled to next Tuesday") == "Rescheduled" + ) + + def test_rescheduled_new_eta(self): + assert classify.comment_intent("New ETA provided by vendor") == "Rescheduled" + + def test_report_docs_needed_service_report(self): + assert ( + classify.comment_intent("Please upload service report") + == "Report / Docs Needed" + ) + + def test_report_docs_needed_completion_report(self): + assert ( + classify.comment_intent("Completion report required") + == "Report / Docs Needed" + ) + + def test_awaiting_report_invoice_awaiting(self): + assert ( + classify.comment_intent("Awaiting the invoice from vendor") + == "Awaiting Report / Invoice" + ) + + def test_awaiting_report_invoice_pending(self): + assert ( + classify.comment_intent("Pending report from contractor") + == "Awaiting Report / Invoice" + ) + + def test_awaiting_scheduling_please_schedule(self): + assert ( + classify.comment_intent( + "Please schedule this WO at your earliest convenience" + ) + == "Awaiting Scheduling" + ) + + def test_status_inquiry_any_update(self): + assert classify.comment_intent("Any update on this WO?") == "Status Inquiry" + + def test_status_inquiry_eta(self): + assert ( + classify.comment_intent("Do you have an ETA on this?") == "Status Inquiry" + ) + + def test_on_hold(self): + assert ( + classify.comment_intent("WO is on hold pending budget approval") + == "On Hold" + ) + + def test_acknowledgement_copy(self): + assert classify.comment_intent("Copy") == "Acknowledgement / No-op" + + def test_acknowledgement_copy_with_date(self): + assert classify.comment_intent("Copy 5/26.") == "Acknowledgement / No-op" + + def test_acknowledgement_noted(self): + assert classify.comment_intent("Noted") == "Acknowledgement / No-op" + + +# --------------------------------------------------------------------------- +# Precedence / negative traps +# --------------------------------------------------------------------------- + + +class TestCommentIntentPrecedence: + def test_3rd_attempt_plus_report_language_resolves_to_3rd_escalation(self): + """A comment that mentions both '3rd attempt' and report language must + resolve to '3rd Escalation' — the most-specific bucket wins.""" + text = ( + "3rd attempt process for schedule confirmation. " + "Please provide service report." + ) + assert classify.comment_intent(text) == "3rd Escalation" + + def test_1st_attempt_for_schedule_confirmation_is_awaiting_scheduling(self): + """'1st attempt process for schedule confirmation' is routine outreach, + not an escalation. The ladder has an explicit Awaiting Scheduling rule + for this canonical phrase.""" + text = "1st attempt process for schedule confirmation" + assert classify.comment_intent(text) == "Awaiting Scheduling" + + def test_bare_1st_attempt_no_escalation_keyword_is_none(self): + """A bare '1st attempt to contact vendor' with no escalation keyword + must not fire any escalation bucket — the ladder requires an explicit + escalation signal beyond the mere ordinal.""" + text = "1st attempt to contact vendor" + assert classify.comment_intent(text) is None + + def test_empty_string_returns_none(self): + assert classify.comment_intent("") is None + + def test_whitespace_only_returns_none(self): + assert classify.comment_intent(" ") is None + + +# --------------------------------------------------------------------------- +# End-to-end precedence via classify() +# (strips HTML, then calls comment_intent, then applies structured-state axis) +# --------------------------------------------------------------------------- + + +class TestClassifyPrecedence: + def test_3rd_attempt_html_with_report_hold_resolves_to_3rd_escalation(self): + """Even when REPORT hold is set, a 3rd-attempt comment must resolve to + '3rd Escalation' because comment intent wins over structured state.""" + cat, mm = classify.classify( + "H", + "REPORT", + "3rd attempt process for schedule confirmation. " + "Please provide service report.", + ) + assert cat == "3rd Escalation" + # 3rd Escalation is not in _DONE_INTENTS, so no mismatch is produced. + assert mm is None + + def test_1st_attempt_schedule_confirmation_via_classify(self): + """End-to-end: HTML-wrapped '1st attempt process for schedule + confirmation' classifies as Awaiting Scheduling.""" + cat, _ = classify.classify( + "IP", + "", + "1st attempt process for schedule confirmation", + ) + assert cat == "Awaiting Scheduling" + + def test_comment_intent_wins_over_structured_state(self): + """When the comment fires a rule, it overrides the Hold Reason axis.""" + # Schedule Confirmed comment wins over SCHEDULING hold. + cat, _ = classify.classify( + "R", + "SCHEDULING", + "WO schedule confirmed with vendor", + ) + assert cat == "Schedule Confirmed" + + def test_structured_state_used_when_no_comment_intent(self): + """When no Axis-1 rule fires (blank comment), structured state decides.""" + cat, _ = classify.classify("IP", "REPORT", "") + assert cat == "Report / Docs Needed" + + def test_other_when_nothing_matches(self): + """Neither comment intent nor structured state → Other.""" + cat, _ = classify.classify("IP", "", "Uplift request submitted") + assert cat == "Other" From 8e87fe669de2cea972836810291336a9a4bdbc8d Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 15:01:25 -0400 Subject: [PATCH 08/12] Add tests for slack-post handler and slackio --- tests/test_slack_post_handler.py | 333 +++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 tests/test_slack_post_handler.py diff --git a/tests/test_slack_post_handler.py b/tests/test_slack_post_handler.py new file mode 100644 index 0000000..d291652 --- /dev/null +++ b/tests/test_slack_post_handler.py @@ -0,0 +1,333 @@ +"""Tests for the slack-post Lambda handler and slackio I/O helpers. + +Handler is loaded via importlib to avoid the handler.py name collision between +the classifier and slack-post Lambdas (both are on the pythonpath and both are +named handler.py — bare ``import handler`` would be ambiguous/cached). + +Monkeypatching strategy +----------------------- +The handler module shares the same ``slackio`` object that was imported during +module load (``sp_handler.slackio is slackio`` is True), so patching attributes +on the ``slackio`` module is sufficient. + +Real ``blockkit`` is used (it is pure — no network, no AWS). + +slackio cache isolation +----------------------- +``_creds`` and ``_dashboard_url`` are module-level globals that persist across +tests within the same process. Each slackio test resets them before running to +guarantee isolation. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +# --------------------------------------------------------------------------- +# Shared pythonpath: slack_post dir must be on sys.path for slackio + blockkit. +# conftest.py sets the env vars; we rely on that here. +# --------------------------------------------------------------------------- + +_SLACK_POST_DIR = Path(__file__).resolve().parents[1] / "lambdas" / "slack_post" +if str(_SLACK_POST_DIR) not in sys.path: + sys.path.insert(0, str(_SLACK_POST_DIR)) + +import slackio # noqa: E402 + +# --------------------------------------------------------------------------- +# Load the slack-post handler via importlib (avoids name collision with +# lambdas/classifier/handler.py which is also on pythonpath). +# --------------------------------------------------------------------------- + +_HANDLER_PATH = _SLACK_POST_DIR / "handler.py" +_spec = importlib.util.spec_from_file_location("slack_post_handler", _HANDLER_PATH) +sp_handler = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(sp_handler) + + +# --------------------------------------------------------------------------- +# Fake Slack WebClient +# --------------------------------------------------------------------------- + + +class _FakeClient: + """Minimal WebClient stand-in that records chat_postMessage calls.""" + + def __init__(self): + self.calls: list[dict] = [] + + def chat_postMessage(self, **kwargs): + self.calls.append(kwargs) + return {"ok": True} + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +SUMMARY_ZERO_THIRDS = { + "dt": "2026-05-28", + "classified_total": 100, + "blank_comment_rows": 5, + "escalation_total": 3, + "action_needed": 10, + "routine": 90, + "third_escalation_count": 0, + "category_counts": {"Schedule Confirmed": 50, "3rd Escalation": 0}, + "top_sites": [], + "mismatches": [], + "generated_at": "2026-05-28T12:00:00Z", +} + +SUMMARY_WITH_THIRDS = { + **SUMMARY_ZERO_THIRDS, + "third_escalation_count": 2, + "category_counts": {"3rd Escalation": 2, "Schedule Confirmed": 48}, +} + +DETAILS_WITH_THIRDS = [ + { + "wo_number": "WO-001", + "category": "3rd Escalation", + "site": "ABQ5", + "wo_description": "Fix HVAC unit", + }, + { + "wo_number": "WO-002", + "category": "3rd Escalation", + "site": "ACY9", + "wo_description": "Repair dock door", + }, + { + "wo_number": "WO-003", + "category": "Schedule Confirmed", # should NOT appear in the alert + "site": "ABQ5", + "wo_description": "Routine PM", + }, +] + + +# --------------------------------------------------------------------------- +# Section 1: handler._yesterday +# --------------------------------------------------------------------------- + + +class TestYesterday: + def test_yesterday_basic(self): + assert sp_handler._yesterday("2026-05-28") == "2026-05-27" + + def test_yesterday_month_boundary(self): + assert sp_handler._yesterday("2026-06-01") == "2026-05-31" + + def test_yesterday_year_boundary(self): + assert sp_handler._yesterday("2026-01-01") == "2025-12-31" + + +# --------------------------------------------------------------------------- +# Section 2: handler.handler — the main Lambda entry point +# --------------------------------------------------------------------------- + + +class TestHandler: + def _make_fake_client(self): + return _FakeClient() + + def test_no_summary_returns_not_posted_and_does_not_call_slack(self, monkeypatch): + """When read_meta_json returns None for summary.json, handler returns + posted=False and must NOT call chat_postMessage.""" + fake_client = self._make_fake_client() + + monkeypatch.setattr(slackio, "read_meta_json", lambda dt, name: None) + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + monkeypatch.setattr(slackio, "channel_id", lambda: "C12345") + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + result = sp_handler.handler({"dt": "2026-05-28"}, None) + + assert result["posted"] is False + assert result["dt"] == "2026-05-28" + assert fake_client.calls == [], "chat_postMessage must not be called" + + def test_summary_zero_thirds_posts_summary_only(self, monkeypatch): + """When third_escalation_count == 0, exactly one chat_postMessage is made + (daily summary); the standalone alert is suppressed.""" + fake_client = self._make_fake_client() + + def fake_read(dt, name): + if name == "summary.json": + return SUMMARY_ZERO_THIRDS + return None + + monkeypatch.setattr(slackio, "read_meta_json", fake_read) + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + monkeypatch.setattr(slackio, "channel_id", lambda: "C12345") + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + result = sp_handler.handler({"dt": "2026-05-28"}, None) + + assert result["posted"] is True + assert result["summary"] is True + assert result["alert"] is False + assert len(fake_client.calls) == 1, "exactly one postMessage (summary)" + # The message text should reference today's dt. + assert "2026-05-28" in fake_client.calls[0]["text"] + + def test_summary_with_thirds_posts_summary_and_alert(self, monkeypatch): + """When third_escalation_count > 0, two chat_postMessage calls are made: + one for the daily summary and one for the 3rd-escalation alert.""" + fake_client = self._make_fake_client() + + def fake_read(dt, name): + if name == "summary.json": + return SUMMARY_WITH_THIRDS + if name == "details.json": + return DETAILS_WITH_THIRDS + return None + + monkeypatch.setattr(slackio, "read_meta_json", fake_read) + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + monkeypatch.setattr(slackio, "channel_id", lambda: "C12345") + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + result = sp_handler.handler({"dt": "2026-05-28"}, None) + + assert result["posted"] is True + assert result["summary"] is True + assert result["alert"] is True + assert len(fake_client.calls) == 2, "summary + alert = two postMessages" + + def test_alert_filtered_to_3rd_escalation_only(self, monkeypatch): + """The details.json rows fed to build_escalation_alert must be pre-filtered + to category == '3rd Escalation'. WO-003 (Schedule Confirmed) must not + appear in the alert message.""" + fake_client = self._make_fake_client() + + def fake_read(dt, name): + if name == "summary.json": + return SUMMARY_WITH_THIRDS + if name == "details.json": + return DETAILS_WITH_THIRDS + return None + + monkeypatch.setattr(slackio, "read_meta_json", fake_read) + monkeypatch.setattr(slackio, "web_client", lambda: fake_client) + monkeypatch.setattr(slackio, "channel_id", lambda: "C12345") + monkeypatch.setattr( + slackio, "get_dashboard_url", lambda: "https://grafana.example.com" + ) + + sp_handler.handler({"dt": "2026-05-28"}, None) + + # The alert is the second postMessage call. + assert len(fake_client.calls) == 2 + alert_call = fake_client.calls[1] + alert_blocks = alert_call["blocks"] + + # Serialise blocks to text for easy searching. + alert_text = json.dumps(alert_blocks) + + # Both 3rd-escalation WOs must be present. + assert "WO-001" in alert_text + assert "WO-002" in alert_text + # The non-3rd WO must NOT appear in the alert. + assert "WO-003" not in alert_text + + +# --------------------------------------------------------------------------- +# Section 3: slackio unit tests +# --------------------------------------------------------------------------- + + +class TestSlackioReadMetaJson: + def test_returns_none_on_no_such_key(self, monkeypatch): + """read_meta_json must return None (not raise) when S3 returns NoSuchKey.""" + # Build a fake get_object that raises the real NoSuchKey exception. + NoSuchKey = slackio._s3.exceptions.NoSuchKey + + def fake_get_object(Bucket, Key): + raise NoSuchKey( + {"Error": {"Code": "NoSuchKey", "Message": "not found"}}, + "GetObject", + ) + + monkeypatch.setattr(slackio._s3, "get_object", fake_get_object) + + result = slackio.read_meta_json("2026-05-28", "summary.json") + assert result is None + + def test_returns_parsed_json_on_success(self, monkeypatch): + """read_meta_json must parse and return the JSON body on a successful get.""" + payload = {"classified_total": 42} + + class _FakeBody: + def read(self): + return json.dumps(payload).encode("utf-8") + + monkeypatch.setattr( + slackio._s3, "get_object", lambda Bucket, Key: {"Body": _FakeBody()} + ) + + result = slackio.read_meta_json("2026-05-28", "summary.json") + assert result == payload + + +class TestSlackioCredentialsCaching: + """get_credentials() and get_dashboard_url() must cache their results and + only call the underlying boto3 client once per container lifetime.""" + + def setup_method(self): + # Reset module-level caches so each test starts from a cold state. + slackio._creds = None + slackio._dashboard_url = None + + def test_get_credentials_calls_secrets_once(self, monkeypatch): + """Calling get_credentials() twice must invoke get_secret_value exactly + once (the result is cached after the first call).""" + creds_payload = { + "botToken": "xoxb-test", + "signingSecret": "abc", + "channelId": "C99", + } + call_count = 0 + + def fake_get_secret(SecretId): + nonlocal call_count + call_count += 1 + return {"SecretString": json.dumps(creds_payload)} + + monkeypatch.setattr(slackio._secrets, "get_secret_value", fake_get_secret) + + first = slackio.get_credentials() + second = slackio.get_credentials() + + assert call_count == 1, "get_secret_value should be called only once" + assert first == creds_payload + assert second is first # same object from cache + + def test_get_dashboard_url_calls_ssm_once(self, monkeypatch): + """Calling get_dashboard_url() twice must invoke get_parameter exactly + once (the URL is cached after the first call).""" + expected_url = "https://grafana.example.com/d/abc" + call_count = 0 + + def fake_get_parameter(Name): + nonlocal call_count + call_count += 1 + return {"Parameter": {"Value": expected_url}} + + monkeypatch.setattr(slackio._ssm, "get_parameter", fake_get_parameter) + + first = slackio.get_dashboard_url() + second = slackio.get_dashboard_url() + + assert call_count == 1, "get_parameter should be called only once" + assert first == expected_url + assert second == expected_url From f47757adf0184c66e488be60cd38b32d44da5a35 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 15:02:47 -0400 Subject: [PATCH 09/12] Add coverage reporting via pytest-cov --- pyproject.toml | 1 + tests/requirements.txt | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 tests/requirements.txt diff --git a/pyproject.toml b/pyproject.toml index d86f343..54757f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,4 @@ [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["lambdas/classifier", "lambdas/slack_post", "cdk"] +addopts = "--cov=lambdas --cov-report=term-missing" diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..9955dec --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +pytest +pytest-cov From 0678f3f46f6c78cda05ba05548e2a784cf8fb077 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 15:05:29 -0400 Subject: [PATCH 10/12] Document CI-run test suite and coverage in README --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e585ad8..c5eedd1 100644 --- a/README.md +++ b/README.md @@ -162,18 +162,24 @@ Any `.xlsx`/`.csv` landing under `raw/` invokes the classifier. ## Local development - pyenv Python 3.12. `ruff check` + `ruff format --check` before pushing (hook-enforced). -- Tests: `python -m pytest tests/ -q` (60 tests — classifier smoke test against a real - export + offline `cdk.assertions` synth checks + Block Kit builders). No AWS needed. -- Smoke-test the classifier against a **real export** before declaring any - classification change done: `~/Downloads/_documents/Sheet1-1.xlsx`. -- `cdk synth` must pass in CI before merge (**Docker required** — Lambda deps are - bundled for ARM64). +- Tests: `python -m pytest tests/ -q` (~158 tests — classifier rule ladder + handler + transforms + Haiku fallback, Block Kit builders, the signature-verified Slack + interactions endpoint, slack-post orchestration, and offline `cdk.assertions` synth + checks). No AWS needed — external boundaries are monkeypatched. **Tests run in CI** + (`ci.yaml` sets `run-tests: true`); coverage is reported via `pytest-cov` + (`tests/requirements.txt`, ~94%, non-gating). +- The classification quality gate (deterministic "Other" share) runs in CI against a + committed synthetic fixture `tests/fixtures/sample_export.csv`. Additionally, + smoke-test against the **real export** before declaring any classification change + done: `~/Downloads/_documents/Sheet1-1.xlsx` (skips automatically when absent). +- `cdk synth` must pass in CI before merge (**Docker + QEMU** — Lambda deps are + bundled for ARM64; `ci.yaml` sets `enable-qemu: true`). ## Deployment CI/CD via the org reusable workflows (no manual prod deploys in steady state): -- **CI** (`.github/workflows/ci.yaml`) → `ci-python-sam.yaml@main`: ruff + `cdk synth`. Runs on PRs into `main`. +- **CI** (`.github/workflows/ci.yaml`) → `ci-python-sam.yaml@main`: ruff + `pytest` + `cdk synth` (QEMU-enabled). Runs on PRs into `main`. - **Deploy** (`.github/workflows/deploy.yaml`) → `cd-cdk.yaml@main`: OIDC assume-role, `cdk deploy --all`, single-flight concurrency. Runs on push to `main`. Stack name/region/account: `apm-wo-analysis-{pipeline,grafana}` / us-east-1 / 328440206208. From 17ce1b918b4db4cb5866aef76fab80963fc5f853 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 15:11:19 -0400 Subject: [PATCH 11/12] Install boto3/pandas/awswrangler as test deps so handler tests import in CI --- tests/requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 9955dec..e7c049e 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,8 @@ +# Test-only deps. boto3/pandas/awswrangler are NOT in the Lambda packages +# (they come from the SDK-for-pandas layer + runtime), but the handler/slackio +# modules import them at load time, so they're needed to import-and-test here. pytest pytest-cov +boto3 +pandas +awswrangler From 094c2123236a9a2d17600b30b7e37550f6b3abe7 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 15:16:24 -0400 Subject: [PATCH 12/12] Set default AWS region in conftest so boto3 clients construct in CI --- tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 90357e0..9b5abf1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,11 @@ Sets dummy environment variables at import time so modules that read ``os.environ`` at module load (slackio, interactions, handler) can be imported without real AWS credentials. Real values are monkeypatched per-test where needed. + +A default AWS region is set too: the handler/slackio modules construct +``boto3.client(...)`` at import time, which raises ``NoRegionError`` on a CI +runner with no AWS config. Client construction is offline; every real AWS call +is monkeypatched in the tests. """ import os @@ -11,3 +16,5 @@ import os os.environ.setdefault("SLACK_SECRET_NAME", "test-slack-secret") os.environ.setdefault("DASHBOARD_URL_PARAM", "/test/dashboard-url") os.environ.setdefault("ANALYTICS_BUCKET", "test-bucket") +os.environ.setdefault("AWS_DEFAULT_REGION", "us-east-1") +os.environ.setdefault("AWS_REGION", "us-east-1")